mirror of
https://github.com/getnora-io/nora.git
synced 2026-04-12 23:10:32 +00:00
quality: MSRV, tarpaulin config, proptest for parsers (#84)
* fix: proxy dedup, multi-registry GC, TOCTOU and credential hygiene - Deduplicate proxy_fetch/proxy_fetch_text into generic proxy_fetch_core with response extractor closure (removes ~50 lines of copy-paste) - GC now scans all registry prefixes, not just docker/ - Add tracing::warn to fire-and-forget cache writes in docker proxy - Mark S3 credentials as skip_serializing to prevent accidental leaks - Remove TOCTOU race in LocalStorage get/delete (redundant exists check) * chore: clean up root directory structure - Move Dockerfile.astra and Dockerfile.redos to deploy/ (niche builds should not clutter the project root) - Harden .gitignore to exclude session files, working notes, and internal review scripts * refactor(metrics): replace 13 atomic fields with CounterMap Per-registry download/upload counters were 13 individual AtomicU64 fields, each duplicated across new(), with_persistence(), save(), record_download(), record_upload(), and get_registry_* (6 touch points per counter). Adding a new registry required changes in 6+ places. Now uses CounterMap (HashMap<String, AtomicU64>) for per-registry counters. Adding a new registry = one entry in REGISTRIES const. Added Go registry to REGISTRIES, gaining go metrics for free. * quality: add MSRV, tarpaulin config, proptest for parsers - Set rust-version = 1.75 in workspace Cargo.toml (MSRV policy) - Add tarpaulin.toml: llvm engine, fail-under=25, json+html output - Add coverage/ to .gitignore - Update CI to use tarpaulin.toml instead of inline flags - Add proptest dev-dependency and property tests: - validation.rs: 16 tests (never-panics + invariants for all 4 validators) - pypi.rs: 5 tests (extract_filename never-panics + format assertions) * test: add unit tests for 14 modules, coverage 21% → 30% Add 149 new tests across auth, backup, gc, metrics, mirror parsers, docker (manifest detection, session cleanup, metadata serde), docker_auth (token cache), maven, npm, pypi (normalize, rewrite, extract), raw (content-type guessing), request_id, and s3 (URI encoding). Update tarpaulin.toml: raise fail-under to 30, exclude UI/main from coverage reporting as they require integration tests. * bench: add criterion benchmarks for validation and manifest parsing Add parsing benchmark suite with 14 benchmarks covering: - Storage key, Docker name, digest, and reference validation - Docker manifest media type detection (v2, OCI index, minimal, invalid) Run with: cargo bench --package nora-registry --bench parsing * test: add 48 integration tests via tower oneshot Add integration tests for all HTTP handlers: - health (3), raw (7), cargo (4), maven (4), request_id (2) - pypi (5), npm (5), docker (12), auth (6) Create test_helpers.rs with TestContext pattern. Add tower and http-body-util dev-dependencies. Update tarpaulin fail-under 30 to 40. Coverage: 29.5% to 43.3% (2089/4825 lines) * fix: clean clippy warnings in tests, fix flaky audit test Add #[allow(clippy::unwrap_used)] to 18 test modules. Fix 3 additional clippy lints: writeln_empty_string, needless_update, unnecessary_get_then_check. Fix flaky audit test: replace single sleep(50ms) with retry loop (max 1s). Prefix unused token variable with underscore. cargo clippy --all-targets = 0 warnings (was 245 errors)
This commit is contained in:
GitHub
@@ -124,3 +124,198 @@ async fn collect_referenced_digests(storage: &Storage) -> HashSet<String> {
|
||||
|
||||
referenced
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_gc_result_defaults() {
|
||||
let result = GcResult {
|
||||
total_blobs: 0,
|
||||
referenced_blobs: 0,
|
||||
orphaned_blobs: 0,
|
||||
deleted_blobs: 0,
|
||||
orphan_keys: vec![],
|
||||
};
|
||||
assert_eq!(result.total_blobs, 0);
|
||||
assert!(result.orphan_keys.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_empty_storage() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::new_local(dir.path().join("data").to_str().unwrap());
|
||||
|
||||
let result = run_gc(&storage, true).await;
|
||||
assert_eq!(result.total_blobs, 0);
|
||||
assert_eq!(result.referenced_blobs, 0);
|
||||
assert_eq!(result.orphaned_blobs, 0);
|
||||
assert_eq!(result.deleted_blobs, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_no_orphans() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::new_local(dir.path().join("data").to_str().unwrap());
|
||||
|
||||
// Create a manifest that references a blob
|
||||
let manifest = serde_json::json!({
|
||||
"config": {"digest": "sha256:configabc"},
|
||||
"layers": [{"digest": "sha256:layer111", "size": 100}]
|
||||
});
|
||||
storage
|
||||
.put(
|
||||
"docker/test/manifests/latest.json",
|
||||
manifest.to_string().as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/test/blobs/sha256:configabc", b"config-data")
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/test/blobs/sha256:layer111", b"layer-data")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = run_gc(&storage, true).await;
|
||||
assert_eq!(result.total_blobs, 2);
|
||||
assert_eq!(result.orphaned_blobs, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_finds_orphans_dry_run() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::new_local(dir.path().join("data").to_str().unwrap());
|
||||
|
||||
// Create a manifest referencing only one blob
|
||||
let manifest = serde_json::json!({
|
||||
"config": {"digest": "sha256:configabc"},
|
||||
"layers": [{"digest": "sha256:layer111", "size": 100}]
|
||||
});
|
||||
storage
|
||||
.put(
|
||||
"docker/test/manifests/latest.json",
|
||||
manifest.to_string().as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/test/blobs/sha256:configabc", b"config-data")
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/test/blobs/sha256:layer111", b"layer-data")
|
||||
.await
|
||||
.unwrap();
|
||||
// Orphan blob (not referenced)
|
||||
storage
|
||||
.put("docker/test/blobs/sha256:orphan999", b"orphan-data")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = run_gc(&storage, true).await;
|
||||
assert_eq!(result.total_blobs, 3);
|
||||
assert_eq!(result.orphaned_blobs, 1);
|
||||
assert_eq!(result.deleted_blobs, 0); // dry run
|
||||
assert!(result.orphan_keys[0].contains("orphan999"));
|
||||
|
||||
// Verify orphan still exists (dry run)
|
||||
assert!(storage
|
||||
.get("docker/test/blobs/sha256:orphan999")
|
||||
.await
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_deletes_orphans() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::new_local(dir.path().join("data").to_str().unwrap());
|
||||
|
||||
let manifest = serde_json::json!({
|
||||
"config": {"digest": "sha256:configabc"},
|
||||
"layers": []
|
||||
});
|
||||
storage
|
||||
.put(
|
||||
"docker/test/manifests/latest.json",
|
||||
manifest.to_string().as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/test/blobs/sha256:configabc", b"config")
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/test/blobs/sha256:orphan1", b"orphan")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = run_gc(&storage, false).await;
|
||||
assert_eq!(result.orphaned_blobs, 1);
|
||||
assert_eq!(result.deleted_blobs, 1);
|
||||
|
||||
// Verify orphan is gone
|
||||
assert!(storage
|
||||
.get("docker/test/blobs/sha256:orphan1")
|
||||
.await
|
||||
.is_err());
|
||||
// Referenced blob still exists
|
||||
assert!(storage
|
||||
.get("docker/test/blobs/sha256:configabc")
|
||||
.await
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_manifest_list_references() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::new_local(dir.path().join("data").to_str().unwrap());
|
||||
|
||||
// Multi-arch manifest list
|
||||
let manifest = serde_json::json!({
|
||||
"manifests": [
|
||||
{"digest": "sha256:platformA", "size": 100},
|
||||
{"digest": "sha256:platformB", "size": 200}
|
||||
]
|
||||
});
|
||||
storage
|
||||
.put(
|
||||
"docker/multi/manifests/latest.json",
|
||||
manifest.to_string().as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/multi/blobs/sha256:platformA", b"arch-a")
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.put("docker/multi/blobs/sha256:platformB", b"arch-b")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = run_gc(&storage, true).await;
|
||||
assert_eq!(result.orphaned_blobs, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gc_multi_registry_blobs() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::new_local(dir.path().join("data").to_str().unwrap());
|
||||
|
||||
// npm tarball (not referenced by Docker manifests => orphan candidate)
|
||||
storage
|
||||
.put("npm/lodash/tarballs/lodash-4.17.21.tgz", b"tarball-data")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = run_gc(&storage, true).await;
|
||||
// npm tarballs contain "tarballs/" which matches the filter
|
||||
assert_eq!(result.total_blobs, 1);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user