From b7d303bf54e2a11ee7fd13a21655d2b2f197095a Mon Sep 17 00:00:00 2001 From: devitway Date: Mon, 16 Mar 2026 13:27:37 +0000 Subject: [PATCH] feat: nora mirror CLI + systemd + install script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nora mirror: - Pre-fetch dependencies through NORA proxy cache - npm: --lockfile (v1/v2/v3) and --packages with --all-versions - pip: requirements.txt parser - cargo: Cargo.lock parser - maven: dependency:list output parser - Concurrent downloads (--concurrency, default 8) - Progress bar with indicatif - Health check before start dist/: - nora.service — systemd unit with security hardening - nora.env.example — environment configuration template - install.sh — automated install (binary + user + systemd + config) Tested: 103 tests pass, 0 clippy warnings, cargo audit clean. Smoke: mirrored 70 npm packages from real lockfile in 5.4s. --- dist/install.sh | 111 +++++++++++ dist/nora.env.example | 31 +++ dist/nora.service | 28 +++ nora-registry/src/main.rs | 22 +++ nora-registry/src/mirror/mod.rs | 325 ++++++++++++++++++++++++++++++++ nora-registry/src/mirror/npm.rs | 323 +++++++++++++++++++++++++++++++ 6 files changed, 840 insertions(+) create mode 100755 dist/install.sh create mode 100644 dist/nora.env.example create mode 100644 dist/nora.service create mode 100644 nora-registry/src/mirror/mod.rs create mode 100644 nora-registry/src/mirror/npm.rs diff --git a/dist/install.sh b/dist/install.sh new file mode 100755 index 0000000..857800c --- /dev/null +++ b/dist/install.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +# NORA Artifact Registry — install script +# Usage: curl -fsSL https://getnora.io/install.sh | bash + +VERSION="${NORA_VERSION:-latest}" +ARCH=$(uname -m) +OS=$(uname -s | tr '[:upper:]' '[:lower:]') +INSTALL_DIR="/usr/local/bin" +CONFIG_DIR="/etc/nora" +DATA_DIR="/var/lib/nora" +LOG_DIR="/var/log/nora" + +case "$ARCH" in + x86_64|amd64) ARCH="x86_64" ;; + aarch64|arm64) ARCH="aarch64" ;; + *) echo "Unsupported architecture: $ARCH"; exit 1 ;; +esac + +echo "Installing NORA ($OS/$ARCH)..." + +# Download binary +if [ "$VERSION" = "latest" ]; then + DOWNLOAD_URL="https://github.com/getnora-io/nora/releases/latest/download/nora-${OS}-${ARCH}" +else + DOWNLOAD_URL="https://github.com/getnora-io/nora/releases/download/${VERSION}/nora-${OS}-${ARCH}" +fi + +echo "Downloading from $DOWNLOAD_URL..." +if command -v curl &>/dev/null; then + curl -fsSL -o /tmp/nora "$DOWNLOAD_URL" +elif command -v wget &>/dev/null; then + wget -qO /tmp/nora "$DOWNLOAD_URL" +else + echo "Error: curl or wget required"; exit 1 +fi + +chmod +x /tmp/nora +sudo mv /tmp/nora "$INSTALL_DIR/nora" + +# Create system user +if ! id nora &>/dev/null; then + sudo useradd --system --shell /usr/sbin/nologin --home-dir "$DATA_DIR" --create-home nora + echo "Created system user: nora" +fi + +# Create directories +sudo mkdir -p "$CONFIG_DIR" "$DATA_DIR" "$LOG_DIR" +sudo chown nora:nora "$DATA_DIR" "$LOG_DIR" + +# Install default config if not exists +if [ ! -f "$CONFIG_DIR/nora.env" ]; then + cat > /tmp/nora.env << 'ENVEOF' +NORA_HOST=0.0.0.0 +NORA_PORT=4000 +NORA_STORAGE_PATH=/var/lib/nora +ENVEOF + sudo mv /tmp/nora.env "$CONFIG_DIR/nora.env" + sudo chmod 600 "$CONFIG_DIR/nora.env" + sudo chown nora:nora "$CONFIG_DIR/nora.env" + echo "Created default config: $CONFIG_DIR/nora.env" +fi + +# Install systemd service +if [ -d /etc/systemd/system ]; then + cat > /tmp/nora.service << 'SVCEOF' +[Unit] +Description=NORA Artifact Registry +Documentation=https://getnora.dev +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=nora +Group=nora +ExecStart=/usr/local/bin/nora serve +WorkingDirectory=/etc/nora +Restart=on-failure +RestartSec=5 +LimitNOFILE=65535 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/nora /var/log/nora +PrivateTmp=true +EnvironmentFile=-/etc/nora/nora.env + +[Install] +WantedBy=multi-user.target +SVCEOF + sudo mv /tmp/nora.service /etc/systemd/system/nora.service + sudo systemctl daemon-reload + sudo systemctl enable nora + echo "Installed systemd service: nora" +fi + +echo "" +echo "NORA installed successfully!" +echo "" +echo " Binary: $INSTALL_DIR/nora" +echo " Config: $CONFIG_DIR/nora.env" +echo " Data: $DATA_DIR" +echo " Version: $(nora --version 2>/dev/null || echo 'unknown')" +echo "" +echo "Next steps:" +echo " 1. Edit $CONFIG_DIR/nora.env" +echo " 2. sudo systemctl start nora" +echo " 3. curl http://localhost:4000/health" +echo "" diff --git a/dist/nora.env.example b/dist/nora.env.example new file mode 100644 index 0000000..03fb9b9 --- /dev/null +++ b/dist/nora.env.example @@ -0,0 +1,31 @@ +# NORA configuration — environment variables +# Copy to /etc/nora/nora.env and adjust + +# Server +NORA_HOST=0.0.0.0 +NORA_PORT=4000 +# NORA_PUBLIC_URL=https://registry.example.com + +# Storage +NORA_STORAGE_PATH=/var/lib/nora +# NORA_STORAGE_MODE=s3 +# NORA_STORAGE_S3_URL=http://minio:9000 +# NORA_STORAGE_BUCKET=registry + +# Auth (optional) +# NORA_AUTH_ENABLED=true +# NORA_AUTH_HTPASSWD_FILE=/etc/nora/users.htpasswd + +# Rate limiting +# NORA_RATE_LIMIT_ENABLED=true + +# npm proxy +# NORA_NPM_PROXY=https://registry.npmjs.org +# NORA_NPM_METADATA_TTL=300 +# NORA_NPM_PROXY_AUTH=user:pass + +# PyPI proxy +# NORA_PYPI_PROXY=https://pypi.org/simple/ + +# Docker upstreams +# NORA_DOCKER_UPSTREAMS=https://registry-1.docker.io diff --git a/dist/nora.service b/dist/nora.service new file mode 100644 index 0000000..6ae146b --- /dev/null +++ b/dist/nora.service @@ -0,0 +1,28 @@ +[Unit] +Description=NORA Artifact Registry +Documentation=https://getnora.dev +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=nora +Group=nora +ExecStart=/usr/local/bin/nora serve +WorkingDirectory=/etc/nora +Restart=on-failure +RestartSec=5 +LimitNOFILE=65535 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/nora /var/log/nora +PrivateTmp=true + +# Environment +EnvironmentFile=-/etc/nora/nora.env + +[Install] +WantedBy=multi-user.target diff --git a/nora-registry/src/main.rs b/nora-registry/src/main.rs index 0ee3c65..9066083 100644 --- a/nora-registry/src/main.rs +++ b/nora-registry/src/main.rs @@ -12,6 +12,7 @@ mod gc; mod health; mod metrics; mod migrate; +mod mirror; mod openapi; mod rate_limit; mod registry; @@ -82,6 +83,17 @@ enum Commands { #[arg(long, default_value = "false")] dry_run: bool, }, + /// Pre-fetch dependencies through NORA proxy cache + Mirror { + #[command(subcommand)] + format: mirror::MirrorFormat, + /// NORA registry URL + #[arg(long, default_value = "http://localhost:4000", global = true)] + registry: String, + /// Max concurrent downloads + #[arg(long, default_value = "8", global = true)] + concurrency: usize, + }, } pub struct AppState { @@ -164,6 +176,16 @@ async fn main() { println!("\nRun without --dry-run to delete orphaned blobs."); } } + Some(Commands::Mirror { + format, + registry, + concurrency, + }) => { + if let Err(e) = mirror::run_mirror(format, ®istry, concurrency).await { + error!("Mirror failed: {}", e); + std::process::exit(1); + } + } Some(Commands::Migrate { from, to, dry_run }) => { let source = match from.as_str() { "local" => Storage::new_local(&config.storage.path), diff --git a/nora-registry/src/mirror/mod.rs b/nora-registry/src/mirror/mod.rs new file mode 100644 index 0000000..6f226de --- /dev/null +++ b/nora-registry/src/mirror/mod.rs @@ -0,0 +1,325 @@ +// Copyright (c) 2026 Volkov Pavel | DevITWay +// SPDX-License-Identifier: MIT + +//! `nora mirror` — pre-fetch dependencies through NORA proxy cache. + +mod npm; + +use clap::Subcommand; +use indicatif::{ProgressBar, ProgressStyle}; +use std::path::PathBuf; +use std::time::Instant; + +#[derive(Subcommand)] +pub enum MirrorFormat { + /// Mirror npm packages + Npm { + /// Path to package-lock.json (v1/v2/v3) + #[arg(long, conflicts_with = "packages")] + lockfile: Option, + /// Comma-separated package names + #[arg(long, conflicts_with = "lockfile", value_delimiter = ',')] + packages: Option>, + /// Fetch all versions (only with --packages) + #[arg(long)] + all_versions: bool, + }, + /// Mirror Python packages + Pip { + /// Path to requirements.txt + #[arg(long)] + lockfile: PathBuf, + }, + /// Mirror Cargo crates + Cargo { + /// Path to Cargo.lock + #[arg(long)] + lockfile: PathBuf, + }, + /// Mirror Maven artifacts + Maven { + /// Path to dependency list (mvn dependency:list output) + #[arg(long)] + lockfile: PathBuf, + }, +} + +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +pub struct MirrorTarget { + pub name: String, + pub version: String, +} + +pub struct MirrorResult { + pub total: usize, + pub fetched: usize, + pub failed: usize, + pub bytes: u64, +} + +pub fn create_progress_bar(total: u64) -> ProgressBar { + let pb = ProgressBar::new(total); + pb.set_style( + ProgressStyle::default_bar() + .template( + "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}", + ) + .unwrap() + .progress_chars("=>-"), + ); + pb +} + +pub async fn run_mirror( + format: MirrorFormat, + registry: &str, + concurrency: usize, +) -> Result<(), String> { + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(120)) + .build() + .map_err(|e| format!("Failed to create HTTP client: {}", e))?; + + // Health check + let health_url = format!("{}/health", registry.trim_end_matches('/')); + match client.get(&health_url).send().await { + Ok(r) if r.status().is_success() => {} + _ => { + return Err(format!( + "Cannot connect to NORA at {}. Is `nora serve` running?", + registry + )) + } + } + + let start = Instant::now(); + + let result = match format { + MirrorFormat::Npm { + lockfile, + packages, + all_versions, + } => { + npm::run_npm_mirror( + &client, + registry, + lockfile, + packages, + all_versions, + concurrency, + ) + .await? + } + MirrorFormat::Pip { lockfile } => { + mirror_lockfile(&client, registry, "pip", &lockfile).await? + } + MirrorFormat::Cargo { lockfile } => { + mirror_lockfile(&client, registry, "cargo", &lockfile).await? + } + MirrorFormat::Maven { lockfile } => { + mirror_lockfile(&client, registry, "maven", &lockfile).await? + } + }; + + let elapsed = start.elapsed(); + println!("\nMirror complete:"); + println!(" Total: {}", result.total); + println!(" Fetched: {}", result.fetched); + println!(" Failed: {}", result.failed); + println!(" Size: {:.1} MB", result.bytes as f64 / 1_048_576.0); + println!(" Time: {:.1}s", elapsed.as_secs_f64()); + + if result.failed > 0 { + Err(format!("{} packages failed to mirror", result.failed)) + } else { + Ok(()) + } +} + +async fn mirror_lockfile( + client: &reqwest::Client, + registry: &str, + format: &str, + lockfile: &PathBuf, +) -> Result { + let content = std::fs::read_to_string(lockfile) + .map_err(|e| format!("Cannot read {}: {}", lockfile.display(), e))?; + + let targets = match format { + "pip" => parse_requirements_txt(&content), + "cargo" => parse_cargo_lock(&content)?, + "maven" => parse_maven_deps(&content), + _ => vec![], + }; + + if targets.is_empty() { + println!("No packages found in {}", lockfile.display()); + return Ok(MirrorResult { + total: 0, + fetched: 0, + failed: 0, + bytes: 0, + }); + } + + let pb = create_progress_bar(targets.len() as u64); + let base = registry.trim_end_matches('/'); + let mut fetched = 0; + let mut failed = 0; + let mut bytes = 0u64; + + for target in &targets { + let url = match format { + "pip" => format!("{}/simple/{}/", base, target.name), + "cargo" => format!( + "{}/cargo/api/v1/crates/{}/{}/download", + base, target.name, target.version + ), + "maven" => { + let parts: Vec<&str> = target.name.split(':').collect(); + if parts.len() == 2 { + let group_path = parts[0].replace('.', "/"); + format!( + "{}/maven2/{}/{}/{}/{}-{}.jar", + base, group_path, parts[1], target.version, parts[1], target.version + ) + } else { + pb.inc(1); + failed += 1; + continue; + } + } + _ => continue, + }; + + match client.get(&url).send().await { + Ok(r) if r.status().is_success() => { + if let Ok(body) = r.bytes().await { + bytes += body.len() as u64; + } + fetched += 1; + } + _ => failed += 1, + } + + pb.set_message(format!("{}@{}", target.name, target.version)); + pb.inc(1); + } + + pb.finish_with_message("done"); + Ok(MirrorResult { + total: targets.len(), + fetched, + failed, + bytes, + }) +} + +fn parse_requirements_txt(content: &str) -> Vec { + content + .lines() + .filter(|l| !l.trim().is_empty() && !l.starts_with('#') && !l.starts_with('-')) + .filter_map(|line| { + let line = line.split('#').next().unwrap().trim(); + if let Some((name, version)) = line.split_once("==") { + Some(MirrorTarget { + name: name.trim().to_string(), + version: version.trim().to_string(), + }) + } else { + let name = line.split(['>', '<', '=', '!', '~', ';']).next()?.trim(); + if name.is_empty() { + None + } else { + Some(MirrorTarget { + name: name.to_string(), + version: "latest".to_string(), + }) + } + } + }) + .collect() +} + +fn parse_cargo_lock(content: &str) -> Result, String> { + let lock: toml::Value = + toml::from_str(content).map_err(|e| format!("Invalid Cargo.lock: {}", e))?; + let packages = lock + .get("package") + .and_then(|p| p.as_array()) + .cloned() + .unwrap_or_default(); + Ok(packages + .iter() + .filter(|p| { + p.get("source") + .and_then(|s| s.as_str()) + .map(|s| s.starts_with("registry+")) + .unwrap_or(false) + }) + .filter_map(|p| { + let name = p.get("name")?.as_str()?.to_string(); + let version = p.get("version")?.as_str()?.to_string(); + Some(MirrorTarget { name, version }) + }) + .collect()) +} + +fn parse_maven_deps(content: &str) -> Vec { + content + .lines() + .filter_map(|line| { + let line = line.trim().trim_start_matches("[INFO]").trim(); + let parts: Vec<&str> = line.split(':').collect(); + if parts.len() >= 4 { + let name = format!("{}:{}", parts[0], parts[1]); + let version = parts[3].to_string(); + Some(MirrorTarget { name, version }) + } else { + None + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_requirements_txt() { + let content = "flask==2.3.0\nrequests>=2.28.0\n# comment\nnumpy==1.24.3\n"; + let targets = parse_requirements_txt(content); + assert_eq!(targets.len(), 3); + assert_eq!(targets[0].name, "flask"); + assert_eq!(targets[0].version, "2.3.0"); + assert_eq!(targets[1].name, "requests"); + assert_eq!(targets[1].version, "latest"); + } + + #[test] + fn test_parse_cargo_lock() { + let content = "\ +[[package]] +name = \"serde\" +version = \"1.0.197\" +source = \"registry+https://github.com/rust-lang/crates.io-index\" + +[[package]] +name = \"my-local-crate\" +version = \"0.1.0\" +"; + let targets = parse_cargo_lock(content).unwrap(); + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].name, "serde"); + } + + #[test] + fn test_parse_maven_deps() { + let content = "[INFO] org.apache.commons:commons-lang3:jar:3.12.0:compile\n"; + let targets = parse_maven_deps(content); + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].name, "org.apache.commons:commons-lang3"); + assert_eq!(targets[0].version, "3.12.0"); + } +} diff --git a/nora-registry/src/mirror/npm.rs b/nora-registry/src/mirror/npm.rs new file mode 100644 index 0000000..ef158c1 --- /dev/null +++ b/nora-registry/src/mirror/npm.rs @@ -0,0 +1,323 @@ +// Copyright (c) 2026 Volkov Pavel | DevITWay +// SPDX-License-Identifier: MIT + +//! npm lockfile parser + mirror logic. + +use super::{create_progress_bar, MirrorResult, MirrorTarget}; +use std::collections::HashSet; +use std::path::PathBuf; +use tokio::sync::Semaphore; + +/// Entry point for npm mirroring +pub async fn run_npm_mirror( + client: &reqwest::Client, + registry: &str, + lockfile: Option, + packages: Option>, + all_versions: bool, + concurrency: usize, +) -> Result { + let targets = if let Some(path) = lockfile { + let content = std::fs::read_to_string(&path) + .map_err(|e| format!("Cannot read {}: {}", path.display(), e))?; + parse_npm_lockfile(&content)? + } else if let Some(names) = packages { + resolve_npm_packages(client, registry, &names, all_versions).await? + } else { + return Err("Specify --lockfile or --packages".to_string()); + }; + + if targets.is_empty() { + println!("No npm packages to mirror"); + return Ok(MirrorResult { + total: 0, + fetched: 0, + failed: 0, + bytes: 0, + }); + } + + println!( + "Mirroring {} npm packages via {}...", + targets.len(), + registry + ); + mirror_npm_packages(client, registry, &targets, concurrency).await +} + +/// Parse package-lock.json (v1, v2, v3) +fn parse_npm_lockfile(content: &str) -> Result, String> { + let json: serde_json::Value = + serde_json::from_str(content).map_err(|e| format!("Invalid JSON: {}", e))?; + + let version = json + .get("lockfileVersion") + .and_then(|v| v.as_u64()) + .unwrap_or(1); + + let mut seen = HashSet::new(); + let mut targets = Vec::new(); + + if version >= 2 { + // v2/v3: use "packages" object + if let Some(packages) = json.get("packages").and_then(|p| p.as_object()) { + for (key, pkg) in packages { + if key.is_empty() { + continue; // root package + } + if let Some(name) = extract_package_name(key) { + if let Some(ver) = pkg.get("version").and_then(|v| v.as_str()) { + let pair = (name.to_string(), ver.to_string()); + if seen.insert(pair.clone()) { + targets.push(MirrorTarget { + name: pair.0, + version: pair.1, + }); + } + } + } + } + } + } + + if version == 1 || targets.is_empty() { + // v1 fallback: recursive "dependencies" + if let Some(deps) = json.get("dependencies").and_then(|d| d.as_object()) { + parse_v1_deps(deps, &mut targets, &mut seen); + } + } + + Ok(targets) +} + +/// Extract package name from lockfile key like "node_modules/@babel/core" +fn extract_package_name(key: &str) -> Option<&str> { + // Handle nested: "node_modules/foo/node_modules/@scope/bar" → "@scope/bar" + let last_nm = key.rfind("node_modules/")?; + let after = &key[last_nm + "node_modules/".len()..]; + let name = after.trim_end_matches('/'); + if name.is_empty() { + None + } else { + Some(name) + } +} + +/// Recursively parse v1 lockfile "dependencies" +fn parse_v1_deps( + deps: &serde_json::Map, + targets: &mut Vec, + seen: &mut HashSet<(String, String)>, +) { + for (name, pkg) in deps { + if let Some(ver) = pkg.get("version").and_then(|v| v.as_str()) { + let pair = (name.clone(), ver.to_string()); + if seen.insert(pair.clone()) { + targets.push(MirrorTarget { + name: pair.0, + version: pair.1, + }); + } + } + // Recurse into nested dependencies + if let Some(nested) = pkg.get("dependencies").and_then(|d| d.as_object()) { + parse_v1_deps(nested, targets, seen); + } + } +} + +/// Resolve --packages list by fetching metadata from NORA +async fn resolve_npm_packages( + client: &reqwest::Client, + registry: &str, + names: &[String], + all_versions: bool, +) -> Result, String> { + let base = registry.trim_end_matches('/'); + let mut targets = Vec::new(); + + for name in names { + let url = format!("{}/npm/{}", base, name); + let resp = client.get(&url).send().await.map_err(|e| e.to_string())?; + + if !resp.status().is_success() { + eprintln!("Warning: {} not found (HTTP {})", name, resp.status()); + continue; + } + + let json: serde_json::Value = resp.json().await.map_err(|e| e.to_string())?; + + if all_versions { + if let Some(versions) = json.get("versions").and_then(|v| v.as_object()) { + for ver in versions.keys() { + targets.push(MirrorTarget { + name: name.clone(), + version: ver.clone(), + }); + } + } + } else { + // Just latest + let latest = json + .get("dist-tags") + .and_then(|d| d.get("latest")) + .and_then(|v| v.as_str()) + .unwrap_or("latest"); + targets.push(MirrorTarget { + name: name.clone(), + version: latest.to_string(), + }); + } + } + + Ok(targets) +} + +/// Fetch packages through NORA (triggers proxy cache) +async fn mirror_npm_packages( + client: &reqwest::Client, + registry: &str, + targets: &[MirrorTarget], + concurrency: usize, +) -> Result { + let base = registry.trim_end_matches('/'); + let pb = create_progress_bar(targets.len() as u64); + let sem = std::sync::Arc::new(Semaphore::new(concurrency)); + + // Deduplicate metadata fetches (one per package name) + let unique_names: HashSet<&str> = targets.iter().map(|t| t.name.as_str()).collect(); + pb.set_message("fetching metadata..."); + for name in &unique_names { + let url = format!("{}/npm/{}", base, name); + let _ = client.get(&url).send().await; // trigger metadata cache + } + + // Fetch tarballs concurrently + let fetched = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let failed = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let bytes = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + + let mut handles = Vec::new(); + + for target in targets { + let permit = sem.clone().acquire_owned().await.unwrap(); + let client = client.clone(); + let pb = pb.clone(); + let fetched = fetched.clone(); + let failed = failed.clone(); + let bytes = bytes.clone(); + + let short_name = target.name.split('/').next_back().unwrap_or(&target.name); + let tarball_url = format!( + "{}/npm/{}/-/{}-{}.tgz", + base, target.name, short_name, target.version + ); + let label = format!("{}@{}", target.name, target.version); + + handles.push(tokio::spawn(async move { + let _permit = permit; + match client.get(&tarball_url).send().await { + Ok(r) if r.status().is_success() => { + if let Ok(body) = r.bytes().await { + bytes.fetch_add(body.len() as u64, std::sync::atomic::Ordering::Relaxed); + } + fetched.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + _ => { + failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + } + pb.set_message(label); + pb.inc(1); + })); + } + + for h in handles { + let _ = h.await; + } + + pb.finish_with_message("done"); + + Ok(MirrorResult { + total: targets.len(), + fetched: fetched.load(std::sync::atomic::Ordering::Relaxed), + failed: failed.load(std::sync::atomic::Ordering::Relaxed), + bytes: bytes.load(std::sync::atomic::Ordering::Relaxed), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_package_name() { + assert_eq!(extract_package_name("node_modules/lodash"), Some("lodash")); + assert_eq!( + extract_package_name("node_modules/@babel/core"), + Some("@babel/core") + ); + assert_eq!( + extract_package_name("node_modules/foo/node_modules/bar"), + Some("bar") + ); + assert_eq!( + extract_package_name("node_modules/foo/node_modules/@types/node"), + Some("@types/node") + ); + assert_eq!(extract_package_name(""), None); + } + + #[test] + fn test_parse_lockfile_v3() { + let content = r#"{ + "lockfileVersion": 3, + "packages": { + "": { "name": "test" }, + "node_modules/lodash": { "version": "4.17.21" }, + "node_modules/@babel/core": { "version": "7.26.0" }, + "node_modules/@babel/core/node_modules/semver": { "version": "6.3.1" } + } + }"#; + let targets = parse_npm_lockfile(content).unwrap(); + assert_eq!(targets.len(), 3); + let names: HashSet<&str> = targets.iter().map(|t| t.name.as_str()).collect(); + assert!(names.contains("lodash")); + assert!(names.contains("@babel/core")); + assert!(names.contains("semver")); + } + + #[test] + fn test_parse_lockfile_v1() { + let content = r#"{ + "lockfileVersion": 1, + "dependencies": { + "express": { + "version": "4.18.2", + "dependencies": { + "accepts": { "version": "1.3.8" } + } + } + } + }"#; + let targets = parse_npm_lockfile(content).unwrap(); + assert_eq!(targets.len(), 2); + assert_eq!(targets[0].name, "express"); + assert_eq!(targets[1].name, "accepts"); + } + + #[test] + fn test_deduplication() { + let content = r#"{ + "lockfileVersion": 3, + "packages": { + "": {}, + "node_modules/debug": { "version": "4.3.4" }, + "node_modules/express/node_modules/debug": { "version": "4.3.4" } + } + }"#; + let targets = parse_npm_lockfile(content).unwrap(); + assert_eq!(targets.len(), 1); // deduplicated + assert_eq!(targets[0].name, "debug"); + } +}