mirror of
https://github.com/getnora-io/nora.git
synced 2026-04-12 13:50:31 +00:00
feat: nora mirror CLI + systemd + install script
nora mirror: - Pre-fetch dependencies through NORA proxy cache - npm: --lockfile (v1/v2/v3) and --packages with --all-versions - pip: requirements.txt parser - cargo: Cargo.lock parser - maven: dependency:list output parser - Concurrent downloads (--concurrency, default 8) - Progress bar with indicatif - Health check before start dist/: - nora.service — systemd unit with security hardening - nora.env.example — environment configuration template - install.sh — automated install (binary + user + systemd + config) Tested: 103 tests pass, 0 clippy warnings, cargo audit clean. Smoke: mirrored 70 npm packages from real lockfile in 5.4s.
This commit is contained in:
@@ -12,6 +12,7 @@ mod gc;
|
||||
mod health;
|
||||
mod metrics;
|
||||
mod migrate;
|
||||
mod mirror;
|
||||
mod openapi;
|
||||
mod rate_limit;
|
||||
mod registry;
|
||||
@@ -82,6 +83,17 @@ enum Commands {
|
||||
#[arg(long, default_value = "false")]
|
||||
dry_run: bool,
|
||||
},
|
||||
/// Pre-fetch dependencies through NORA proxy cache
|
||||
Mirror {
|
||||
#[command(subcommand)]
|
||||
format: mirror::MirrorFormat,
|
||||
/// NORA registry URL
|
||||
#[arg(long, default_value = "http://localhost:4000", global = true)]
|
||||
registry: String,
|
||||
/// Max concurrent downloads
|
||||
#[arg(long, default_value = "8", global = true)]
|
||||
concurrency: usize,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct AppState {
|
||||
@@ -164,6 +176,16 @@ async fn main() {
|
||||
println!("\nRun without --dry-run to delete orphaned blobs.");
|
||||
}
|
||||
}
|
||||
Some(Commands::Mirror {
|
||||
format,
|
||||
registry,
|
||||
concurrency,
|
||||
}) => {
|
||||
if let Err(e) = mirror::run_mirror(format, ®istry, concurrency).await {
|
||||
error!("Mirror failed: {}", e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
Some(Commands::Migrate { from, to, dry_run }) => {
|
||||
let source = match from.as_str() {
|
||||
"local" => Storage::new_local(&config.storage.path),
|
||||
|
||||
325
nora-registry/src/mirror/mod.rs
Normal file
325
nora-registry/src/mirror/mod.rs
Normal file
@@ -0,0 +1,325 @@
|
||||
// Copyright (c) 2026 Volkov Pavel | DevITWay
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//! `nora mirror` — pre-fetch dependencies through NORA proxy cache.
|
||||
|
||||
mod npm;
|
||||
|
||||
use clap::Subcommand;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum MirrorFormat {
|
||||
/// Mirror npm packages
|
||||
Npm {
|
||||
/// Path to package-lock.json (v1/v2/v3)
|
||||
#[arg(long, conflicts_with = "packages")]
|
||||
lockfile: Option<PathBuf>,
|
||||
/// Comma-separated package names
|
||||
#[arg(long, conflicts_with = "lockfile", value_delimiter = ',')]
|
||||
packages: Option<Vec<String>>,
|
||||
/// Fetch all versions (only with --packages)
|
||||
#[arg(long)]
|
||||
all_versions: bool,
|
||||
},
|
||||
/// Mirror Python packages
|
||||
Pip {
|
||||
/// Path to requirements.txt
|
||||
#[arg(long)]
|
||||
lockfile: PathBuf,
|
||||
},
|
||||
/// Mirror Cargo crates
|
||||
Cargo {
|
||||
/// Path to Cargo.lock
|
||||
#[arg(long)]
|
||||
lockfile: PathBuf,
|
||||
},
|
||||
/// Mirror Maven artifacts
|
||||
Maven {
|
||||
/// Path to dependency list (mvn dependency:list output)
|
||||
#[arg(long)]
|
||||
lockfile: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Hash, Eq, PartialEq)]
|
||||
pub struct MirrorTarget {
|
||||
pub name: String,
|
||||
pub version: String,
|
||||
}
|
||||
|
||||
pub struct MirrorResult {
|
||||
pub total: usize,
|
||||
pub fetched: usize,
|
||||
pub failed: usize,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
pub fn create_progress_bar(total: u64) -> ProgressBar {
|
||||
let pb = ProgressBar::new(total);
|
||||
pb.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(
|
||||
"{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}",
|
||||
)
|
||||
.unwrap()
|
||||
.progress_chars("=>-"),
|
||||
);
|
||||
pb
|
||||
}
|
||||
|
||||
pub async fn run_mirror(
|
||||
format: MirrorFormat,
|
||||
registry: &str,
|
||||
concurrency: usize,
|
||||
) -> Result<(), String> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(120))
|
||||
.build()
|
||||
.map_err(|e| format!("Failed to create HTTP client: {}", e))?;
|
||||
|
||||
// Health check
|
||||
let health_url = format!("{}/health", registry.trim_end_matches('/'));
|
||||
match client.get(&health_url).send().await {
|
||||
Ok(r) if r.status().is_success() => {}
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"Cannot connect to NORA at {}. Is `nora serve` running?",
|
||||
registry
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let result = match format {
|
||||
MirrorFormat::Npm {
|
||||
lockfile,
|
||||
packages,
|
||||
all_versions,
|
||||
} => {
|
||||
npm::run_npm_mirror(
|
||||
&client,
|
||||
registry,
|
||||
lockfile,
|
||||
packages,
|
||||
all_versions,
|
||||
concurrency,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
MirrorFormat::Pip { lockfile } => {
|
||||
mirror_lockfile(&client, registry, "pip", &lockfile).await?
|
||||
}
|
||||
MirrorFormat::Cargo { lockfile } => {
|
||||
mirror_lockfile(&client, registry, "cargo", &lockfile).await?
|
||||
}
|
||||
MirrorFormat::Maven { lockfile } => {
|
||||
mirror_lockfile(&client, registry, "maven", &lockfile).await?
|
||||
}
|
||||
};
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
println!("\nMirror complete:");
|
||||
println!(" Total: {}", result.total);
|
||||
println!(" Fetched: {}", result.fetched);
|
||||
println!(" Failed: {}", result.failed);
|
||||
println!(" Size: {:.1} MB", result.bytes as f64 / 1_048_576.0);
|
||||
println!(" Time: {:.1}s", elapsed.as_secs_f64());
|
||||
|
||||
if result.failed > 0 {
|
||||
Err(format!("{} packages failed to mirror", result.failed))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn mirror_lockfile(
|
||||
client: &reqwest::Client,
|
||||
registry: &str,
|
||||
format: &str,
|
||||
lockfile: &PathBuf,
|
||||
) -> Result<MirrorResult, String> {
|
||||
let content = std::fs::read_to_string(lockfile)
|
||||
.map_err(|e| format!("Cannot read {}: {}", lockfile.display(), e))?;
|
||||
|
||||
let targets = match format {
|
||||
"pip" => parse_requirements_txt(&content),
|
||||
"cargo" => parse_cargo_lock(&content)?,
|
||||
"maven" => parse_maven_deps(&content),
|
||||
_ => vec![],
|
||||
};
|
||||
|
||||
if targets.is_empty() {
|
||||
println!("No packages found in {}", lockfile.display());
|
||||
return Ok(MirrorResult {
|
||||
total: 0,
|
||||
fetched: 0,
|
||||
failed: 0,
|
||||
bytes: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let pb = create_progress_bar(targets.len() as u64);
|
||||
let base = registry.trim_end_matches('/');
|
||||
let mut fetched = 0;
|
||||
let mut failed = 0;
|
||||
let mut bytes = 0u64;
|
||||
|
||||
for target in &targets {
|
||||
let url = match format {
|
||||
"pip" => format!("{}/simple/{}/", base, target.name),
|
||||
"cargo" => format!(
|
||||
"{}/cargo/api/v1/crates/{}/{}/download",
|
||||
base, target.name, target.version
|
||||
),
|
||||
"maven" => {
|
||||
let parts: Vec<&str> = target.name.split(':').collect();
|
||||
if parts.len() == 2 {
|
||||
let group_path = parts[0].replace('.', "/");
|
||||
format!(
|
||||
"{}/maven2/{}/{}/{}/{}-{}.jar",
|
||||
base, group_path, parts[1], target.version, parts[1], target.version
|
||||
)
|
||||
} else {
|
||||
pb.inc(1);
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
match client.get(&url).send().await {
|
||||
Ok(r) if r.status().is_success() => {
|
||||
if let Ok(body) = r.bytes().await {
|
||||
bytes += body.len() as u64;
|
||||
}
|
||||
fetched += 1;
|
||||
}
|
||||
_ => failed += 1,
|
||||
}
|
||||
|
||||
pb.set_message(format!("{}@{}", target.name, target.version));
|
||||
pb.inc(1);
|
||||
}
|
||||
|
||||
pb.finish_with_message("done");
|
||||
Ok(MirrorResult {
|
||||
total: targets.len(),
|
||||
fetched,
|
||||
failed,
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_requirements_txt(content: &str) -> Vec<MirrorTarget> {
|
||||
content
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty() && !l.starts_with('#') && !l.starts_with('-'))
|
||||
.filter_map(|line| {
|
||||
let line = line.split('#').next().unwrap().trim();
|
||||
if let Some((name, version)) = line.split_once("==") {
|
||||
Some(MirrorTarget {
|
||||
name: name.trim().to_string(),
|
||||
version: version.trim().to_string(),
|
||||
})
|
||||
} else {
|
||||
let name = line.split(['>', '<', '=', '!', '~', ';']).next()?.trim();
|
||||
if name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(MirrorTarget {
|
||||
name: name.to_string(),
|
||||
version: "latest".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_cargo_lock(content: &str) -> Result<Vec<MirrorTarget>, String> {
|
||||
let lock: toml::Value =
|
||||
toml::from_str(content).map_err(|e| format!("Invalid Cargo.lock: {}", e))?;
|
||||
let packages = lock
|
||||
.get("package")
|
||||
.and_then(|p| p.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
Ok(packages
|
||||
.iter()
|
||||
.filter(|p| {
|
||||
p.get("source")
|
||||
.and_then(|s| s.as_str())
|
||||
.map(|s| s.starts_with("registry+"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.filter_map(|p| {
|
||||
let name = p.get("name")?.as_str()?.to_string();
|
||||
let version = p.get("version")?.as_str()?.to_string();
|
||||
Some(MirrorTarget { name, version })
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn parse_maven_deps(content: &str) -> Vec<MirrorTarget> {
|
||||
content
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let line = line.trim().trim_start_matches("[INFO]").trim();
|
||||
let parts: Vec<&str> = line.split(':').collect();
|
||||
if parts.len() >= 4 {
|
||||
let name = format!("{}:{}", parts[0], parts[1]);
|
||||
let version = parts[3].to_string();
|
||||
Some(MirrorTarget { name, version })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_requirements_txt() {
|
||||
let content = "flask==2.3.0\nrequests>=2.28.0\n# comment\nnumpy==1.24.3\n";
|
||||
let targets = parse_requirements_txt(content);
|
||||
assert_eq!(targets.len(), 3);
|
||||
assert_eq!(targets[0].name, "flask");
|
||||
assert_eq!(targets[0].version, "2.3.0");
|
||||
assert_eq!(targets[1].name, "requests");
|
||||
assert_eq!(targets[1].version, "latest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_cargo_lock() {
|
||||
let content = "\
|
||||
[[package]]
|
||||
name = \"serde\"
|
||||
version = \"1.0.197\"
|
||||
source = \"registry+https://github.com/rust-lang/crates.io-index\"
|
||||
|
||||
[[package]]
|
||||
name = \"my-local-crate\"
|
||||
version = \"0.1.0\"
|
||||
";
|
||||
let targets = parse_cargo_lock(content).unwrap();
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert_eq!(targets[0].name, "serde");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_maven_deps() {
|
||||
let content = "[INFO] org.apache.commons:commons-lang3:jar:3.12.0:compile\n";
|
||||
let targets = parse_maven_deps(content);
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert_eq!(targets[0].name, "org.apache.commons:commons-lang3");
|
||||
assert_eq!(targets[0].version, "3.12.0");
|
||||
}
|
||||
}
|
||||
323
nora-registry/src/mirror/npm.rs
Normal file
323
nora-registry/src/mirror/npm.rs
Normal file
@@ -0,0 +1,323 @@
|
||||
// Copyright (c) 2026 Volkov Pavel | DevITWay
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//! npm lockfile parser + mirror logic.
|
||||
|
||||
use super::{create_progress_bar, MirrorResult, MirrorTarget};
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
/// Entry point for npm mirroring
|
||||
pub async fn run_npm_mirror(
|
||||
client: &reqwest::Client,
|
||||
registry: &str,
|
||||
lockfile: Option<PathBuf>,
|
||||
packages: Option<Vec<String>>,
|
||||
all_versions: bool,
|
||||
concurrency: usize,
|
||||
) -> Result<MirrorResult, String> {
|
||||
let targets = if let Some(path) = lockfile {
|
||||
let content = std::fs::read_to_string(&path)
|
||||
.map_err(|e| format!("Cannot read {}: {}", path.display(), e))?;
|
||||
parse_npm_lockfile(&content)?
|
||||
} else if let Some(names) = packages {
|
||||
resolve_npm_packages(client, registry, &names, all_versions).await?
|
||||
} else {
|
||||
return Err("Specify --lockfile or --packages".to_string());
|
||||
};
|
||||
|
||||
if targets.is_empty() {
|
||||
println!("No npm packages to mirror");
|
||||
return Ok(MirrorResult {
|
||||
total: 0,
|
||||
fetched: 0,
|
||||
failed: 0,
|
||||
bytes: 0,
|
||||
});
|
||||
}
|
||||
|
||||
println!(
|
||||
"Mirroring {} npm packages via {}...",
|
||||
targets.len(),
|
||||
registry
|
||||
);
|
||||
mirror_npm_packages(client, registry, &targets, concurrency).await
|
||||
}
|
||||
|
||||
/// Parse package-lock.json (v1, v2, v3)
|
||||
fn parse_npm_lockfile(content: &str) -> Result<Vec<MirrorTarget>, String> {
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_str(content).map_err(|e| format!("Invalid JSON: {}", e))?;
|
||||
|
||||
let version = json
|
||||
.get("lockfileVersion")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(1);
|
||||
|
||||
let mut seen = HashSet::new();
|
||||
let mut targets = Vec::new();
|
||||
|
||||
if version >= 2 {
|
||||
// v2/v3: use "packages" object
|
||||
if let Some(packages) = json.get("packages").and_then(|p| p.as_object()) {
|
||||
for (key, pkg) in packages {
|
||||
if key.is_empty() {
|
||||
continue; // root package
|
||||
}
|
||||
if let Some(name) = extract_package_name(key) {
|
||||
if let Some(ver) = pkg.get("version").and_then(|v| v.as_str()) {
|
||||
let pair = (name.to_string(), ver.to_string());
|
||||
if seen.insert(pair.clone()) {
|
||||
targets.push(MirrorTarget {
|
||||
name: pair.0,
|
||||
version: pair.1,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if version == 1 || targets.is_empty() {
|
||||
// v1 fallback: recursive "dependencies"
|
||||
if let Some(deps) = json.get("dependencies").and_then(|d| d.as_object()) {
|
||||
parse_v1_deps(deps, &mut targets, &mut seen);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(targets)
|
||||
}
|
||||
|
||||
/// Extract package name from lockfile key like "node_modules/@babel/core"
|
||||
fn extract_package_name(key: &str) -> Option<&str> {
|
||||
// Handle nested: "node_modules/foo/node_modules/@scope/bar" → "@scope/bar"
|
||||
let last_nm = key.rfind("node_modules/")?;
|
||||
let after = &key[last_nm + "node_modules/".len()..];
|
||||
let name = after.trim_end_matches('/');
|
||||
if name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(name)
|
||||
}
|
||||
}
|
||||
|
||||
/// Recursively parse v1 lockfile "dependencies"
|
||||
fn parse_v1_deps(
|
||||
deps: &serde_json::Map<String, serde_json::Value>,
|
||||
targets: &mut Vec<MirrorTarget>,
|
||||
seen: &mut HashSet<(String, String)>,
|
||||
) {
|
||||
for (name, pkg) in deps {
|
||||
if let Some(ver) = pkg.get("version").and_then(|v| v.as_str()) {
|
||||
let pair = (name.clone(), ver.to_string());
|
||||
if seen.insert(pair.clone()) {
|
||||
targets.push(MirrorTarget {
|
||||
name: pair.0,
|
||||
version: pair.1,
|
||||
});
|
||||
}
|
||||
}
|
||||
// Recurse into nested dependencies
|
||||
if let Some(nested) = pkg.get("dependencies").and_then(|d| d.as_object()) {
|
||||
parse_v1_deps(nested, targets, seen);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve --packages list by fetching metadata from NORA
|
||||
async fn resolve_npm_packages(
|
||||
client: &reqwest::Client,
|
||||
registry: &str,
|
||||
names: &[String],
|
||||
all_versions: bool,
|
||||
) -> Result<Vec<MirrorTarget>, String> {
|
||||
let base = registry.trim_end_matches('/');
|
||||
let mut targets = Vec::new();
|
||||
|
||||
for name in names {
|
||||
let url = format!("{}/npm/{}", base, name);
|
||||
let resp = client.get(&url).send().await.map_err(|e| e.to_string())?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
eprintln!("Warning: {} not found (HTTP {})", name, resp.status());
|
||||
continue;
|
||||
}
|
||||
|
||||
let json: serde_json::Value = resp.json().await.map_err(|e| e.to_string())?;
|
||||
|
||||
if all_versions {
|
||||
if let Some(versions) = json.get("versions").and_then(|v| v.as_object()) {
|
||||
for ver in versions.keys() {
|
||||
targets.push(MirrorTarget {
|
||||
name: name.clone(),
|
||||
version: ver.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Just latest
|
||||
let latest = json
|
||||
.get("dist-tags")
|
||||
.and_then(|d| d.get("latest"))
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("latest");
|
||||
targets.push(MirrorTarget {
|
||||
name: name.clone(),
|
||||
version: latest.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(targets)
|
||||
}
|
||||
|
||||
/// Fetch packages through NORA (triggers proxy cache)
|
||||
async fn mirror_npm_packages(
|
||||
client: &reqwest::Client,
|
||||
registry: &str,
|
||||
targets: &[MirrorTarget],
|
||||
concurrency: usize,
|
||||
) -> Result<MirrorResult, String> {
|
||||
let base = registry.trim_end_matches('/');
|
||||
let pb = create_progress_bar(targets.len() as u64);
|
||||
let sem = std::sync::Arc::new(Semaphore::new(concurrency));
|
||||
|
||||
// Deduplicate metadata fetches (one per package name)
|
||||
let unique_names: HashSet<&str> = targets.iter().map(|t| t.name.as_str()).collect();
|
||||
pb.set_message("fetching metadata...");
|
||||
for name in &unique_names {
|
||||
let url = format!("{}/npm/{}", base, name);
|
||||
let _ = client.get(&url).send().await; // trigger metadata cache
|
||||
}
|
||||
|
||||
// Fetch tarballs concurrently
|
||||
let fetched = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let failed = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let bytes = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for target in targets {
|
||||
let permit = sem.clone().acquire_owned().await.unwrap();
|
||||
let client = client.clone();
|
||||
let pb = pb.clone();
|
||||
let fetched = fetched.clone();
|
||||
let failed = failed.clone();
|
||||
let bytes = bytes.clone();
|
||||
|
||||
let short_name = target.name.split('/').next_back().unwrap_or(&target.name);
|
||||
let tarball_url = format!(
|
||||
"{}/npm/{}/-/{}-{}.tgz",
|
||||
base, target.name, short_name, target.version
|
||||
);
|
||||
let label = format!("{}@{}", target.name, target.version);
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
match client.get(&tarball_url).send().await {
|
||||
Ok(r) if r.status().is_success() => {
|
||||
if let Ok(body) = r.bytes().await {
|
||||
bytes.fetch_add(body.len() as u64, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
fetched.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
_ => {
|
||||
failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pb.set_message(label);
|
||||
pb.inc(1);
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
let _ = h.await;
|
||||
}
|
||||
|
||||
pb.finish_with_message("done");
|
||||
|
||||
Ok(MirrorResult {
|
||||
total: targets.len(),
|
||||
fetched: fetched.load(std::sync::atomic::Ordering::Relaxed),
|
||||
failed: failed.load(std::sync::atomic::Ordering::Relaxed),
|
||||
bytes: bytes.load(std::sync::atomic::Ordering::Relaxed),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_extract_package_name() {
|
||||
assert_eq!(extract_package_name("node_modules/lodash"), Some("lodash"));
|
||||
assert_eq!(
|
||||
extract_package_name("node_modules/@babel/core"),
|
||||
Some("@babel/core")
|
||||
);
|
||||
assert_eq!(
|
||||
extract_package_name("node_modules/foo/node_modules/bar"),
|
||||
Some("bar")
|
||||
);
|
||||
assert_eq!(
|
||||
extract_package_name("node_modules/foo/node_modules/@types/node"),
|
||||
Some("@types/node")
|
||||
);
|
||||
assert_eq!(extract_package_name(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_lockfile_v3() {
|
||||
let content = r#"{
|
||||
"lockfileVersion": 3,
|
||||
"packages": {
|
||||
"": { "name": "test" },
|
||||
"node_modules/lodash": { "version": "4.17.21" },
|
||||
"node_modules/@babel/core": { "version": "7.26.0" },
|
||||
"node_modules/@babel/core/node_modules/semver": { "version": "6.3.1" }
|
||||
}
|
||||
}"#;
|
||||
let targets = parse_npm_lockfile(content).unwrap();
|
||||
assert_eq!(targets.len(), 3);
|
||||
let names: HashSet<&str> = targets.iter().map(|t| t.name.as_str()).collect();
|
||||
assert!(names.contains("lodash"));
|
||||
assert!(names.contains("@babel/core"));
|
||||
assert!(names.contains("semver"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_lockfile_v1() {
|
||||
let content = r#"{
|
||||
"lockfileVersion": 1,
|
||||
"dependencies": {
|
||||
"express": {
|
||||
"version": "4.18.2",
|
||||
"dependencies": {
|
||||
"accepts": { "version": "1.3.8" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
let targets = parse_npm_lockfile(content).unwrap();
|
||||
assert_eq!(targets.len(), 2);
|
||||
assert_eq!(targets[0].name, "express");
|
||||
assert_eq!(targets[1].name, "accepts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deduplication() {
|
||||
let content = r#"{
|
||||
"lockfileVersion": 3,
|
||||
"packages": {
|
||||
"": {},
|
||||
"node_modules/debug": { "version": "4.3.4" },
|
||||
"node_modules/express/node_modules/debug": { "version": "4.3.4" }
|
||||
}
|
||||
}"#;
|
||||
let targets = parse_npm_lockfile(content).unwrap();
|
||||
assert_eq!(targets.len(), 1); // deduplicated
|
||||
assert_eq!(targets[0].name, "debug");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user