From d7deae9b30091f4b499aedd2fee0ec8611ad943f Mon Sep 17 00:00:00 2001 From: DevITWay | Pavel Volkov Date: Mon, 6 Apr 2026 13:33:13 +0300 Subject: [PATCH] fix: cargo upstream proxy + mirror UX improvements (#107) - Add CargoConfig with upstream proxy to crates.io (NORA_CARGO_PROXY) - Cargo download/metadata: try local storage first, fetch from upstream on miss, cache in background - Detect yarn.lock in npm subcommand and suggest correct command - Add pip transitive dependency warning for air-gapped installs - Improve mirror error messages with HTTP status codes --- nora-registry/src/config.rs | 53 ++++++++++++ nora-registry/src/mirror/mod.rs | 28 +++++- nora-registry/src/registry/cargo_registry.rs | 91 ++++++++++++++++++-- nora-registry/src/test_helpers.rs | 5 ++ 4 files changed, 168 insertions(+), 9 deletions(-) diff --git a/nora-registry/src/config.rs b/nora-registry/src/config.rs index 713e1ad..0c582fa 100644 --- a/nora-registry/src/config.rs +++ b/nora-registry/src/config.rs @@ -28,6 +28,8 @@ pub struct Config { #[serde(default)] pub go: GoConfig, #[serde(default)] + pub cargo: CargoConfig, + #[serde(default)] pub raw: RawConfig, #[serde(default)] pub auth: AuthConfig, @@ -129,6 +131,32 @@ pub struct PypiConfig { pub proxy_timeout: u64, } +/// Cargo registry configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CargoConfig { + /// Upstream Cargo registry (crates.io API) + #[serde(default = "default_cargo_proxy")] + pub proxy: Option, + #[serde(default)] + pub proxy_auth: Option, + #[serde(default = "default_timeout")] + pub proxy_timeout: u64, +} + +fn default_cargo_proxy() -> Option { + Some("https://crates.io".to_string()) +} + +impl Default for CargoConfig { + fn default() -> Self { + Self { + proxy: default_cargo_proxy(), + proxy_auth: None, + proxy_timeout: 30, + } + } +} + /// Go module proxy configuration (GOPROXY protocol) #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GoConfig { @@ -446,6 +474,10 @@ impl Config { if self.pypi.proxy_auth.is_some() && std::env::var("NORA_PYPI_PROXY_AUTH").is_err() { tracing::warn!("PyPI proxy credentials in config.toml are plaintext — consider NORA_PYPI_PROXY_AUTH env var"); } + // Cargo + if self.cargo.proxy_auth.is_some() && std::env::var("NORA_CARGO_PROXY_AUTH").is_err() { + tracing::warn!("Cargo proxy credentials in config.toml are plaintext — consider NORA_CARGO_PROXY_AUTH env var"); + } } /// Validate configuration and return (warnings, errors). @@ -703,6 +735,19 @@ impl Config { } } + // Cargo config + if let Ok(val) = env::var("NORA_CARGO_PROXY") { + self.cargo.proxy = if val.is_empty() { None } else { Some(val) }; + } + if let Ok(val) = env::var("NORA_CARGO_PROXY_TIMEOUT") { + if let Ok(timeout) = val.parse() { + self.cargo.proxy_timeout = timeout; + } + } + if let Ok(val) = env::var("NORA_CARGO_PROXY_AUTH") { + self.cargo.proxy_auth = if val.is_empty() { None } else { Some(val) }; + } + // Raw config if let Ok(val) = env::var("NORA_RAW_ENABLED") { self.raw.enabled = val.to_lowercase() == "true" || val == "1"; @@ -785,6 +830,7 @@ impl Default for Config { npm: NpmConfig::default(), pypi: PypiConfig::default(), go: GoConfig::default(), + cargo: CargoConfig::default(), docker: DockerConfig::default(), raw: RawConfig::default(), auth: AuthConfig::default(), @@ -1371,4 +1417,11 @@ mod tests { assert_eq!(config.go.proxy_auth, Some("user:pass".to_string())); std::env::remove_var("NORA_GO_PROXY_AUTH"); } + + #[test] + fn test_cargo_config_default() { + let c = CargoConfig::default(); + assert_eq!(c.proxy, Some("https://crates.io".to_string())); + assert_eq!(c.proxy_timeout, 30); + } } diff --git a/nora-registry/src/mirror/mod.rs b/nora-registry/src/mirror/mod.rs index f963212..90c137e 100644 --- a/nora-registry/src/mirror/mod.rs +++ b/nora-registry/src/mirror/mod.rs @@ -118,6 +118,15 @@ pub async fn run_mirror( packages, all_versions, } => { + if let Some(ref lf) = lockfile { + let content = std::fs::read_to_string(lf) + .map_err(|e| format!("Cannot read {}: {}", lf.display(), e))?; + if content.contains("# yarn lockfile v1") + || content.starts_with("# THIS IS AN AUTOGENERATED FILE") + { + return Err("This looks like a yarn.lock file. Use `nora mirror yarn --lockfile` instead.".to_string()); + } + } npm::run_npm_mirror( &client, registry, @@ -269,7 +278,19 @@ async fn mirror_lockfile( } fetched += 1; } - _ => failed += 1, + Ok(r) => { + eprintln!( + " WARN: {} {} -> HTTP {}", + target.name, + target.version, + r.status() + ); + failed += 1; + } + Err(e) => { + eprintln!(" WARN: {} {} -> {}", target.name, target.version, e); + failed += 1; + } } pb.set_message(format!("{}@{}", target.name, target.version)); @@ -277,6 +298,11 @@ async fn mirror_lockfile( } pb.finish_with_message("done"); + if format == "pip" && fetched > 0 { + eprintln!( + " NOTE: Only top-level packages were mirrored. For air-gapped installs,\n use `pip freeze > requirements.txt` to include all transitive dependencies." + ); + } Ok(MirrorResult { total: targets.len(), fetched, diff --git a/nora-registry/src/registry/cargo_registry.rs b/nora-registry/src/registry/cargo_registry.rs index 22fae05..0171f61 100644 --- a/nora-registry/src/registry/cargo_registry.rs +++ b/nora-registry/src/registry/cargo_registry.rs @@ -3,6 +3,7 @@ use crate::activity_log::{ActionType, ActivityEntry}; use crate::audit::AuditEntry; +use crate::registry::proxy_fetch; use crate::validation::validate_storage_key; use crate::AppState; use axum::{ @@ -27,13 +28,44 @@ async fn get_metadata( State(state): State>, Path(crate_name): Path, ) -> Response { - // Validate input to prevent path traversal if validate_storage_key(&crate_name).is_err() { return StatusCode::BAD_REQUEST.into_response(); } let key = format!("cargo/{}/metadata.json", crate_name); - match state.storage.get(&key).await { - Ok(data) => (StatusCode::OK, data).into_response(), + + if let Ok(data) = state.storage.get(&key).await { + return (StatusCode::OK, data).into_response(); + } + + // Proxy fetch metadata from upstream + let proxy_url = match &state.config.cargo.proxy { + Some(url) => url.clone(), + None => return StatusCode::NOT_FOUND.into_response(), + }; + + let url = format!( + "{}/api/v1/crates/{}", + proxy_url.trim_end_matches('/'), + crate_name + ); + + match proxy_fetch( + &state.http_client, + &url, + state.config.cargo.proxy_timeout, + state.config.cargo.proxy_auth.as_deref(), + ) + .await + { + Ok(data) => { + let storage = state.storage.clone(); + let key_clone = key.clone(); + let data_clone = data.clone(); + tokio::spawn(async move { + let _ = storage.put(&key_clone, &data_clone).await; + }); + (StatusCode::OK, data).into_response() + } Err(_) => StatusCode::NOT_FOUND.into_response(), } } @@ -42,7 +74,6 @@ async fn download( State(state): State>, Path((crate_name, version)): Path<(String, String)>, ) -> Response { - // Validate inputs to prevent path traversal if validate_storage_key(&crate_name).is_err() || validate_storage_key(&version).is_err() { return StatusCode::BAD_REQUEST.into_response(); } @@ -50,19 +81,63 @@ async fn download( "cargo/{}/{}/{}-{}.crate", crate_name, version, crate_name, version ); - match state.storage.get(&key).await { + + // Try local storage first + if let Ok(data) = state.storage.get(&key).await { + state.metrics.record_download("cargo"); + state.metrics.record_cache_hit(); + state.activity.push(ActivityEntry::new( + ActionType::Pull, + format!("{}@{}", crate_name, version), + "cargo", + "LOCAL", + )); + state + .audit + .log(AuditEntry::new("pull", "api", "", "cargo", "")); + return (StatusCode::OK, data).into_response(); + } + + // Proxy fetch from upstream + let proxy_url = match &state.config.cargo.proxy { + Some(url) => url.clone(), + None => return StatusCode::NOT_FOUND.into_response(), + }; + + let url = format!( + "{}/api/v1/crates/{}/{}/download", + proxy_url.trim_end_matches('/'), + crate_name, + version + ); + + match proxy_fetch( + &state.http_client, + &url, + state.config.cargo.proxy_timeout, + state.config.cargo.proxy_auth.as_deref(), + ) + .await + { Ok(data) => { + // Cache in background + let storage = state.storage.clone(); + let key_clone = key.clone(); + let data_clone = data.clone(); + tokio::spawn(async move { + let _ = storage.put(&key_clone, &data_clone).await; + }); state.metrics.record_download("cargo"); - state.metrics.record_cache_hit(); + state.metrics.record_cache_miss(); state.activity.push(ActivityEntry::new( ActionType::Pull, format!("{}@{}", crate_name, version), "cargo", - "LOCAL", + "PROXY", )); state .audit - .log(AuditEntry::new("pull", "api", "", "cargo", "")); + .log(AuditEntry::new("proxy_fetch", "api", "", "cargo", "")); (StatusCode::OK, data).into_response() } Err(_) => StatusCode::NOT_FOUND.into_response(), diff --git a/nora-registry/src/test_helpers.rs b/nora-registry/src/test_helpers.rs index 5bf70dc..386e158 100644 --- a/nora-registry/src/test_helpers.rs +++ b/nora-registry/src/test_helpers.rs @@ -102,6 +102,11 @@ fn build_context( proxy_timeout_zip: 30, max_zip_size: 10_485_760, }, + cargo: CargoConfig { + proxy: None, + proxy_auth: None, + proxy_timeout: 5, + }, docker: DockerConfig { proxy_timeout: 5, upstreams: vec![],