taiars/nora
1
0
Fork 0
mirror of https://github.com/getnora-io/nora.git synced 2026-04-12 06:50:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: cargo upstream proxy + mirror UX improvements (#107)

Browse Source 
- Add CargoConfig with upstream proxy to crates.io (NORA_CARGO_PROXY)
- Cargo download/metadata: try local storage first, fetch from upstream on miss, cache in background
- Detect yarn.lock in npm subcommand and suggest correct command
- Add pip transitive dependency warning for air-gapped installs
- Improve mirror error messages with HTTP status codes
This commit is contained in:
DevITWay | Pavel Volkov
2026-04-06 13:33:13 +03:00
committed by GitHub
parent 38828ec31e
commit d7deae9b30
4 changed files with 168 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
nora-registry/src/config.rs
View File

@@ -28,6 +28,8 @@ pub struct Config {
#[serde(default)]
pub go: GoConfig,
#[serde(default)]
pub cargo: CargoConfig,
#[serde(default)]
pub raw: RawConfig,
#[serde(default)]
pub auth: AuthConfig,
@@ -129,6 +131,32 @@ pub struct PypiConfig {
pub proxy_timeout: u64,
}
/// Cargo registry configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CargoConfig {
/// Upstream Cargo registry (crates.io API)
#[serde(default = "default_cargo_proxy")]
pub proxy: Option<String>,
#[serde(default)]
pub proxy_auth: Option<String>,
#[serde(default = "default_timeout")]
pub proxy_timeout: u64,
}
fn default_cargo_proxy() -> Option<String> {
Some("https://crates.io".to_string())
}
impl Default for CargoConfig {
fn default() -> Self {
Self {
proxy: default_cargo_proxy(),
proxy_auth: None,
proxy_timeout: 30,
}
}
}
/// Go module proxy configuration (GOPROXY protocol)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GoConfig {
@@ -446,6 +474,10 @@ impl Config {
if self.pypi.proxy_auth.is_some() && std::env::var("NORA_PYPI_PROXY_AUTH").is_err() {
tracing::warn!("PyPI proxy credentials in config.toml are plaintext — consider NORA_PYPI_PROXY_AUTH env var");
}
// Cargo
if self.cargo.proxy_auth.is_some() && std::env::var("NORA_CARGO_PROXY_AUTH").is_err() {
tracing::warn!("Cargo proxy credentials in config.toml are plaintext — consider NORA_CARGO_PROXY_AUTH env var");
}
}
/// Validate configuration and return (warnings, errors).
@@ -703,6 +735,19 @@ impl Config {
}
}
// Cargo config
if let Ok(val) = env::var("NORA_CARGO_PROXY") {
self.cargo.proxy = if val.is_empty() { None } else { Some(val) };
}
if let Ok(val) = env::var("NORA_CARGO_PROXY_TIMEOUT") {
if let Ok(timeout) = val.parse() {
self.cargo.proxy_timeout = timeout;
}
}
if let Ok(val) = env::var("NORA_CARGO_PROXY_AUTH") {
self.cargo.proxy_auth = if val.is_empty() { None } else { Some(val) };
}
// Raw config
if let Ok(val) = env::var("NORA_RAW_ENABLED") {
self.raw.enabled = val.to_lowercase() == "true" || val == "1";
@@ -785,6 +830,7 @@ impl Default for Config {
npm: NpmConfig::default(),
pypi: PypiConfig::default(),
go: GoConfig::default(),
cargo: CargoConfig::default(),
docker: DockerConfig::default(),
raw: RawConfig::default(),
auth: AuthConfig::default(),
@@ -1371,4 +1417,11 @@ mod tests {
assert_eq!(config.go.proxy_auth, Some("user:pass".to_string()));
std::env::remove_var("NORA_GO_PROXY_AUTH");
}
#[test]
fn test_cargo_config_default() {
let c = CargoConfig::default();
assert_eq!(c.proxy, Some("https://crates.io".to_string()));
assert_eq!(c.proxy_timeout, 30);
}
}

28
nora-registry/src/mirror/mod.rs
View File

@@ -118,6 +118,15 @@ pub async fn run_mirror(
packages,
all_versions,
} => {
if let Some(ref lf) = lockfile {
let content = std::fs::read_to_string(lf)
.map_err(|e| format!("Cannot read {}: {}", lf.display(), e))?;
if content.contains("# yarn lockfile v1")
|| content.starts_with("# THIS IS AN AUTOGENERATED FILE")
{
return Err("This looks like a yarn.lock file. Use `nora mirror yarn --lockfile` instead.".to_string());
}
}
npm::run_npm_mirror(
&client,
registry,
@@ -269,7 +278,19 @@ async fn mirror_lockfile(
}
fetched += 1;
}
_ => failed += 1,
Ok(r) => {
eprintln!(
" WARN: {} {} -> HTTP {}",
target.name,
target.version,
r.status()
);
failed += 1;
}
Err(e) => {
eprintln!(" WARN: {} {} -> {}", target.name, target.version, e);
failed += 1;
}
}
pb.set_message(format!("{}@{}", target.name, target.version));
@@ -277,6 +298,11 @@ async fn mirror_lockfile(
}
pb.finish_with_message("done");
if format == "pip" && fetched > 0 {
eprintln!(
" NOTE: Only top-level packages were mirrored. For air-gapped installs,\n use `pip freeze > requirements.txt` to include all transitive dependencies."
);
}
Ok(MirrorResult {
total: targets.len(),
fetched,

91
nora-registry/src/registry/cargo_registry.rs
View File

@@ -3,6 +3,7 @@
use crate::activity_log::{ActionType, ActivityEntry};
use crate::audit::AuditEntry;
use crate::registry::proxy_fetch;
use crate::validation::validate_storage_key;
use crate::AppState;
use axum::{
@@ -27,13 +28,44 @@ async fn get_metadata(
State(state): State<Arc<AppState>>,
Path(crate_name): Path<String>,
) -> Response {
// Validate input to prevent path traversal
if validate_storage_key(&crate_name).is_err() {
return StatusCode::BAD_REQUEST.into_response();
}
let key = format!("cargo/{}/metadata.json", crate_name);
match state.storage.get(&key).await {
Ok(data) => (StatusCode::OK, data).into_response(),
if let Ok(data) = state.storage.get(&key).await {
return (StatusCode::OK, data).into_response();
}
// Proxy fetch metadata from upstream
let proxy_url = match &state.config.cargo.proxy {
Some(url) => url.clone(),
None => return StatusCode::NOT_FOUND.into_response(),
};
let url = format!(
"{}/api/v1/crates/{}",
proxy_url.trim_end_matches('/'),
crate_name
);
match proxy_fetch(
&state.http_client,
&url,
state.config.cargo.proxy_timeout,
state.config.cargo.proxy_auth.as_deref(),
)
.await
{
Ok(data) => {
let storage = state.storage.clone();
let key_clone = key.clone();
let data_clone = data.clone();
tokio::spawn(async move {
let _ = storage.put(&key_clone, &data_clone).await;
});
(StatusCode::OK, data).into_response()
}
Err(_) => StatusCode::NOT_FOUND.into_response(),
}
}
@@ -42,7 +74,6 @@ async fn download(
State(state): State<Arc<AppState>>,
Path((crate_name, version)): Path<(String, String)>,
) -> Response {
// Validate inputs to prevent path traversal
if validate_storage_key(&crate_name).is_err() || validate_storage_key(&version).is_err() {
return StatusCode::BAD_REQUEST.into_response();
}
@@ -50,19 +81,63 @@ async fn download(
"cargo/{}/{}/{}-{}.crate",
crate_name, version, crate_name, version
);
match state.storage.get(&key).await {
// Try local storage first
if let Ok(data) = state.storage.get(&key).await {
state.metrics.record_download("cargo");
state.metrics.record_cache_hit();
state.activity.push(ActivityEntry::new(
ActionType::Pull,
format!("{}@{}", crate_name, version),
"cargo",
"LOCAL",
));
state
.audit
.log(AuditEntry::new("pull", "api", "", "cargo", ""));
return (StatusCode::OK, data).into_response();
}
// Proxy fetch from upstream
let proxy_url = match &state.config.cargo.proxy {
Some(url) => url.clone(),
None => return StatusCode::NOT_FOUND.into_response(),
};
let url = format!(
"{}/api/v1/crates/{}/{}/download",
proxy_url.trim_end_matches('/'),
crate_name,
version
);
match proxy_fetch(
&state.http_client,
&url,
state.config.cargo.proxy_timeout,
state.config.cargo.proxy_auth.as_deref(),
)
.await
{
Ok(data) => {
// Cache in background
let storage = state.storage.clone();
let key_clone = key.clone();
let data_clone = data.clone();
tokio::spawn(async move {
let _ = storage.put(&key_clone, &data_clone).await;
});
state.metrics.record_download("cargo");
state.metrics.record_cache_hit();
state.metrics.record_cache_miss();
state.activity.push(ActivityEntry::new(
ActionType::Pull,
format!("{}@{}", crate_name, version),
"cargo",
"LOCAL",
"PROXY",
));
state
.audit
.log(AuditEntry::new("pull", "api", "", "cargo", ""));
.log(AuditEntry::new("proxy_fetch", "api", "", "cargo", ""));
(StatusCode::OK, data).into_response()
}
Err(_) => StatusCode::NOT_FOUND.into_response(),

5
nora-registry/src/test_helpers.rs
View File

@@ -102,6 +102,11 @@ fn build_context(
proxy_timeout_zip: 30,
max_zip_size: 10_485_760,
},
cargo: CargoConfig {
proxy: None,
proxy_auth: None,
proxy_timeout: 5,
},
docker: DockerConfig {
proxy_timeout: 5,
upstreams: vec![],