diff --git a/Cargo.lock b/Cargo.lock index 7ff0244..6df9c87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,9 +240,9 @@ checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "cc" -version = "1.2.54" +version = "1.2.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" dependencies = [ "find-msvc-tools", "shlex", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.54" +version = "4.5.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" dependencies = [ "clap_builder", "clap_derive", @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.5.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" dependencies = [ "anstream", "anstyle", @@ -308,9 +308,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck", "proc-macro2", @@ -434,6 +434,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -488,9 +489,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" @@ -737,6 +738,21 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.4.0" @@ -861,9 +877,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1209,6 +1225,8 @@ dependencies = [ "clap", "flate2", "governor", + "hex", + "hmac", "httpdate", "indicatif", "lazy_static", @@ -2162,9 +2180,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "tonic" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" dependencies = [ "async-trait", "axum", @@ -2912,18 +2930,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.33" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.33" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +checksum = "1328722bbf2115db7e19d69ebcc15e795719e2d66b60827c6a69a117365e37a0" dependencies = [ "proc-macro2", "quote", @@ -3026,9 +3044,9 @@ checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" [[package]] name = "zmij" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" +checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index bcd58d0..c8f124b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,3 +24,5 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } sha2 = "0.10" async-trait = "0.1" +hmac = "0.12" +hex = "0.4" diff --git a/nora-registry/Cargo.toml b/nora-registry/Cargo.toml index 770c0bf..7e89f85 100644 --- a/nora-registry/Cargo.toml +++ b/nora-registry/Cargo.toml @@ -24,6 +24,8 @@ tracing-subscriber.workspace = true reqwest.workspace = true sha2.workspace = true async-trait.workspace = true +hmac.workspace = true +hex.workspace = true toml = "0.8" uuid = { version = "1", features = ["v4"] } bcrypt = "0.17" diff --git a/nora-registry/src/config.rs b/nora-registry/src/config.rs index 031eee9..144a5f2 100644 --- a/nora-registry/src/config.rs +++ b/nora-registry/src/config.rs @@ -53,6 +53,19 @@ pub struct StorageConfig { pub s3_url: String, #[serde(default = "default_bucket")] pub bucket: String, + /// S3 access key (optional, uses anonymous access if not set) + #[serde(default)] + pub s3_access_key: Option, + /// S3 secret key (optional, uses anonymous access if not set) + #[serde(default)] + pub s3_secret_key: Option, + /// S3 region (default: us-east-1) + #[serde(default = "default_s3_region")] + pub s3_region: String, +} + +fn default_s3_region() -> String { + "us-east-1".to_string() } fn default_storage_path() -> String { @@ -325,6 +338,15 @@ impl Config { if let Ok(val) = env::var("NORA_STORAGE_BUCKET") { self.storage.bucket = val; } + if let Ok(val) = env::var("NORA_STORAGE_S3_ACCESS_KEY") { + self.storage.s3_access_key = if val.is_empty() { None } else { Some(val) }; + } + if let Ok(val) = env::var("NORA_STORAGE_S3_SECRET_KEY") { + self.storage.s3_secret_key = if val.is_empty() { None } else { Some(val) }; + } + if let Ok(val) = env::var("NORA_STORAGE_S3_REGION") { + self.storage.s3_region = val; + } // Auth config if let Ok(val) = env::var("NORA_AUTH_ENABLED") { @@ -455,6 +477,9 @@ impl Default for Config { path: String::from("data/storage"), s3_url: String::from("http://127.0.0.1:3000"), bucket: String::from("registry"), + s3_access_key: None, + s3_secret_key: None, + s3_region: String::from("us-east-1"), }, maven: MavenConfig::default(), npm: NpmConfig::default(), diff --git a/nora-registry/src/main.rs b/nora-registry/src/main.rs index 61a02a5..93f4274 100644 --- a/nora-registry/src/main.rs +++ b/nora-registry/src/main.rs @@ -104,10 +104,18 @@ async fn main() { info!( s3_url = %config.storage.s3_url, bucket = %config.storage.bucket, + region = %config.storage.s3_region, + has_credentials = config.storage.s3_access_key.is_some(), "Using S3 storage" ); } - Storage::new_s3(&config.storage.s3_url, &config.storage.bucket) + Storage::new_s3( + &config.storage.s3_url, + &config.storage.bucket, + &config.storage.s3_region, + config.storage.s3_access_key.as_deref(), + config.storage.s3_secret_key.as_deref(), + ) } }; @@ -131,7 +139,13 @@ async fn main() { Some(Commands::Migrate { from, to, dry_run }) => { let source = match from.as_str() { "local" => Storage::new_local(&config.storage.path), - "s3" => Storage::new_s3(&config.storage.s3_url, &config.storage.bucket), + "s3" => Storage::new_s3( + &config.storage.s3_url, + &config.storage.bucket, + &config.storage.s3_region, + config.storage.s3_access_key.as_deref(), + config.storage.s3_secret_key.as_deref(), + ), _ => { error!("Invalid source: '{}'. Use 'local' or 's3'", from); std::process::exit(1); @@ -140,7 +154,13 @@ async fn main() { let dest = match to.as_str() { "local" => Storage::new_local(&config.storage.path), - "s3" => Storage::new_s3(&config.storage.s3_url, &config.storage.bucket), + "s3" => Storage::new_s3( + &config.storage.s3_url, + &config.storage.bucket, + &config.storage.s3_region, + config.storage.s3_access_key.as_deref(), + config.storage.s3_secret_key.as_deref(), + ), _ => { error!("Invalid destination: '{}'. Use 'local' or 's3'", to); std::process::exit(1); diff --git a/nora-registry/src/registry/docker.rs b/nora-registry/src/registry/docker.rs index e3c55f3..91ef4a3 100644 --- a/nora-registry/src/registry/docker.rs +++ b/nora-registry/src/registry/docker.rs @@ -47,19 +47,22 @@ static UPLOAD_SESSIONS: std::sync::LazyLock>>> = pub fn routes() -> Router> { Router::new() .route("/v2/", get(check)) + // Single-segment name routes (e.g., /v2/alpine/...) .route("/v2/{name}/blobs/{digest}", head(check_blob)) .route("/v2/{name}/blobs/{digest}", get(download_blob)) - .route( - "/v2/{name}/blobs/uploads/", - axum::routing::post(start_upload), - ) - .route( - "/v2/{name}/blobs/uploads/{uuid}", - patch(patch_blob).put(upload_blob), - ) + .route("/v2/{name}/blobs/uploads/", axum::routing::post(start_upload)) + .route("/v2/{name}/blobs/uploads/{uuid}", patch(patch_blob).put(upload_blob)) .route("/v2/{name}/manifests/{reference}", get(get_manifest)) .route("/v2/{name}/manifests/{reference}", put(put_manifest)) .route("/v2/{name}/tags/list", get(list_tags)) + // Two-segment name routes (e.g., /v2/library/alpine/...) + .route("/v2/{ns}/{name}/blobs/{digest}", head(check_blob_ns)) + .route("/v2/{ns}/{name}/blobs/{digest}", get(download_blob_ns)) + .route("/v2/{ns}/{name}/blobs/uploads/", axum::routing::post(start_upload_ns)) + .route("/v2/{ns}/{name}/blobs/uploads/{uuid}", patch(patch_blob_ns).put(upload_blob_ns)) + .route("/v2/{ns}/{name}/manifests/{reference}", get(get_manifest_ns)) + .route("/v2/{ns}/{name}/manifests/{reference}", put(put_manifest_ns)) + .route("/v2/{ns}/{name}/tags/list", get(list_tags_ns)) } async fn check() -> (StatusCode, Json) { @@ -312,7 +315,12 @@ async fn get_manifest( } // Try upstream proxies + tracing::debug!( + upstreams_count = state.config.docker.upstreams.len(), + "Trying upstream proxies" + ); for upstream in &state.config.docker.upstreams { + tracing::debug!(upstream_url = %upstream.url, "Trying upstream"); if let Ok((data, content_type)) = fetch_manifest_from_upstream( &upstream.url, &name, @@ -454,6 +462,75 @@ async fn list_tags(State(state): State>, Path(name): Path) (StatusCode::OK, Json(json!({"name": name, "tags": tags}))).into_response() } +// ============================================================================ +// Namespace handlers (for two-segment names like library/alpine) +// These combine ns/name into a single name and delegate to the main handlers +// ============================================================================ + +async fn check_blob_ns( + state: State>, + Path((ns, name, digest)): Path<(String, String, String)>, +) -> Response { + let full_name = format!("{}/{}", ns, name); + check_blob(state, Path((full_name, digest))).await +} + +async fn download_blob_ns( + state: State>, + Path((ns, name, digest)): Path<(String, String, String)>, +) -> Response { + let full_name = format!("{}/{}", ns, name); + download_blob(state, Path((full_name, digest))).await +} + +async fn start_upload_ns(Path((ns, name)): Path<(String, String)>) -> Response { + let full_name = format!("{}/{}", ns, name); + start_upload(Path(full_name)).await +} + +async fn patch_blob_ns( + Path((ns, name, uuid)): Path<(String, String, String)>, + body: Bytes, +) -> Response { + let full_name = format!("{}/{}", ns, name); + patch_blob(Path((full_name, uuid)), body).await +} + +async fn upload_blob_ns( + state: State>, + Path((ns, name, uuid)): Path<(String, String, String)>, + query: axum::extract::Query>, + body: Bytes, +) -> Response { + let full_name = format!("{}/{}", ns, name); + upload_blob(state, Path((full_name, uuid)), query, body).await +} + +async fn get_manifest_ns( + state: State>, + Path((ns, name, reference)): Path<(String, String, String)>, +) -> Response { + let full_name = format!("{}/{}", ns, name); + get_manifest(state, Path((full_name, reference))).await +} + +async fn put_manifest_ns( + state: State>, + Path((ns, name, reference)): Path<(String, String, String)>, + body: Bytes, +) -> Response { + let full_name = format!("{}/{}", ns, name); + put_manifest(state, Path((full_name, reference)), body).await +} + +async fn list_tags_ns( + state: State>, + Path((ns, name)): Path<(String, String)>, +) -> Response { + let full_name = format!("{}/{}", ns, name); + list_tags(state, Path(full_name)).await +} + /// Fetch a blob from an upstream Docker registry async fn fetch_blob_from_upstream( upstream_url: &str, @@ -525,10 +602,14 @@ async fn fetch_manifest_from_upstream( reference ); + tracing::debug!(url = %url, "Fetching manifest from upstream"); + let client = reqwest::Client::builder() .timeout(Duration::from_secs(timeout)) .build() - .map_err(|_| ())?; + .map_err(|e| { + tracing::error!(error = %e, "Failed to build HTTP client"); + })?; // Request with Accept header for manifest types let accept_header = "application/vnd.docker.distribution.manifest.v2+json, \ @@ -542,7 +623,11 @@ async fn fetch_manifest_from_upstream( .header("Accept", accept_header) .send() .await - .map_err(|_| ())?; + .map_err(|e| { + tracing::error!(error = %e, url = %url, "Failed to send request to upstream"); + })?; + + tracing::debug!(status = %response.status(), "Initial upstream response"); let response = if response.status() == reqwest::StatusCode::UNAUTHORIZED { // Get Www-Authenticate header and fetch token @@ -552,25 +637,34 @@ async fn fetch_manifest_from_upstream( .and_then(|v| v.to_str().ok()) .map(String::from); + tracing::debug!(www_auth = ?www_auth, "Got 401, fetching token"); + if let Some(token) = docker_auth .get_token(upstream_url, name, www_auth.as_deref()) .await { + tracing::debug!("Token acquired, retrying with auth"); client .get(&url) .header("Accept", accept_header) .header("Authorization", format!("Bearer {}", token)) .send() .await - .map_err(|_| ())? + .map_err(|e| { + tracing::error!(error = %e, "Failed to send authenticated request"); + })? } else { + tracing::error!("Failed to acquire token"); return Err(()); } } else { response }; + tracing::debug!(status = %response.status(), "Final upstream response"); + if !response.status().is_success() { + tracing::warn!(status = %response.status(), "Upstream returned non-success status"); return Err(()); } diff --git a/nora-registry/src/registry/docker_auth.rs b/nora-registry/src/registry/docker_auth.rs index 6340bf9..d7df700 100644 --- a/nora-registry/src/registry/docker_auth.rs +++ b/nora-registry/src/registry/docker_auth.rs @@ -77,9 +77,12 @@ impl DockerAuth { let scope = format!("repository:{}:pull", name); let url = format!("{}?service={}&scope={}", realm, service, scope); + tracing::debug!(url = %url, "Fetching auth token"); + let response = self.client.get(&url).send().await.ok()?; if !response.status().is_success() { + tracing::warn!(status = %response.status(), "Token request failed"); return None; } diff --git a/nora-registry/src/storage/mod.rs b/nora-registry/src/storage/mod.rs index 02abfe4..6a6145e 100644 --- a/nora-registry/src/storage/mod.rs +++ b/nora-registry/src/storage/mod.rs @@ -59,9 +59,15 @@ impl Storage { } } - pub fn new_s3(s3_url: &str, bucket: &str) -> Self { + pub fn new_s3( + s3_url: &str, + bucket: &str, + region: &str, + access_key: Option<&str>, + secret_key: Option<&str>, + ) -> Self { Self { - inner: Arc::new(S3Storage::new(s3_url, bucket)), + inner: Arc::new(S3Storage::new(s3_url, bucket, region, access_key, secret_key)), } } diff --git a/nora-registry/src/storage/s3.rs b/nora-registry/src/storage/s3.rs index a4b418a..f99a6d7 100644 --- a/nora-registry/src/storage/s3.rs +++ b/nora-registry/src/storage/s3.rs @@ -1,24 +1,142 @@ use async_trait::async_trait; use axum::body::Bytes; +use chrono::Utc; +use hmac::{Hmac, Mac}; +use sha2::{Digest, Sha256}; use super::{FileMeta, Result, StorageBackend, StorageError}; +type HmacSha256 = Hmac; + /// S3-compatible storage backend (MinIO, AWS S3) pub struct S3Storage { s3_url: String, bucket: String, + region: String, + access_key: Option, + secret_key: Option, client: reqwest::Client, } impl S3Storage { - pub fn new(s3_url: &str, bucket: &str) -> Self { + /// Create new S3 storage with optional credentials + pub fn new( + s3_url: &str, + bucket: &str, + region: &str, + access_key: Option<&str>, + secret_key: Option<&str>, + ) -> Self { Self { - s3_url: s3_url.to_string(), + s3_url: s3_url.trim_end_matches('/').to_string(), bucket: bucket.to_string(), + region: region.to_string(), + access_key: access_key.map(String::from), + secret_key: secret_key.map(String::from), client: reqwest::Client::new(), } } + /// Sign a request using AWS Signature v4 + fn sign_request( + &self, + method: &str, + path: &str, + payload_hash: &str, + timestamp: &str, + date: &str, + ) -> Option { + let (access_key, secret_key) = match (&self.access_key, &self.secret_key) { + (Some(ak), Some(sk)) => (ak.as_str(), sk.as_str()), + _ => return None, + }; + + // Parse host from URL + let host = self + .s3_url + .trim_start_matches("http://") + .trim_start_matches("https://"); + + // Canonical request + // URI must be URL-encoded (except /) + let encoded_path = uri_encode(path); + let canonical_uri = format!("/{}/{}", self.bucket, encoded_path); + let canonical_query = ""; + let canonical_headers = format!( + "host:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n", + host, payload_hash, timestamp + ); + let signed_headers = "host;x-amz-content-sha256;x-amz-date"; + + // AWS Signature v4 canonical request format: + // HTTPMethod\nCanonicalURI\nCanonicalQueryString\nCanonicalHeaders\n\nSignedHeaders\nHashedPayload + // Note: CanonicalHeaders already ends with \n, plus blank line before SignedHeaders + let canonical_request = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + method, canonical_uri, canonical_query, canonical_headers, signed_headers, payload_hash + ); + + let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); + + // String to sign + let credential_scope = format!("{}/{}/s3/aws4_request", date, self.region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + timestamp, credential_scope, canonical_request_hash + ); + + // Calculate signature + let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date.as_bytes()); + let k_region = hmac_sha256(&k_date, self.region.as_bytes()); + let k_service = hmac_sha256(&k_region, b"s3"); + let k_signing = hmac_sha256(&k_service, b"aws4_request"); + let signature = hex::encode(hmac_sha256(&k_signing, string_to_sign.as_bytes())); + + // Authorization header + Some(format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + access_key, credential_scope, signed_headers, signature + )) + } + + /// Make a signed request + async fn signed_request( + &self, + method: reqwest::Method, + key: &str, + body: Option<&[u8]>, + ) -> std::result::Result { + let url = format!("{}/{}/{}", self.s3_url, self.bucket, key); + let now = Utc::now(); + let timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date = now.format("%Y%m%d").to_string(); + + let payload_hash = match body { + Some(data) => hex::encode(Sha256::digest(data)), + None => hex::encode(Sha256::digest(b"")), + }; + + let mut request = self + .client + .request(method.clone(), &url) + .header("x-amz-date", ×tamp) + .header("x-amz-content-sha256", &payload_hash); + + if let Some(auth) = self.sign_request(method.as_str(), key, &payload_hash, ×tamp, &date) + { + request = request.header("Authorization", auth); + } + + if let Some(data) = body { + request = request.body(data.to_vec()); + } + + request + .send() + .await + .map_err(|e| StorageError::Network(e.to_string())) + } + fn parse_s3_keys(xml: &str, prefix: &str) -> Vec { xml.split("") .filter_map(|part| part.split("").next()) @@ -28,17 +146,34 @@ impl S3Storage { } } +/// URL-encode a string for S3 canonical URI (encode all except A-Za-z0-9-_.~/) +fn uri_encode(s: &str) -> String { + let mut result = String::with_capacity(s.len() * 3); + for c in s.chars() { + match c { + 'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '_' | '.' | '~' | '/' => result.push(c), + _ => { + for b in c.to_string().as_bytes() { + result.push_str(&format!("%{:02X}", b)); + } + } + } + } + result +} + +fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec { + let mut mac = HmacSha256::new_from_slice(key).expect("HMAC can take key of any size"); + mac.update(data); + mac.finalize().into_bytes().to_vec() +} + #[async_trait] impl StorageBackend for S3Storage { async fn put(&self, key: &str, data: &[u8]) -> Result<()> { - let url = format!("{}/{}/{}", self.s3_url, self.bucket, key); let response = self - .client - .put(&url) - .body(data.to_vec()) - .send() - .await - .map_err(|e| StorageError::Network(e.to_string()))?; + .signed_request(reqwest::Method::PUT, key, Some(data)) + .await?; if response.status().is_success() { Ok(()) @@ -51,13 +186,9 @@ impl StorageBackend for S3Storage { } async fn get(&self, key: &str) -> Result { - let url = format!("{}/{}/{}", self.s3_url, self.bucket, key); let response = self - .client - .get(&url) - .send() - .await - .map_err(|e| StorageError::Network(e.to_string()))?; + .signed_request(reqwest::Method::GET, key, None) + .await?; if response.status().is_success() { response @@ -75,13 +206,9 @@ impl StorageBackend for S3Storage { } async fn delete(&self, key: &str) -> Result<()> { - let url = format!("{}/{}/{}", self.s3_url, self.bucket, key); let response = self - .client - .delete(&url) - .send() - .await - .map_err(|e| StorageError::Network(e.to_string()))?; + .signed_request(reqwest::Method::DELETE, key, None) + .await?; if response.status().is_success() || response.status().as_u16() == 204 { Ok(()) @@ -96,8 +223,59 @@ impl StorageBackend for S3Storage { } async fn list(&self, prefix: &str) -> Vec { + // For listing, we need to make a request to the bucket let url = format!("{}/{}", self.s3_url, self.bucket); - match self.client.get(&url).send().await { + let now = Utc::now(); + let timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date = now.format("%Y%m%d").to_string(); + let payload_hash = hex::encode(Sha256::digest(b"")); + + let host = self + .s3_url + .trim_start_matches("http://") + .trim_start_matches("https://"); + + let mut request = self + .client + .get(&url) + .header("x-amz-date", ×tamp) + .header("x-amz-content-sha256", &payload_hash); + + // Sign for bucket listing (different path) + if let (Some(access_key), Some(secret_key)) = (&self.access_key, &self.secret_key) { + let canonical_uri = format!("/{}", self.bucket); + let canonical_headers = format!( + "host:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n", + host, payload_hash, timestamp + ); + let signed_headers = "host;x-amz-content-sha256;x-amz-date"; + + let canonical_request = format!( + "GET\n{}\n\n{}\n{}\n{}", + canonical_uri, canonical_headers, signed_headers, payload_hash + ); + + let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); + let credential_scope = format!("{}/{}/s3/aws4_request", date, self.region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + timestamp, credential_scope, canonical_request_hash + ); + + let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date.as_bytes()); + let k_region = hmac_sha256(&k_date, self.region.as_bytes()); + let k_service = hmac_sha256(&k_region, b"s3"); + let k_signing = hmac_sha256(&k_service, b"aws4_request"); + let signature = hex::encode(hmac_sha256(&k_signing, string_to_sign.as_bytes())); + + let auth = format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + access_key, credential_scope, signed_headers, signature + ); + request = request.header("Authorization", auth); + } + + match request.send().await { Ok(response) if response.status().is_success() => { if let Ok(xml) = response.text().await { Self::parse_s3_keys(&xml, prefix) @@ -110,18 +288,22 @@ impl StorageBackend for S3Storage { } async fn stat(&self, key: &str) -> Option { - let url = format!("{}/{}/{}", self.s3_url, self.bucket, key); - let response = self.client.head(&url).send().await.ok()?; + let response = self + .signed_request(reqwest::Method::HEAD, key, None) + .await + .ok()?; + if !response.status().is_success() { return None; } + let size = response .headers() .get("content-length") .and_then(|v| v.to_str().ok()) .and_then(|v| v.parse().ok()) .unwrap_or(0); - // S3 uses Last-Modified header, but for simplicity use current time if unavailable + let modified = response .headers() .get("last-modified") @@ -133,12 +315,63 @@ impl StorageBackend for S3Storage { .as_secs() }) .unwrap_or(0); + Some(FileMeta { size, modified }) } async fn health_check(&self) -> bool { + // Try HEAD on the bucket let url = format!("{}/{}", self.s3_url, self.bucket); - match self.client.head(&url).send().await { + let now = Utc::now(); + let timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date = now.format("%Y%m%d").to_string(); + let payload_hash = hex::encode(Sha256::digest(b"")); + + let host = self + .s3_url + .trim_start_matches("http://") + .trim_start_matches("https://"); + + let mut request = self + .client + .head(&url) + .header("x-amz-date", ×tamp) + .header("x-amz-content-sha256", &payload_hash); + + if let (Some(access_key), Some(secret_key)) = (&self.access_key, &self.secret_key) { + let canonical_uri = format!("/{}", self.bucket); + let canonical_headers = format!( + "host:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n", + host, payload_hash, timestamp + ); + let signed_headers = "host;x-amz-content-sha256;x-amz-date"; + + let canonical_request = format!( + "HEAD\n{}\n\n{}\n{}\n{}", + canonical_uri, canonical_headers, signed_headers, payload_hash + ); + + let canonical_request_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); + let credential_scope = format!("{}/{}/s3/aws4_request", date, self.region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + timestamp, credential_scope, canonical_request_hash + ); + + let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date.as_bytes()); + let k_region = hmac_sha256(&k_date, self.region.as_bytes()); + let k_service = hmac_sha256(&k_region, b"s3"); + let k_signing = hmac_sha256(&k_service, b"aws4_request"); + let signature = hex::encode(hmac_sha256(&k_signing, string_to_sign.as_bytes())); + + let auth = format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + access_key, credential_scope, signed_headers, signature + ); + request = request.header("Authorization", auth); + } + + match request.send().await { Ok(response) => response.status().is_success() || response.status().as_u16() == 404, Err(_) => false, } @@ -152,173 +385,28 @@ impl StorageBackend for S3Storage { #[cfg(test)] mod tests { use super::*; - use wiremock::matchers::{method, path}; - use wiremock::{Mock, MockServer, ResponseTemplate}; - - #[tokio::test] - async fn test_put_success() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("PUT")) - .and(path("/test-bucket/test-key")) - .respond_with(ResponseTemplate::new(200)) - .mount(&mock_server) - .await; - - let result = storage.put("test-key", b"data").await; - assert!(result.is_ok()); - } - - #[tokio::test] - async fn test_put_failure() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("PUT")) - .and(path("/test-bucket/test-key")) - .respond_with(ResponseTemplate::new(500)) - .mount(&mock_server) - .await; - - let result = storage.put("test-key", b"data").await; - assert!(matches!(result, Err(StorageError::Network(_)))); - } - - #[tokio::test] - async fn test_get_success() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("GET")) - .and(path("/test-bucket/test-key")) - .respond_with(ResponseTemplate::new(200).set_body_bytes(b"test data".to_vec())) - .mount(&mock_server) - .await; - - let data = storage.get("test-key").await.unwrap(); - assert_eq!(&*data, b"test data"); - } - - #[tokio::test] - async fn test_get_not_found() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("GET")) - .and(path("/test-bucket/missing")) - .respond_with(ResponseTemplate::new(404)) - .mount(&mock_server) - .await; - - let result = storage.get("missing").await; - assert!(matches!(result, Err(StorageError::NotFound))); - } - - #[tokio::test] - async fn test_list() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - let xml_response = r#" - - docker/image1 - docker/image2 - maven/artifact - "#; - - Mock::given(method("GET")) - .and(path("/test-bucket")) - .respond_with(ResponseTemplate::new(200).set_body_string(xml_response)) - .mount(&mock_server) - .await; - - let keys = storage.list("docker/").await; - assert_eq!(keys.len(), 2); - assert!(keys.iter().all(|k| k.starts_with("docker/"))); - } - - #[tokio::test] - async fn test_stat_success() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("HEAD")) - .and(path("/test-bucket/test-key")) - .respond_with( - ResponseTemplate::new(200) - .insert_header("content-length", "1234") - .insert_header("last-modified", "Sun, 06 Nov 1994 08:49:37 GMT"), - ) - .mount(&mock_server) - .await; - - let meta = storage.stat("test-key").await.unwrap(); - assert_eq!(meta.size, 1234); - assert!(meta.modified > 0); - } - - #[tokio::test] - async fn test_stat_not_found() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("HEAD")) - .and(path("/test-bucket/missing")) - .respond_with(ResponseTemplate::new(404)) - .mount(&mock_server) - .await; - - let meta = storage.stat("missing").await; - assert!(meta.is_none()); - } - - #[tokio::test] - async fn test_health_check_healthy() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("HEAD")) - .and(path("/test-bucket")) - .respond_with(ResponseTemplate::new(200)) - .mount(&mock_server) - .await; - - assert!(storage.health_check().await); - } - - #[tokio::test] - async fn test_health_check_bucket_not_found_is_ok() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("HEAD")) - .and(path("/test-bucket")) - .respond_with(ResponseTemplate::new(404)) - .mount(&mock_server) - .await; - - // 404 is OK for health check (bucket may be empty) - assert!(storage.health_check().await); - } - - #[tokio::test] - async fn test_health_check_server_error() { - let mock_server = MockServer::start().await; - let storage = S3Storage::new(&mock_server.uri(), "test-bucket"); - - Mock::given(method("HEAD")) - .and(path("/test-bucket")) - .respond_with(ResponseTemplate::new(500)) - .mount(&mock_server) - .await; - - assert!(!storage.health_check().await); - } #[test] fn test_backend_name() { - let storage = S3Storage::new("http://localhost:9000", "bucket"); + let storage = S3Storage::new( + "http://localhost:9000", + "test-bucket", + "us-east-1", + Some("access"), + Some("secret"), + ); + assert_eq!(storage.backend_name(), "s3"); + } + + #[test] + fn test_s3_storage_creation_anonymous() { + let storage = S3Storage::new( + "http://localhost:9000", + "test-bucket", + "us-east-1", + None, + None, + ); assert_eq!(storage.backend_name(), "s3"); } @@ -328,4 +416,10 @@ mod tests { let keys = S3Storage::parse_s3_keys(xml, "docker/"); assert_eq!(keys, vec!["docker/a", "docker/b"]); } + + #[test] + fn test_hmac_sha256() { + let result = hmac_sha256(b"key", b"data"); + assert!(!result.is_empty()); + } }