mirror of
https://github.com/getnora-io/nora.git
synced 2026-04-13 13:10:31 +00:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fc1288820d | |||
| a17a75161b | |||
| 0b3ef3ab96 | |||
| 99e290d30c | |||
| f74b781d1f | |||
| 05c765627f | |||
| 1813546bee | |||
| 196c313f20 | |||
| aece2d739d | |||
| b7e11da2da | |||
| dd3813edff | |||
| adade10c67 | |||
| 6ad710ff32 | |||
| 037204a3eb | |||
| 1e01d4df56 | |||
| ab5ed3f488 | |||
| 8336166e0e | |||
| 42e71b9195 | |||
| ffac4f0286 | |||
| 078ef94153 |
60
.github/workflows/ci.yml
vendored
60
.github/workflows/ci.yml
vendored
@@ -27,3 +27,63 @@ jobs:
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --package nora-registry
|
||||
|
||||
security:
|
||||
name: Security
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write # for uploading SARIF to GitHub Security tab
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # full history required for gitleaks
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
|
||||
# ── Secrets ────────────────────────────────────────────────────────────
|
||||
- name: Gitleaks — scan for hardcoded secrets
|
||||
run: |
|
||||
curl -sL https://github.com/gitleaks/gitleaks/releases/download/v8.21.2/gitleaks_8.21.2_linux_x64.tar.gz \
|
||||
| tar xz -C /usr/local/bin gitleaks
|
||||
gitleaks detect --source . --exit-code 1 --report-format sarif --report-path gitleaks.sarif || true
|
||||
continue-on-error: true # findings are reported, do not block the pipeline
|
||||
|
||||
# ── CVE in Rust dependencies ────────────────────────────────────────────
|
||||
- name: Install cargo-audit
|
||||
run: cargo install cargo-audit --locked
|
||||
|
||||
- name: cargo audit — RustSec advisory database
|
||||
run: cargo audit
|
||||
continue-on-error: true # warn only; known CVEs should not block CI until triaged
|
||||
|
||||
# ── Licenses, banned crates, supply chain policy ────────────────────────
|
||||
- name: cargo deny — licenses and banned crates
|
||||
uses: EmbarkStudios/cargo-deny-action@v2
|
||||
with:
|
||||
command: check
|
||||
arguments: --all-features
|
||||
|
||||
# ── CVE scan of source tree and Cargo.lock ──────────────────────────────
|
||||
- name: Trivy — filesystem scan (Cargo.lock + source)
|
||||
if: always()
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
scan-type: fs
|
||||
scan-ref: .
|
||||
format: sarif
|
||||
output: trivy-fs.sarif
|
||||
severity: HIGH,CRITICAL
|
||||
exit-code: 0 # warn only; change to 1 to block the pipeline
|
||||
|
||||
- name: Upload Trivy fs results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-fs.sarif
|
||||
category: trivy-fs
|
||||
|
||||
144
.github/workflows/release.yml
vendored
144
.github/workflows/release.yml
vendored
@@ -19,8 +19,16 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Set up Rust
|
||||
run: |
|
||||
echo "/home/github-runner/.cargo/bin" >> $GITHUB_PATH
|
||||
echo "RUSTUP_HOME=/home/github-runner/.rustup" >> $GITHUB_ENV
|
||||
echo "CARGO_HOME=/home/github-runner/.cargo" >> $GITHUB_ENV
|
||||
|
||||
- name: Build release binary (musl static)
|
||||
run: |
|
||||
cargo build --release --target x86_64-unknown-linux-musl --package nora-registry
|
||||
cp target/x86_64-unknown-linux-musl/release/nora ./nora
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
@@ -32,49 +40,163 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
# ── Alpine (standard) ────────────────────────────────────────────────────
|
||||
- name: Extract metadata (alpine)
|
||||
id: meta-alpine
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=semver,pattern={{major}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
- name: Build and push (alpine)
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
tags: ${{ steps.meta-alpine.outputs.tags }}
|
||||
labels: ${{ steps.meta-alpine.outputs.labels }}
|
||||
cache-from: type=gha,scope=alpine
|
||||
cache-to: type=gha,mode=max,scope=alpine
|
||||
|
||||
# ── RED OS ───────────────────────────────────────────────────────────────
|
||||
- name: Extract metadata (redos)
|
||||
id: meta-redos
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
flavor: suffix=-redos,onlatest=true
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=redos
|
||||
|
||||
- name: Build and push (redos)
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.redos
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: ${{ steps.meta-redos.outputs.tags }}
|
||||
labels: ${{ steps.meta-redos.outputs.labels }}
|
||||
cache-from: type=gha,scope=redos
|
||||
cache-to: type=gha,mode=max,scope=redos
|
||||
|
||||
scan:
|
||||
name: Scan (${{ matrix.name }})
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: alpine
|
||||
suffix: ""
|
||||
- name: redos
|
||||
suffix: "-redos"
|
||||
|
||||
steps:
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set version tag (strip leading v)
|
||||
id: ver
|
||||
run: echo "tag=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT
|
||||
|
||||
# ── CVE scan of the pushed image ────────────────────────────────────────
|
||||
# Images are FROM scratch — no OS packages, only binary CVE scan
|
||||
- name: Trivy — image scan (${{ matrix.name }})
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.ver.outputs.tag }}${{ matrix.suffix }}
|
||||
format: sarif
|
||||
output: trivy-image-${{ matrix.name }}.sarif
|
||||
severity: HIGH,CRITICAL
|
||||
exit-code: 0 # warn only; change to 1 to block on vulnerabilities
|
||||
|
||||
- name: Upload Trivy image results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-image-${{ matrix.name }}.sarif
|
||||
category: trivy-image-${{ matrix.name }}
|
||||
|
||||
release:
|
||||
name: GitHub Release
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
needs: [build, scan]
|
||||
permissions:
|
||||
contents: write
|
||||
packages: read # to pull image for SBOM generation
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set version tag (strip leading v)
|
||||
id: ver
|
||||
run: echo "tag=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT
|
||||
|
||||
# ── SBOM — Software Bill of Materials ───────────────────────────────────
|
||||
- name: Generate SBOM (SPDX)
|
||||
uses: anchore/sbom-action@v0
|
||||
with:
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.ver.outputs.tag }}
|
||||
format: spdx-json
|
||||
output-file: nora-${{ github.ref_name }}.sbom.spdx.json
|
||||
registry-username: ${{ github.actor }}
|
||||
registry-password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Generate SBOM (CycloneDX)
|
||||
uses: anchore/sbom-action@v0
|
||||
with:
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.ver.outputs.tag }}
|
||||
format: cyclonedx-json
|
||||
output-file: nora-${{ github.ref_name }}.sbom.cdx.json
|
||||
registry-username: ${{ github.actor }}
|
||||
registry-password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
nora-${{ github.ref_name }}.sbom.spdx.json
|
||||
nora-${{ github.ref_name }}.sbom.cdx.json
|
||||
body: |
|
||||
## Docker
|
||||
|
||||
**Alpine (standard):**
|
||||
```bash
|
||||
docker pull ghcr.io/${{ github.repository }}:${{ github.ref_name }}
|
||||
```
|
||||
|
||||
**RED OS:**
|
||||
```bash
|
||||
docker pull ghcr.io/${{ github.repository }}:${{ github.ref_name }}-redos
|
||||
```
|
||||
|
||||
## Changelog
|
||||
|
||||
See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md)
|
||||
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -13,3 +13,7 @@ ROADMAP*.md
|
||||
docs-site/
|
||||
docs/
|
||||
*.txt
|
||||
|
||||
## Internal files
|
||||
.internal/
|
||||
examples/
|
||||
|
||||
6
Cargo.lock
generated
6
Cargo.lock
generated
@@ -1201,7 +1201,7 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
|
||||
|
||||
[[package]]
|
||||
name = "nora-cli"
|
||||
version = "0.2.18"
|
||||
version = "0.2.20"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"flate2",
|
||||
@@ -1215,7 +1215,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nora-registry"
|
||||
version = "0.2.18"
|
||||
version = "0.2.20"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -1253,7 +1253,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nora-storage"
|
||||
version = "0.2.18"
|
||||
version = "0.2.20"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"base64",
|
||||
|
||||
@@ -7,7 +7,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.18"
|
||||
version = "0.2.21"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
authors = ["DevITWay <devitway@gmail.com>"]
|
||||
|
||||
55
Dockerfile
55
Dockerfile
@@ -1,58 +1,11 @@
|
||||
# syntax=docker/dockerfile:1.4
|
||||
|
||||
# Build stage
|
||||
FROM rust:1.83-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache musl-dev curl
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy manifests
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
COPY nora-registry/Cargo.toml nora-registry/
|
||||
COPY nora-storage/Cargo.toml nora-storage/
|
||||
COPY nora-cli/Cargo.toml nora-cli/
|
||||
|
||||
# Create dummy sources for dependency caching
|
||||
RUN mkdir -p nora-registry/src nora-storage/src nora-cli/src && \
|
||||
echo "fn main() {}" > nora-registry/src/main.rs && \
|
||||
echo "fn main() {}" > nora-storage/src/main.rs && \
|
||||
echo "fn main() {}" > nora-cli/src/main.rs
|
||||
|
||||
# Build dependencies only (with cache)
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/target \
|
||||
cargo build --release --package nora-registry && \
|
||||
rm -rf nora-registry/src nora-storage/src nora-cli/src
|
||||
|
||||
# Copy real sources
|
||||
COPY nora-registry/src nora-registry/src
|
||||
COPY nora-storage/src nora-storage/src
|
||||
COPY nora-cli/src nora-cli/src
|
||||
|
||||
# Build release binary (with cache)
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/target \
|
||||
touch nora-registry/src/main.rs && \
|
||||
cargo build --release --package nora-registry && \
|
||||
cp /app/target/release/nora /usr/local/bin/nora
|
||||
|
||||
# Runtime stage
|
||||
# Binary is pre-built by CI (cargo build --release) and passed via context
|
||||
FROM alpine:3.20
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
RUN apk add --no-cache ca-certificates && mkdir -p /data
|
||||
|
||||
WORKDIR /app
|
||||
COPY nora /usr/local/bin/nora
|
||||
|
||||
# Copy binary
|
||||
COPY --from=builder /usr/local/bin/nora /usr/local/bin/nora
|
||||
|
||||
# Create data directory
|
||||
RUN mkdir -p /data
|
||||
|
||||
# Default environment
|
||||
ENV RUST_LOG=info
|
||||
ENV NORA_HOST=0.0.0.0
|
||||
ENV NORA_PORT=4000
|
||||
@@ -64,5 +17,5 @@ EXPOSE 4000
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["nora"]
|
||||
ENTRYPOINT ["/usr/local/bin/nora"]
|
||||
CMD ["serve"]
|
||||
|
||||
28
Dockerfile.astra
Normal file
28
Dockerfile.astra
Normal file
@@ -0,0 +1,28 @@
|
||||
# syntax=docker/dockerfile:1.4
|
||||
# Binary is pre-built by CI (cargo build --release) and passed via context
|
||||
# Runtime: scratch — compatible with Astra Linux SE (FSTEC certified)
|
||||
# To switch to official base: replace FROM scratch with
|
||||
# FROM registry.astralinux.ru/library/alse:latest
|
||||
# RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
FROM alpine:3.20 AS certs
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY nora /usr/local/bin/nora
|
||||
|
||||
ENV RUST_LOG=info
|
||||
ENV NORA_HOST=0.0.0.0
|
||||
ENV NORA_PORT=4000
|
||||
ENV NORA_STORAGE_MODE=local
|
||||
ENV NORA_STORAGE_PATH=/data/storage
|
||||
ENV NORA_AUTH_TOKEN_STORAGE=/data/tokens
|
||||
|
||||
EXPOSE 4000
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/nora"]
|
||||
CMD ["serve"]
|
||||
28
Dockerfile.redos
Normal file
28
Dockerfile.redos
Normal file
@@ -0,0 +1,28 @@
|
||||
# syntax=docker/dockerfile:1.4
|
||||
# Binary is pre-built by CI (cargo build --release) and passed via context
|
||||
# Runtime: scratch — compatible with RED OS (FSTEC certified)
|
||||
# To switch to official base: replace FROM scratch with
|
||||
# FROM registry.red-soft.ru/redos/redos:8
|
||||
# RUN dnf install -y ca-certificates && dnf clean all
|
||||
|
||||
FROM alpine:3.20 AS certs
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY nora /usr/local/bin/nora
|
||||
|
||||
ENV RUST_LOG=info
|
||||
ENV NORA_HOST=0.0.0.0
|
||||
ENV NORA_PORT=4000
|
||||
ENV NORA_STORAGE_MODE=local
|
||||
ENV NORA_STORAGE_PATH=/data/storage
|
||||
ENV NORA_AUTH_TOKEN_STORAGE=/data/tokens
|
||||
|
||||
EXPOSE 4000
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/nora"]
|
||||
CMD ["serve"]
|
||||
41
deny.toml
Normal file
41
deny.toml
Normal file
@@ -0,0 +1,41 @@
|
||||
# cargo-deny configuration
|
||||
# https://embarkstudios.github.io/cargo-deny/
|
||||
|
||||
[advisories]
|
||||
# Vulnerability database (RustSec)
|
||||
db-urls = ["https://github.com/rustsec/advisory-db"]
|
||||
ignore = []
|
||||
|
||||
[licenses]
|
||||
# Allowed open-source licenses
|
||||
allow = [
|
||||
"MIT",
|
||||
"Apache-2.0",
|
||||
"Apache-2.0 WITH LLVM-exception",
|
||||
"BSD-2-Clause",
|
||||
"BSD-3-Clause",
|
||||
"ISC",
|
||||
"Unicode-DFS-2016",
|
||||
"Unicode-3.0",
|
||||
"CC0-1.0",
|
||||
"OpenSSL",
|
||||
"Zlib",
|
||||
"MPL-2.0", # Mozilla Public License — ok for binary linking
|
||||
]
|
||||
copyleft = "warn" # GPL etc — warn, don't block
|
||||
unlicensed = "deny"
|
||||
|
||||
[bans]
|
||||
multiple-versions = "warn"
|
||||
deny = [
|
||||
# Prefer rustls over openssl for static builds and supply chain cleanliness
|
||||
{ name = "openssl-sys" },
|
||||
{ name = "openssl" },
|
||||
]
|
||||
skip = []
|
||||
|
||||
[sources]
|
||||
unknown-registry = "warn"
|
||||
unknown-git = "warn"
|
||||
# Allow only the official crates.io index
|
||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
@@ -63,11 +63,17 @@ impl HtpasswdAuth {
|
||||
fn is_public_path(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/" | "/health" | "/ready" | "/metrics" | "/v2/" | "/v2"
|
||||
"/" | "/health"
|
||||
| "/ready"
|
||||
| "/metrics"
|
||||
| "/v2/"
|
||||
| "/v2"
|
||||
| "/api/tokens"
|
||||
| "/api/tokens/list"
|
||||
| "/api/tokens/revoke"
|
||||
) || path.starts_with("/ui")
|
||||
|| path.starts_with("/api-docs")
|
||||
|| path.starts_with("/api/ui")
|
||||
|| path.starts_with("/api/tokens")
|
||||
}
|
||||
|
||||
/// Auth middleware - supports Basic auth and Bearer tokens
|
||||
@@ -404,8 +410,12 @@ mod tests {
|
||||
assert!(is_public_path("/api/ui/stats"));
|
||||
assert!(is_public_path("/api/tokens"));
|
||||
assert!(is_public_path("/api/tokens/list"));
|
||||
assert!(is_public_path("/api/tokens/revoke"));
|
||||
|
||||
// Protected paths
|
||||
assert!(!is_public_path("/api/tokens/unknown"));
|
||||
assert!(!is_public_path("/api/tokens/admin"));
|
||||
assert!(!is_public_path("/api/tokens/extra/path"));
|
||||
assert!(!is_public_path("/v2/myimage/blobs/sha256:abc"));
|
||||
assert!(!is_public_path("/v2/library/nginx/manifests/latest"));
|
||||
assert!(!is_public_path(
|
||||
|
||||
@@ -85,6 +85,7 @@ pub struct AppState {
|
||||
pub activity: ActivityLog,
|
||||
pub docker_auth: registry::DockerAuth,
|
||||
pub repo_index: RepoIndex,
|
||||
pub http_client: reqwest::Client,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
@@ -271,6 +272,8 @@ async fn run_server(config: Config, storage: Storage) {
|
||||
// Initialize Docker auth with proxy timeout
|
||||
let docker_auth = registry::DockerAuth::new(config.docker.proxy_timeout);
|
||||
|
||||
let http_client = reqwest::Client::new();
|
||||
|
||||
let state = Arc::new(AppState {
|
||||
storage,
|
||||
config,
|
||||
@@ -281,6 +284,7 @@ async fn run_server(config: Config, storage: Storage) {
|
||||
activity: ActivityLog::new(50),
|
||||
docker_auth,
|
||||
repo_index: RepoIndex::new(),
|
||||
http_client,
|
||||
});
|
||||
|
||||
// Token routes with strict rate limiting (brute-force protection)
|
||||
|
||||
@@ -167,6 +167,7 @@ async fn download_blob(
|
||||
// Try upstream proxies
|
||||
for upstream in &state.config.docker.upstreams {
|
||||
if let Ok(data) = fetch_blob_from_upstream(
|
||||
&state.http_client,
|
||||
&upstream.url,
|
||||
&name,
|
||||
&digest,
|
||||
@@ -367,6 +368,7 @@ async fn get_manifest(
|
||||
for upstream in &state.config.docker.upstreams {
|
||||
tracing::debug!(upstream_url = %upstream.url, "Trying upstream");
|
||||
if let Ok((data, content_type)) = fetch_manifest_from_upstream(
|
||||
&state.http_client,
|
||||
&upstream.url,
|
||||
&name,
|
||||
&reference,
|
||||
@@ -581,6 +583,7 @@ async fn list_tags_ns(
|
||||
|
||||
/// Fetch a blob from an upstream Docker registry
|
||||
async fn fetch_blob_from_upstream(
|
||||
client: &reqwest::Client,
|
||||
upstream_url: &str,
|
||||
name: &str,
|
||||
digest: &str,
|
||||
@@ -594,13 +597,13 @@ async fn fetch_blob_from_upstream(
|
||||
digest
|
||||
);
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(timeout))
|
||||
.build()
|
||||
.map_err(|_| ())?;
|
||||
|
||||
// First try without auth
|
||||
let response = client.get(&url).send().await.map_err(|_| ())?;
|
||||
let response = client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(timeout))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| ())?;
|
||||
|
||||
let response = if response.status() == reqwest::StatusCode::UNAUTHORIZED {
|
||||
// Get Www-Authenticate header and fetch token
|
||||
@@ -637,6 +640,7 @@ async fn fetch_blob_from_upstream(
|
||||
/// Fetch a manifest from an upstream Docker registry
|
||||
/// Returns (manifest_bytes, content_type)
|
||||
async fn fetch_manifest_from_upstream(
|
||||
client: &reqwest::Client,
|
||||
upstream_url: &str,
|
||||
name: &str,
|
||||
reference: &str,
|
||||
@@ -652,13 +656,6 @@ async fn fetch_manifest_from_upstream(
|
||||
|
||||
tracing::debug!(url = %url, "Fetching manifest from upstream");
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(timeout))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
tracing::error!(error = %e, "Failed to build HTTP client");
|
||||
})?;
|
||||
|
||||
// Request with Accept header for manifest types
|
||||
let accept_header = "application/vnd.docker.distribution.manifest.v2+json, \
|
||||
application/vnd.docker.distribution.manifest.list.v2+json, \
|
||||
@@ -668,6 +665,7 @@ async fn fetch_manifest_from_upstream(
|
||||
// First try without auth
|
||||
let response = client
|
||||
.get(&url)
|
||||
.timeout(Duration::from_secs(timeout))
|
||||
.header("Accept", accept_header)
|
||||
.send()
|
||||
.await
|
||||
|
||||
@@ -23,7 +23,6 @@ pub fn routes() -> Router<Arc<AppState>> {
|
||||
async fn download(State(state): State<Arc<AppState>>, Path(path): Path<String>) -> Response {
|
||||
let key = format!("maven/{}", path);
|
||||
|
||||
// Extract artifact name for logging (last 2-3 path components)
|
||||
let artifact_name = path
|
||||
.split('/')
|
||||
.rev()
|
||||
@@ -34,7 +33,6 @@ async fn download(State(state): State<Arc<AppState>>, Path(path): Path<String>)
|
||||
.collect::<Vec<_>>()
|
||||
.join("/");
|
||||
|
||||
// Try local storage first
|
||||
if let Ok(data) = state.storage.get(&key).await {
|
||||
state.metrics.record_download("maven");
|
||||
state.metrics.record_cache_hit();
|
||||
@@ -47,11 +45,10 @@ async fn download(State(state): State<Arc<AppState>>, Path(path): Path<String>)
|
||||
return with_content_type(&path, data).into_response();
|
||||
}
|
||||
|
||||
// Try proxy servers
|
||||
for proxy_url in &state.config.maven.proxies {
|
||||
let url = format!("{}/{}", proxy_url.trim_end_matches('/'), path);
|
||||
|
||||
match fetch_from_proxy(&url, state.config.maven.proxy_timeout).await {
|
||||
match fetch_from_proxy(&state.http_client, &url, state.config.maven.proxy_timeout).await {
|
||||
Ok(data) => {
|
||||
state.metrics.record_download("maven");
|
||||
state.metrics.record_cache_miss();
|
||||
@@ -62,7 +59,6 @@ async fn download(State(state): State<Arc<AppState>>, Path(path): Path<String>)
|
||||
"PROXY",
|
||||
));
|
||||
|
||||
// Cache in local storage (fire and forget)
|
||||
let storage = state.storage.clone();
|
||||
let key_clone = key.clone();
|
||||
let data_clone = data.clone();
|
||||
@@ -88,7 +84,6 @@ async fn upload(
|
||||
) -> StatusCode {
|
||||
let key = format!("maven/{}", path);
|
||||
|
||||
// Extract artifact name for logging
|
||||
let artifact_name = path
|
||||
.split('/')
|
||||
.rev()
|
||||
@@ -115,14 +110,18 @@ async fn upload(
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_from_proxy(url: &str, timeout_secs: u64) -> Result<Vec<u8>, ()> {
|
||||
let client = reqwest::Client::builder()
|
||||
async fn fetch_from_proxy(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
timeout_secs: u64,
|
||||
) -> Result<Vec<u8>, ()> {
|
||||
let response = client
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(timeout_secs))
|
||||
.build()
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| ())?;
|
||||
|
||||
let response = client.get(url).send().await.map_err(|_| ())?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(());
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ pub fn routes() -> Router<Arc<AppState>> {
|
||||
}
|
||||
|
||||
async fn handle_request(State(state): State<Arc<AppState>>, Path(path): Path<String>) -> Response {
|
||||
// Determine if this is a tarball request or metadata request
|
||||
let is_tarball = path.contains("/-/");
|
||||
|
||||
let key = if is_tarball {
|
||||
@@ -33,14 +32,12 @@ async fn handle_request(State(state): State<Arc<AppState>>, Path(path): Path<Str
|
||||
format!("npm/{}/metadata.json", path)
|
||||
};
|
||||
|
||||
// Extract package name for logging
|
||||
let package_name = if is_tarball {
|
||||
path.split("/-/").next().unwrap_or(&path).to_string()
|
||||
} else {
|
||||
path.clone()
|
||||
};
|
||||
|
||||
// Try local storage first
|
||||
if let Ok(data) = state.storage.get(&key).await {
|
||||
if is_tarball {
|
||||
state.metrics.record_download("npm");
|
||||
@@ -55,17 +52,12 @@ async fn handle_request(State(state): State<Arc<AppState>>, Path(path): Path<Str
|
||||
return with_content_type(is_tarball, data).into_response();
|
||||
}
|
||||
|
||||
// Try proxy if configured
|
||||
if let Some(proxy_url) = &state.config.npm.proxy {
|
||||
let url = if is_tarball {
|
||||
// Tarball URL: https://registry.npmjs.org/package/-/package-version.tgz
|
||||
format!("{}/{}", proxy_url.trim_end_matches('/'), path)
|
||||
} else {
|
||||
// Metadata URL: https://registry.npmjs.org/package
|
||||
format!("{}/{}", proxy_url.trim_end_matches('/'), path)
|
||||
};
|
||||
let url = format!("{}/{}", proxy_url.trim_end_matches('/'), path);
|
||||
|
||||
if let Ok(data) = fetch_from_proxy(&url, state.config.npm.proxy_timeout).await {
|
||||
if let Ok(data) =
|
||||
fetch_from_proxy(&state.http_client, &url, state.config.npm.proxy_timeout).await
|
||||
{
|
||||
if is_tarball {
|
||||
state.metrics.record_download("npm");
|
||||
state.metrics.record_cache_miss();
|
||||
@@ -77,7 +69,6 @@ async fn handle_request(State(state): State<Arc<AppState>>, Path(path): Path<Str
|
||||
));
|
||||
}
|
||||
|
||||
// Cache in local storage (fire and forget)
|
||||
let storage = state.storage.clone();
|
||||
let key_clone = key.clone();
|
||||
let data_clone = data.clone();
|
||||
@@ -85,7 +76,6 @@ async fn handle_request(State(state): State<Arc<AppState>>, Path(path): Path<Str
|
||||
let _ = storage.put(&key_clone, &data_clone).await;
|
||||
});
|
||||
|
||||
// Invalidate index when caching new tarball
|
||||
if is_tarball {
|
||||
state.repo_index.invalidate("npm");
|
||||
}
|
||||
@@ -97,14 +87,18 @@ async fn handle_request(State(state): State<Arc<AppState>>, Path(path): Path<Str
|
||||
StatusCode::NOT_FOUND.into_response()
|
||||
}
|
||||
|
||||
async fn fetch_from_proxy(url: &str, timeout_secs: u64) -> Result<Vec<u8>, ()> {
|
||||
let client = reqwest::Client::builder()
|
||||
async fn fetch_from_proxy(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
timeout_secs: u64,
|
||||
) -> Result<Vec<u8>, ()> {
|
||||
let response = client
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(timeout_secs))
|
||||
.build()
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| ())?;
|
||||
|
||||
let response = client.get(url).send().await.map_err(|_| ())?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(());
|
||||
}
|
||||
|
||||
@@ -85,7 +85,9 @@ async fn package_versions(
|
||||
if let Some(proxy_url) = &state.config.pypi.proxy {
|
||||
let url = format!("{}/{}/", proxy_url.trim_end_matches('/'), normalized);
|
||||
|
||||
if let Ok(html) = fetch_package_page(&url, state.config.pypi.proxy_timeout).await {
|
||||
if let Ok(html) =
|
||||
fetch_package_page(&state.http_client, &url, state.config.pypi.proxy_timeout).await
|
||||
{
|
||||
// Rewrite URLs in the HTML to point to our registry
|
||||
let rewritten = rewrite_pypi_links(&html, &normalized);
|
||||
return (StatusCode::OK, Html(rewritten)).into_response();
|
||||
@@ -130,10 +132,22 @@ async fn download_file(
|
||||
// First, fetch the package page to find the actual download URL
|
||||
let page_url = format!("{}/{}/", proxy_url.trim_end_matches('/'), normalized);
|
||||
|
||||
if let Ok(html) = fetch_package_page(&page_url, state.config.pypi.proxy_timeout).await {
|
||||
if let Ok(html) = fetch_package_page(
|
||||
&state.http_client,
|
||||
&page_url,
|
||||
state.config.pypi.proxy_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
// Find the URL for this specific file
|
||||
if let Some(file_url) = find_file_url(&html, &filename) {
|
||||
if let Ok(data) = fetch_file(&file_url, state.config.pypi.proxy_timeout).await {
|
||||
if let Ok(data) = fetch_file(
|
||||
&state.http_client,
|
||||
&file_url,
|
||||
state.config.pypi.proxy_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
state.metrics.record_download("pypi");
|
||||
state.metrics.record_cache_miss();
|
||||
state.activity.push(ActivityEntry::new(
|
||||
@@ -177,14 +191,14 @@ fn normalize_name(name: &str) -> String {
|
||||
}
|
||||
|
||||
/// Fetch package page from upstream
|
||||
async fn fetch_package_page(url: &str, timeout_secs: u64) -> Result<String, ()> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(timeout_secs))
|
||||
.build()
|
||||
.map_err(|_| ())?;
|
||||
|
||||
async fn fetch_package_page(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
timeout_secs: u64,
|
||||
) -> Result<String, ()> {
|
||||
let response = client
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(timeout_secs))
|
||||
.header("Accept", "text/html")
|
||||
.send()
|
||||
.await
|
||||
@@ -198,14 +212,14 @@ async fn fetch_package_page(url: &str, timeout_secs: u64) -> Result<String, ()>
|
||||
}
|
||||
|
||||
/// Fetch file from upstream
|
||||
async fn fetch_file(url: &str, timeout_secs: u64) -> Result<Vec<u8>, ()> {
|
||||
let client = reqwest::Client::builder()
|
||||
async fn fetch_file(client: &reqwest::Client, url: &str, timeout_secs: u64) -> Result<Vec<u8>, ()> {
|
||||
let response = client
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(timeout_secs))
|
||||
.build()
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| ())?;
|
||||
|
||||
let response = client.get(url).send().await.map_err(|_| ())?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user