diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..153dd9f --- /dev/null +++ b/TODO.md @@ -0,0 +1,440 @@ +# NORA Roadmap / TODO + +## v0.2.0 - DONE +- [x] Unit tests (75 tests passing) +- [x] Input validation (path traversal protection) +- [x] Rate limiting (brute-force protection) +- [x] Request ID tracking +- [x] Migrate command (local <-> S3) +- [x] Error handling (thiserror) +- [x] SVG brand icons + +--- + +## v0.3.0 - OIDC / Workload Identity Federation + +### Killer Feature: OIDC for CI/CD +Zero-secret authentication for GitHub Actions, GitLab CI, etc. + +**Goal:** Replace manual `ROBOT_TOKEN` rotation with federated identity. + +```yaml +# GitHub Actions example +permissions: + id-token: write +steps: + - name: Login to NORA + uses: nora/login-action@v1 +``` + +### Config Structure (draft) + +```toml +[auth.oidc] +enabled = true + +# GitHub Actions +[[auth.oidc.providers]] +name = "github-actions" +issuer = "https://token.actions.githubusercontent.com" +audience = "https://nora.example.com" + +[[auth.oidc.providers.rules]] +# Claim matching (supports glob) +match = { repository = "my-org/*", ref = "refs/heads/main" } +# Granted permissions +permissions = ["push:my-org/*", "pull:*"] + +[[auth.oidc.providers.rules]] +match = { repository = "my-org/*", ref = "refs/heads/*" } +permissions = ["pull:*"] + +# GitLab CI +[[auth.oidc.providers]] +name = "gitlab-ci" +issuer = "https://gitlab.com" +audience = "https://nora.example.com" + +[[auth.oidc.providers.rules]] +match = { project_path = "my-group/*" } +permissions = ["push:my-group/*", "pull:*"] +``` + +### Implementation Tasks +- [ ] JWT validation library (jsonwebtoken crate) +- [ ] OIDC discovery (/.well-known/openid-configuration) +- [ ] JWKS fetching and caching +- [ ] Claims extraction and glob matching +- [ ] Permission resolution from rules +- [ ] Token exchange endpoint (POST /auth/oidc/token) +- [ ] GitHub Action: `nora/login-action` + +--- + +## v0.4.0 - Transparent Docker Hub Proxy + +### Pain Point +Harbor forces tag changes: `docker pull my-harbor/proxy-cache/library/nginx` +This breaks Helm charts hardcoded to `nginx`. + +### Goal +Transparent pull-through cache: +```bash +docker pull nora.example.com/nginx # -> proxies to Docker Hub +``` + +### Implementation Tasks +- [ ] Registry v2 API interception +- [ ] Upstream registry configuration +- [ ] Cache layer management +- [ ] Rate limit handling (Docker Hub limits) + +--- + +## v0.5.0 - Repo-level RBAC + +### Challenge +Per-repository permissions need fast lookup (100 layers per push). + +### Solution +Glob patterns for 90% of cases: +```toml +[[auth.rules]] +subject = "team-frontend" +permissions = ["push:frontend/*", "pull:*"] + +[[auth.rules]] +subject = "ci-bot" +permissions = ["push:*/release-*", "pull:*"] +``` + +### Implementation Tasks +- [ ] In-memory permission cache +- [ ] Glob pattern matcher (globset crate) +- [ ] Permission inheritance (org -> project -> repo) + +--- + +## Target Audience + +1. DevOps engineers tired of Java/Go monsters +2. Edge/IoT installations (Raspberry Pi, branch offices) +3. Educational platforms (student labs) +4. CI/CD pipelines (GitHub Actions, GitLab CI) + +## Competitive Advantages + +| Feature | NORA | Harbor | Nexus | +|---------|------|--------|-------| +| Memory | <100MB | 2GB+ | 4GB+ | +| OIDC for CI | v0.3.0 | No | No | +| Transparent proxy | v0.4.0 | No (tag rewrite) | Partial | +| Single binary | Yes | No (microservices) | No (Java) | +| Zero-config upgrade | Yes | Complex | Complex | + +--- + +## v0.6.0 - Online Garbage Collection + +### Pain Point +Harbor GC blocks registry for hours. Can't push during cleanup. + +### Goal +Non-blocking garbage collection with zero downtime. + +### Implementation Tasks +- [ ] Mark-and-sweep without locking +- [ ] Background blob cleanup +- [ ] Progress reporting via API/CLI +- [ ] `nora gc --dry-run` preview + +--- + +## v0.7.0 - Retention Policies + +### Pain Point +"Keep last 10 tags" sounds simple, works poorly everywhere. + +### Goal +Declarative retention rules in config: + +```toml +[[retention]] +match = "*/dev-*" +keep_last = 5 + +[[retention]] +match = "*/release-*" +keep_last = 20 +older_than = "90d" + +[[retention]] +match = "**/pr-*" +older_than = "7d" +``` + +### Implementation Tasks +- [ ] Glob pattern matching for repos/tags +- [ ] Age-based and count-based rules +- [ ] Dry-run mode +- [ ] Scheduled execution (cron-style) + +--- + +## v0.8.0 - Multi-tenancy & Quotas + +### Pain Point +Harbor projects have quotas but configuration is painful. Nexus has no real isolation. + +### Goal +Simple namespaces with limits: + +```toml +[[tenants]] +name = "team-frontend" +storage_quota = "50GB" +rate_limit = { push = 100, pull = 1000 } # per hour + +[[tenants]] +name = "team-backend" +storage_quota = "100GB" +``` + +### Implementation Tasks +- [ ] Tenant isolation (namespace prefix) +- [ ] Storage quota tracking +- [ ] Per-tenant rate limiting +- [ ] Usage reporting API + +--- + +## v0.9.0 - Smart Replication + +### Pain Point +Harbor replication rules are complex, errors silently swallowed. + +### Goal +Simple CLI-driven replication with clear feedback: + +```bash +nora replicate --to remote-dc --filter "prod/*" --dry-run +nora replicate --from gcr.io/my-project/* --to local/imported/ +``` + +### Implementation Tasks +- [ ] Push-based replication to remote NORA +- [ ] Pull-based import from external registries (Docker Hub, GCR, ECR, Quay) +- [ ] Filter by glob patterns +- [ ] Progress bar and detailed logs +- [ ] Retry logic with exponential backoff + +--- + +## v1.0.0 - Production Ready + +### Features to polish +- [ ] Full CLI (`nora images ls`, `nora tag`, `nora delete`) +- [ ] Webhooks with filters and retry logic +- [ ] Enhanced Prometheus metrics (per-repo stats, cache hit ratio, bandwidth per tenant) +- [ ] TUI dashboard (optional) +- [ ] Helm chart for Kubernetes deployment +- [ ] Official Docker image on ghcr.io + +--- + +## Future Ideas (v1.x+) + +### Cold Storage Tiering +Auto-move old tags to S3 Glacier: +```toml +[[storage.tiering]] +match = "*" +older_than = "180d" +move_to = "s3-glacier" +``` + +### Vulnerability Scanning Integration +Not built-in (use Trivy), but: +- [ ] Webhook on push -> trigger external scan +- [ ] Store scan results as OCI artifacts +- [ ] Block pull if critical CVEs (policy) + +### Image Signing (Cosign/Notation) +- [ ] Signature storage (OCI artifacts) +- [ ] Policy enforcement (reject unsigned) + +### P2P Distribution (Dragonfly/Kraken style) +For large clusters pulling same image simultaneously. + +--- + +--- + +## Architecture / DDD + +### Current State (v0.2.0) +Monolithic structure, all in `nora-registry/src/`: +``` +src/ +├── main.rs # CLI + server setup +├── auth.rs # htpasswd + basic auth +├── tokens.rs # API tokens +├── storage/ # Storage backends (local, s3) +├── registry/ # Protocol handlers (docker, maven, npm, cargo, pypi) +├── ui/ # Web dashboard +└── ... +``` + +### Target Architecture (v1.0+) + +#### Domain-Driven Design Boundaries + +``` +nora/ +├── nora-core/ # Domain layer (no dependencies) +│ ├── src/ +│ │ ├── artifact.rs # Artifact, Digest, Tag, Manifest +│ │ ├── repository.rs # Repository, Namespace +│ │ ├── identity.rs # User, ServiceAccount, Token +│ │ ├── policy.rs # Permission, Rule, Quota +│ │ └── events.rs # DomainEvent (ArtifactPushed, etc.) +│ +├── nora-auth/ # Authentication bounded context +│ ├── src/ +│ │ ├── htpasswd.rs # Basic auth provider +│ │ ├── oidc.rs # OIDC/JWT provider +│ │ ├── token.rs # API token provider +│ │ └── rbac.rs # Permission resolver +│ +├── nora-storage/ # Storage bounded context +│ ├── src/ +│ │ ├── backend.rs # StorageBackend trait +│ │ ├── local.rs # Filesystem +│ │ ├── s3.rs # S3-compatible +│ │ ├── tiered.rs # Hot/cold tiering +│ │ └── gc.rs # Garbage collection +│ +├── nora-registry/ # Application layer (HTTP API) +│ ├── src/ +│ │ ├── api/ +│ │ │ ├── oci.rs # OCI Distribution API (/v2/) +│ │ │ ├── maven.rs # Maven repository +│ │ │ ├── npm.rs # npm registry +│ │ │ ├── cargo.rs # Cargo registry +│ │ │ └── pypi.rs # PyPI (simple API) +│ │ ├── proxy/ # Upstream proxy/cache +│ │ ├── webhook/ # Event webhooks +│ │ └── ui/ # Web dashboard +│ +├── nora-cli/ # CLI application +│ ├── src/ +│ │ ├── commands/ +│ │ │ ├── serve.rs +│ │ │ ├── images.rs # nora images ls/delete/tag +│ │ │ ├── gc.rs # nora gc +│ │ │ ├── backup.rs # nora backup/restore +│ │ │ ├── migrate.rs # nora migrate +│ │ │ └── replicate.rs +│ │ └── tui/ # Optional TUI dashboard +│ +└── nora-sdk/ # Client SDK (for nora/login-action) + └── src/ + ├── client.rs # HTTP client + └── oidc.rs # Token exchange +``` + +#### Key Principles + +1. **Hexagonal Architecture** + - Core domain has no external dependencies + - Ports (traits) define boundaries + - Adapters implement ports (S3, filesystem, OIDC providers) + +2. **Event-Driven** + - Domain events: `ArtifactPushed`, `ArtifactDeleted`, `TagCreated` + - Webhooks subscribe to events + - Async processing for GC, replication + +3. **CQRS-lite** + - Commands: Push, Delete, CreateToken + - Queries: List, Get, Search + - Separate read/write paths for hot endpoints + +4. **Configuration as Code** + - All policies in `nora.toml` + - No database for config (file-based) + - GitOps friendly + +#### Trait Boundaries (Ports) + +```rust +// nora-core/src/ports.rs + +#[async_trait] +pub trait ArtifactStore { + async fn push_blob(&self, digest: &Digest, data: Bytes) -> Result<()>; + async fn get_blob(&self, digest: &Digest) -> Result; + async fn push_manifest(&self, repo: &Repository, tag: &Tag, manifest: &Manifest) -> Result<()>; + async fn get_manifest(&self, repo: &Repository, reference: &Reference) -> Result; + async fn list_tags(&self, repo: &Repository) -> Result>; + async fn delete(&self, repo: &Repository, reference: &Reference) -> Result<()>; +} + +#[async_trait] +pub trait IdentityProvider { + async fn authenticate(&self, credentials: &Credentials) -> Result; + async fn authorize(&self, identity: &Identity, action: &Action, resource: &Resource) -> Result; +} + +#[async_trait] +pub trait EventPublisher { + async fn publish(&self, event: DomainEvent) -> Result<()>; +} +``` + +#### Migration Path + +| Phase | Action | +|-------|--------| +| v0.3 | Extract `nora-auth` crate (OIDC work) | +| v0.4 | Extract `nora-core` domain types | +| v0.5 | Extract `nora-storage` with trait boundaries | +| v0.6+ | Refactor registry handlers to use ports | +| v1.0 | Full hexagonal architecture | + +### Technical Debt to Address + +- [ ] Remove `unwrap()` in non-test code (started in e9984cf) +- [ ] Add tracing spans to all handlers +- [ ] Consistent error types across modules +- [ ] Extract hardcoded limits to config +- [ ] Add OpenTelemetry support (traces, not just metrics) + +### Performance Requirements + +| Metric | Target | +|--------|--------| +| Memory (idle) | <50MB | +| Memory (under load) | <100MB | +| Startup time | <1s | +| Blob throughput | Wire speed (no processing overhead) | +| Manifest latency | <10ms p99 | +| Auth check | <1ms (cached) | + +### Security Requirements + +- [ ] No secrets in logs (already redacting) +- [ ] TLS termination (or trust reverse proxy) +- [ ] Content-addressable storage (immutable blobs) +- [ ] Audit log for all mutations +- [ ] SBOM generation for NORA itself + +--- + +## Notes + +- S3 storage: already implemented +- Web UI: minimalist read-only dashboard (done) +- TUI: consider for v1.0 +- Vulnerability scanning: out of scope (use Trivy externally) +- Image signing: out of scope for now (use cosign externally) diff --git a/nora-registry/src/auth.rs b/nora-registry/src/auth.rs index 9b59334..e8cd9fb 100644 --- a/nora-registry/src/auth.rs +++ b/nora-registry/src/auth.rs @@ -405,7 +405,9 @@ mod tests { // Protected paths assert!(!is_public_path("/v2/myimage/blobs/sha256:abc")); assert!(!is_public_path("/v2/library/nginx/manifests/latest")); - assert!(!is_public_path("/maven2/com/example/artifact/1.0/artifact.jar")); + assert!(!is_public_path( + "/maven2/com/example/artifact/1.0/artifact.jar" + )); assert!(!is_public_path("/npm/lodash")); } diff --git a/nora-registry/src/error.rs b/nora-registry/src/error.rs index 7b33030..90ef176 100644 --- a/nora-registry/src/error.rs +++ b/nora-registry/src/error.rs @@ -1,3 +1,4 @@ +#![allow(dead_code)] //! Application error handling with HTTP response conversion //! //! Provides a unified error type that can be converted to HTTP responses diff --git a/nora-registry/src/main.rs b/nora-registry/src/main.rs index 4e9cb2f..796bde5 100644 --- a/nora-registry/src/main.rs +++ b/nora-registry/src/main.rs @@ -29,11 +29,7 @@ pub use storage::Storage; use tokens::TokenStore; #[derive(Parser)] -#[command( - name = "nora", - version, - about = "Multi-protocol artifact registry" -)] +#[command(name = "nora", version, about = "Multi-protocol artifact registry")] struct Cli { #[command(subcommand)] command: Option, diff --git a/nora-registry/src/migrate.rs b/nora-registry/src/migrate.rs index 52bfa0d..e5f3702 100644 --- a/nora-registry/src/migrate.rs +++ b/nora-registry/src/migrate.rs @@ -8,17 +8,12 @@ use indicatif::{ProgressBar, ProgressStyle}; use tracing::{info, warn}; /// Migration options +#[derive(Default)] pub struct MigrateOptions { /// If true, show what would be migrated without copying pub dry_run: bool, } -impl Default for MigrateOptions { - fn default() -> Self { - Self { dry_run: false } - } -} - /// Migration statistics #[derive(Debug, Default)] pub struct MigrateStats { @@ -64,7 +59,9 @@ pub async fn migrate( let pb = ProgressBar::new(keys.len() as u64); pb.set_style( ProgressStyle::default_bar() - .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})") + .template( + "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})", + ) .expect("Invalid progress bar template") .progress_chars("#>-"), ); diff --git a/nora-registry/src/rate_limit.rs b/nora-registry/src/rate_limit.rs index 0f47a8b..4c0e2c8 100644 --- a/nora-registry/src/rate_limit.rs +++ b/nora-registry/src/rate_limit.rs @@ -1,3 +1,4 @@ +#![allow(dead_code)] //! Rate limiting configuration and middleware //! //! Provides rate limiting to protect against: @@ -27,11 +28,11 @@ pub struct RateLimitConfig { impl Default for RateLimitConfig { fn default() -> Self { Self { - auth_rps: 1, // 1 req/sec for auth (strict) - auth_burst: 5, // Allow burst of 5 - upload_rps: 10, // 10 req/sec for uploads - upload_burst: 20, // Allow burst of 20 - general_rps: 100, // 100 req/sec general + auth_rps: 1, // 1 req/sec for auth (strict) + auth_burst: 5, // Allow burst of 5 + upload_rps: 10, // 10 req/sec for uploads + upload_burst: 20, // Allow burst of 20 + general_rps: 100, // 100 req/sec general general_burst: 200, // Allow burst of 200 } } diff --git a/nora-registry/src/registry/docker.rs b/nora-registry/src/registry/docker.rs index a724d99..b9da170 100644 --- a/nora-registry/src/registry/docker.rs +++ b/nora-registry/src/registry/docker.rs @@ -178,10 +178,7 @@ async fn put_manifest( } } -async fn list_tags( - State(state): State>, - Path(name): Path, -) -> Response { +async fn list_tags(State(state): State>, Path(name): Path) -> Response { if let Err(e) = validate_docker_name(&name) { return (StatusCode::BAD_REQUEST, e.to_string()).into_response(); } diff --git a/nora-registry/src/storage/mod.rs b/nora-registry/src/storage/mod.rs index 891c348..74a36be 100644 --- a/nora-registry/src/storage/mod.rs +++ b/nora-registry/src/storage/mod.rs @@ -76,10 +76,8 @@ impl Storage { pub async fn list(&self, prefix: &str) -> Vec { // Empty prefix is valid for listing all - if !prefix.is_empty() { - if let Err(_) = validate_storage_key(prefix) { - return Vec::new(); - } + if !prefix.is_empty() && validate_storage_key(prefix).is_err() { + return Vec::new(); } self.inner.list(prefix).await } diff --git a/nora-registry/src/ui/templates.rs b/nora-registry/src/ui/templates.rs index 4960809..f280518 100644 --- a/nora-registry/src/ui/templates.rs +++ b/nora-registry/src/ui/templates.rs @@ -59,7 +59,13 @@ pub fn render_dashboard(stats: &RegistryStats) -> String { "##, - stat_card("Docker", icons::DOCKER, stats.docker, "/ui/docker", "images"), + stat_card( + "Docker", + icons::DOCKER, + stats.docker, + "/ui/docker", + "images" + ), stat_card("Maven", icons::MAVEN, stats.maven, "/ui/maven", "artifacts"), stat_card("npm", icons::NPM, stats.npm, "/ui/npm", "packages"), stat_card("Cargo", icons::CARGO, stats.cargo, "/ui/cargo", "crates"), @@ -455,7 +461,9 @@ fn get_registry_icon(registry_type: &str) -> &'static str { "npm" => icons::NPM, "cargo" => icons::CARGO, "pypi" => icons::PYPI, - _ => r#""#, + _ => { + r#""# + } } } diff --git a/nora-registry/src/validation.rs b/nora-registry/src/validation.rs index 8acf074..ea6d534 100644 --- a/nora-registry/src/validation.rs +++ b/nora-registry/src/validation.rs @@ -1,3 +1,4 @@ +#![allow(dead_code)] //! Input validation for artifact registry paths and identifiers //! //! Provides security validation to prevent path traversal attacks and @@ -92,7 +93,7 @@ pub fn validate_storage_key(key: &str) -> Result<(), ValidationError> { // Check each segment for segment in key.split('/') { - if segment.is_empty() && key != "" { + if segment.is_empty() && !key.is_empty() { // Allow trailing slash but not double slashes continue; } diff --git a/nora-storage/src/main.rs b/nora-storage/src/main.rs index e215b35..1c75db3 100644 --- a/nora-storage/src/main.rs +++ b/nora-storage/src/main.rs @@ -133,9 +133,7 @@ async fn main() { .expect("Failed to bind to address"); info!("nora-storage (S3 compatible) running on http://{}", addr); - axum::serve(listener, app) - .await - .expect("Server error"); + axum::serve(listener, app).await.expect("Server error"); } async fn list_buckets(State(state): State>) -> Response {