From 91199476e167e18e0b22d5b9cf397df8d8e14da0 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Wed, 29 Apr 2026 03:35:10 -0400 Subject: [PATCH] feat: comprehensive audit-driven hardening pass Closes 36 findings from the project audit. The headline win is correctness: the daemon now actually does what the README claims, all three audit-log integrity layers verify, and the rotation control loop terminates. Critical fixes: - Orchestrator bumps Credential.last_rotated_at on success and writes a sealed credential_versions row, so the policy evaluator stops re-firing the same rotation every tick - Envelope encryption (AES-256-GCM, KEK-wrapped DEK, AAD-bound) is now wired into the rotation success path; the credential_versions table is no longer dead schema - cre audit verify checks all three layers (hash chain + HMAC ratchet + Merkle batches) instead of only the chain - BatchSealerScheduler runs every 300s and on shutdown so audit_batches actually fills with signed Merkle roots - Compliance bundle exports the real audit_batches list and covers public_key.pem under the manifest's checksum - cre run/watch hard-fail without CRE_HMAC_KEY_HEX + CRE_KEK_HEX rather than silently using zero defaults Quality fixes: - HTTP::Client wrapper with connect/read timeouts + bounded retry-with- jitter on 408/429/5xx; Telegram error logs redact the bot token - EventBus dispatch isolates Block subscribers behind select+timeout so one stuck subscriber can't pin the bus - RotationWorker checks rotations.in_flight before dispatching to dedupe duplicate schedules; PG enforces it at the DB via partial unique index - Commit-step failures transition rotations to Inconsistent (a terminal state) and raise a critical alert instead of pretending success - env_file rotator uses per-PID pending paths plus an exclusive flock on a sibling .lock file so two daemons can't race on the rename - Versioned migration runner replaces the soup of CREATE IF NOT EXISTS; SQLite gains BEFORE UPDATE / BEFORE DELETE triggers on audit_events - AuditSubscriber publishes a critical alert + panics by default when log writes fail, instead of silently dropping - Engine.stop drains via an ack channel with a 2s deadline rather than the previous magic 50ms sleep - Policy DSL moved into CRE::Policy::Dsl module (consumers `include` it); multiple matching policies raise PolicyConflictError - Evaluator uses credentials.overdue() per-max_age group instead of loading every credential each tick - AuditRepo gains each_in_range streaming and all_batches enumeration; bundle export streams instead of buffering - New cre verify-bundle CLI re-runs every check the bundle README documents; new cre tui-demo for synthetic event preview Tests: 179 unit (up from 159), all passing. New rotation_worker_spec, new SigV4 regression vector, new orchestrator success-path coverage (envelope write + last_rotated_at bump + Inconsistent state). Docs: README + learn/00..04 aligned with the post-audit behavior; removed /snooze (was a stub) and the AwsIamKey/Database CredentialKind variants that had no rotators. Added required-env-var documentation. --- .../credential-rotation-enforcer/README.md | 29 ++- .../learn/00-OVERVIEW.md | 22 +- .../learn/01-CONCEPTS.md | 11 +- .../learn/02-ARCHITECTURE.md | 46 ++-- .../learn/03-IMPLEMENTATION.md | 81 +++++-- .../learn/04-CHALLENGES.md | 4 +- .../spec/unit/audit/audit_log_spec.cr | 21 ++ .../spec/unit/aws/signer_aws_vector_spec.cr | 123 ++++++++++ .../spec/unit/domain/credential_spec.cr | 29 ++- .../unit/engine/rotation_orchestrator_spec.cr | 119 ++++++++++ .../spec/unit/engine/rotation_worker_spec.cr | 197 ++++++++++++++++ .../spec/unit/policy/dsl_spec.cr | 12 +- .../spec/unit/policy/evaluator_spec.cr | 2 + .../spec/unit/policy/policy_spec.cr | 52 ++++- .../spec/unit/rotators/env_file_spec.cr | 8 +- .../src/cre/audit/audit_log.cr | 71 ++++++ .../src/cre/audit/batch_sealer_scheduler.cr | 64 +++++ .../src/cre/aws/secrets_client.cr | 3 +- .../src/cre/cli/bootstrap.cr | 122 ++++++++++ .../src/cre/cli/cli.cr | 20 +- .../src/cre/cli/commands.cr | 1 + .../src/cre/cli/commands/audit.cr | 56 ++++- .../src/cre/cli/commands/rotate.cr | 58 ++--- .../src/cre/cli/commands/run.cr | 85 +++---- .../src/cre/cli/commands/verify_bundle.cr | 218 +++++++++++++++++ .../src/cre/cli/commands/watch.cr | 51 ++-- .../src/cre/compliance/bundle.cr | 96 +++++--- .../src/cre/domain/credential.cr | 8 +- .../src/cre/engine/engine.cr | 8 +- .../src/cre/engine/event_bus.cr | 18 +- .../src/cre/engine/rotation_orchestrator.cr | 107 +++++++-- .../src/cre/engine/rotation_worker.cr | 11 +- .../engine/subscribers/audit_subscriber.cr | 34 +++ .../src/cre/events/system_events.cr | 10 + .../src/cre/github/client.cr | 16 +- .../src/cre/http/retry.cr | 90 +++++++ .../src/cre/notifiers/telegram.cr | 24 +- .../src/cre/notifiers/telegram_bot.cr | 7 - .../cre/persistence/postgres/audit_repo.cr | 50 +++- .../persistence/postgres/credentials_repo.cr | 29 ++- .../cre/persistence/postgres/migrations.cr | 221 ++++++++++-------- .../persistence/postgres/rotations_repo.cr | 2 +- .../src/cre/persistence/repos.cr | 4 +- .../src/cre/persistence/sqlite/audit_repo.cr | 50 +++- .../persistence/sqlite/credentials_repo.cr | 32 ++- .../src/cre/persistence/sqlite/migrations.cr | 192 +++++++++------ .../src/cre/policy/dsl.cr | 44 ++-- .../src/cre/policy/evaluator.cr | 35 ++- .../src/cre/policy/policy.cr | 4 +- .../src/cre/rotators/env_file.cr | 40 +++- .../src/cre/tui/renderer.cr | 8 +- .../src/cre/tui/snapshotter.cr | 69 ++++++ .../src/cre/tui/state.cr | 15 ++ .../src/cre/tui/tui.cr | 1 + .../src/cre/vault/client.cr | 7 +- 55 files changed, 2216 insertions(+), 521 deletions(-) create mode 100644 PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/aws/signer_aws_vector_spec.cr create mode 100644 PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_worker_spec.cr create mode 100644 PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/batch_sealer_scheduler.cr create mode 100644 PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/bootstrap.cr create mode 100644 PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/verify_bundle.cr create mode 100644 PROJECTS/intermediate/credential-rotation-enforcer/src/cre/http/retry.cr create mode 100644 PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/snapshotter.cr diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/README.md b/PROJECTS/intermediate/credential-rotation-enforcer/README.md index 85449afe..24346065 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/README.md +++ b/PROJECTS/intermediate/credential-rotation-enforcer/README.md @@ -29,7 +29,7 @@ README.md - Three-layer tamper-evident audit log: SHA-256 hash chain + ratcheting HMAC-SHA256 + Ed25519-signed Merkle batches - AEAD envelope encryption (AES-256-GCM, per-row DEKs wrapped by KEK, AAD-bound to credential identity, reserved `algorithm_id` byte for crypto agility) - Hand-rolled live TUI (no external TUI framework — stdlib ANSI escapes only) with event-driven repaints coalesced to a tick interval -- Bidirectional Telegram bot — viewer tier (`/status`, `/queue`, `/history`) + operator tier (`/rotate`, `/snooze`) +- Bidirectional Telegram bot — viewer tier (`/status`, `/queue`, `/history`, `/alerts`) + operator tier (`/rotate`) - Compliance evidence export bundle (signed ZIP with audit log, Merkle batches, control mapping for SOC 2 / PCI-DSS / ISO 27001 / HIPAA) ## Quick Start @@ -63,14 +63,25 @@ just demo-full-down # tear down the stack ### Daemon usage +`cre run` and `cre watch` require two 32-byte secrets — the seed key for the audit-log HMAC ratchet and the KEK that wraps per-row data keys. Generate fresh values once and store them somewhere durable (KMS, password manager, sealed env file): + ```bash -cre run --db=sqlite:cre.db # headless daemon -cre watch --db=sqlite:cre.db # daemon + live TUI -cre check --db=sqlite:cre.db --output=json # one-shot CI gate -cre rotate # manual rotation -cre policy list # inspect compiled policies -cre audit verify # check hash chain integrity -cre export --framework=soc2 --out=evidence.zip # signed compliance bundle +export CRE_HMAC_KEY_HEX=$(openssl rand -hex 32) +export CRE_KEK_HEX=$(openssl rand -hex 32) +export CRE_SIGNING_KEY_HEX=$(openssl rand -hex 32) # optional: enables Merkle-batch sealing +``` + +Then: + +```bash +cre run --db=sqlite:cre.db # headless daemon +cre watch --db=sqlite:cre.db # daemon + live TUI +cre check --db=sqlite:cre.db --output=json # one-shot CI gate (no key required) +cre rotate # manual rotation (uses same env-driven rotators as run) +cre policy list # inspect compiled policies +cre audit verify # hash chain + HMAC ratchet (+ Merkle if --public-key given) +cre export --framework=soc2 --out=evidence.zip # signed compliance bundle +cre verify-bundle evidence.zip # offline re-verify a bundle ``` `cre check` exits 1 when any credential violates its policy — drop into any CI pipeline. @@ -131,7 +142,7 @@ All long-lived components are fibers in one OS process. The bus is in-process (C **Direct LibCrypto FFI** for AES-256-GCM AEAD (Crystal stdlib `OpenSSL::Cipher` lacks GCM auth_data/auth_tag) and Ed25519 signing (stdlib lacks high-level wrapper). Bindings live in `src/cre/crypto/aead.cr` and `src/cre/audit/signing.cr`. -**Testing:** stdlib `Spec` runner, 159+ unit tests + integration tests against real PostgreSQL via Docker. +**Testing:** stdlib `Spec` runner, 179+ unit tests + integration tests against real PostgreSQL via Docker. ## Configuration diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/learn/00-OVERVIEW.md b/PROJECTS/intermediate/credential-rotation-enforcer/learn/00-OVERVIEW.md index 6d0501b5..92b5d363 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/learn/00-OVERVIEW.md +++ b/PROJECTS/intermediate/credential-rotation-enforcer/learn/00-OVERVIEW.md @@ -11,7 +11,7 @@ A Crystal daemon that **tracks** credentials, **enforces** rotation policies as | Concept | What you'll see in the code | |---|---| -| Compile-time-checked policy DSL | `policies/*.cr` evaluated by the Crystal compiler; typo'd action symbols, missing fields, or bad credential property references all fail `crystal build` | +| Statically-validated policy DSL | `policies/*.cr` evaluated by the Crystal compiler; single-symbol enum args (`enforce :rotate_immediately`) and `match {}` block typos fail `crystal build`. Splat-symbol args (`notify_via :telegram, :slack`) and missing required fields raise `BuilderError` at policy registration time. Either way, a misformed policy never reaches a running daemon. | | Bus + plugin architecture | Typed events fan out across Crystal channels; subscribers (audit, TUI, Telegram, log) react independently; rotators register at compile time via `register_as :kind` macro | | 4-step rotation contract | `generate -> apply -> verify -> commit`, dual-version safe (AWSCURRENT / AWSPENDING analog), with rollback on failure between apply and commit | | Tamper-evident audit log | SHA-256 hash chain + ratcheting HMAC-SHA256 + Ed25519-signed Merkle batches (3 independent layers of integrity) | @@ -58,21 +58,31 @@ Setup time: ~2 minutes (mostly image pulls). ### Tier 3 - Real Cloud -Copy `config/demo-full.cr.example` and edit env vars to point at your real AWS account / Vault server / GitHub. Run `cre run` headless or `cre watch` for the live TUI. +`cre run` and `cre watch` refuse to start without two 32-byte secrets: + +``` +export CRE_HMAC_KEY_HEX=$(openssl rand -hex 32) # audit log seed key +export CRE_KEK_HEX=$(openssl rand -hex 32) # envelope KEK +export CRE_SIGNING_KEY_HEX=$(openssl rand -hex 32) # optional: enables Merkle-batch sealing +``` + +Then copy `config/demo-full.cr.example`, set the AWS / Vault / GitHub env vars it documents, and run `cre run` headless or `cre watch` for the live TUI. Without `CRE_SIGNING_KEY_HEX`, the daemon still runs but skips Layer 3 of the audit log — `cre audit verify` will skip the Merkle layer too. ## Subcommand Cheat Sheet | Command | Purpose | |---|---| -| `cre run` | Headless daemon (production / systemd) | -| `cre watch` | Engine + live TUI in one process | +| `cre run` | Headless daemon (production / systemd) — requires `CRE_HMAC_KEY_HEX` + `CRE_KEK_HEX` | +| `cre watch` | Engine + live TUI in one process — same env requirements | | `cre check` | One-shot policy eval, exit code by violations (CI-friendly) | -| `cre rotate ` | Manual rotation of a single credential | +| `cre rotate ` | Manual rotation of a single credential — uses the same env-driven rotators as `cre run` | | `cre policy list` | List compiled-in policies | | `cre policy show ` | Inspect one policy in detail | | `cre export --framework=soc2` | Generate signed compliance evidence ZIP | -| `cre audit verify` | Verify hash chain + HMAC + Merkle batch signatures | +| `cre audit verify` | Hash chain + HMAC ratchet (Merkle layer adds when `--public-key=PATH` or `CRE_AUDIT_PUBLIC_KEY_HEX`) | +| `cre verify-bundle ` | Offline re-verify of an evidence bundle (sha256 + manifest sig + chain + Merkle) | | `cre demo` | Tier 1 zero-deps demo | +| `cre tui-demo` | 8-second TUI preview using synthetic events (no daemon, no DB) | | `cre version` | Print version | ## Where to Read Next diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/learn/01-CONCEPTS.md b/PROJECTS/intermediate/credential-rotation-enforcer/learn/01-CONCEPTS.md index ca24b177..132381dc 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/learn/01-CONCEPTS.md +++ b/PROJECTS/intermediate/credential-rotation-enforcer/learn/01-CONCEPTS.md @@ -80,12 +80,15 @@ Step 4 is the only **irreversible** step. By design. If step 4 fails partially ( Verification is exposed as a CLI command: ``` -$ cre audit verify -✓ chain valid: 14,892 entries -✓ hmac ratchet: 14 generations traversed; all valid -✓ merkle batches: 24 sealed, all signatures verify against pubkey v1 +$ cre audit verify --public-key=/etc/cre/audit_pubkey.hex + ✓ hash chain: OK + ✓ HMAC ratchet: OK + ✓ Merkle batches: OK +✓ audit chain valid: 14892 entries ``` +The HMAC ratchet replays from the seed key supplied via `CRE_HMAC_KEY_HEX`, so the verifier needs the *same* seed the writer used. Merkle batches are signed under `CRE_SIGNING_KEY_HEX` and verified with the matching public key — bundled in the `cre export` ZIP so an offline auditor can `cre verify-bundle` without DB access. + ## Compliance Framework Coverage The export bundle (`cre export --framework=soc2`) maps audit events to specific framework controls. Per `src/cre/compliance/control_mapping.cr`: diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/learn/02-ARCHITECTURE.md b/PROJECTS/intermediate/credential-rotation-enforcer/learn/02-ARCHITECTURE.md index 81f68a01..34363075 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/learn/02-ARCHITECTURE.md +++ b/PROJECTS/intermediate/credential-rotation-enforcer/learn/02-ARCHITECTURE.md @@ -41,12 +41,15 @@ Fanout dispatch via Crystal channels. Each subscriber gets its own bounded chann | Subscriber | Overflow | Reason | |---|---|---| | `AuditSubscriber` | `Block` | Never drop audit events; compliance requirement | -| `TuiSubscriber` | `Drop` | Stale UI is fine; can't block engine | -| `MetricsSubscriber` | `Drop` | Best-effort metrics | -| `TelegramSubscriber` | `Drop` (large buffer) | Network-flaky anyway | -| `RotationOrchestrator` | `Block` | Must process scheduled rotations | +| `Tui` | `Drop` | Stale UI is fine; can't block engine | +| `LogNotifier` | `Drop` | Best-effort structured logs | +| `TelegramSubscriber` | `Drop` (buffer 128) | Network-flaky anyway | +| `RotationWorker` | `Block` (buffer 32) | Must dispatch scheduled rotations | -The dispatcher is a single fiber reading from the inbox channel and writing to all subscriber channels in order. A slow subscriber configured `Block` causes the dispatcher to block on that subscriber's `send` - which is exactly what you want for audit (better to backpressure than to lose). +The dispatcher is a single fiber reading from the inbox channel and writing to subscriber channels. Both overflow modes use `select` so a stuck subscriber can't pin the bus indefinitely: + +- `Drop`: non-blocking `select … else …` — full buffer logs a warn and drops the event. +- `Block`: `select … when timeout(@block_send_timeout) …` — if a subscriber's buffer stays full past the timeout (default 5s), the bus drops that one event, logs the stall, and moves on. Operators tune the timeout up for slow downstreams they trust (audit DB writes) and down for unreliable ones. ### Rotators (`src/cre/rotators/`) @@ -66,10 +69,10 @@ Rotators receive their cloud client through their constructor (DI). The CLI `run ### Persistence (`src/cre/persistence/`) Two adapters behind one interface: -- `Sqlite::SqlitePersistence` - WAL mode, single connection (avoids `:memory:` per-connection split), application-level mutex for advisory lock simulation. Used for Tier 1 demo. -- `Postgres::PostgresPersistence` - JSONB tags, BIGSERIAL audit, append-only triggers refusing UPDATE/DELETE/TRUNCATE on `audit_events`, `pg_advisory_xact_lock` for cross-process row locking. Used for Tier 2/3. +- `Sqlite::SqlitePersistence` - single connection (`max_pool_size=1`) so SQLite's writer-serialization is safe; `synchronous=NORMAL`; `BEFORE UPDATE` and `BEFORE DELETE` triggers on `audit_events` raise `audit_events is append-only`. Application-level mutex for advisory-lock simulation (the abstraction is in-process only on SQLite). Used for Tier 1 demo. +- `Postgres::PostgresPersistence` - JSONB tags, `BIGSERIAL` audit seq, `audit_events_no_update` trigger refusing UPDATE/DELETE/TRUNCATE, `pg_advisory_xact_lock` for cross-process locking, and a partial unique index on `rotations(credential_id) WHERE state NOT IN ('completed','failed','aborted','inconsistent')` so two daemons can never insert overlapping in-flight rotations for the same credential. Used for Tier 2/3. -Same repo contracts (`CredentialsRepo`, `VersionsRepo`, `RotationsRepo`, `AuditRepo`) for both. The `Persistence` superclass exposes `transaction(&)` and `with_advisory_lock(key, &)` so the rest of the system is backend-agnostic. +Both backends share the same migration runner (`schema_migrations` table + version-tracked `Step` records), so adding a column is a one-line `Step.new(N, ["ALTER TABLE ..."])` instead of editing a soup of `IF NOT EXISTS`. The repo contracts (`CredentialsRepo`, `VersionsRepo`, `RotationsRepo`, `AuditRepo`) are identical between adapters; `Persistence` exposes `transaction(&)` and `with_advisory_lock(key, &)` so the rest of the system stays backend-agnostic. ### Crypto layers (`src/cre/crypto/`, `src/cre/audit/`) @@ -150,25 +153,26 @@ The PG triggers are not strictly necessary (the chain catches tampering anyway), | Scope | Bound | Mechanism | |---|---|---| -| Per-credential | 1 active rotation | PG advisory lock keyed on `credential_id` (or per-process Mutex on SQLite) | -| Per-rotator-kind | configurable | `Channel(Nil).new(capacity: N)` semaphore | -| Global | 20 (default) | Global rotation worker pool | +| Per-credential | 1 active rotation | `RotationWorker` checks `rotations.in_flight` before dispatching; PG also enforces a partial unique index so cross-process duplicates fail at the DB | +| Per-rotation lifecycle | 1 step at a time | Orchestrator runs `generate -> apply -> verify -> commit` sequentially; `rollback_apply` fires on apply/verify failure; commit failure marks the rotation `inconsistent` and raises a critical alert | +| Engine event bus | per-subscriber buffer + timeout | `EventBus#dispatch` uses `select` for both Block and Drop overflow (see Bus subscribers table above) | Crystal fibers + bounded channels = clean rate limiting without threads or locks. ## Lifecycle (cre run) ``` -1. Load config (env + flags) -2. Open persistence (PG or SQLite); migrate! -3. Initialize crypto (load KEK from env or KMS) -4. Load + validate compiled-in policies (REGISTRY) +1. Bootstrap: validate CRE_HMAC_KEY_HEX + CRE_KEK_HEX (hard-fail if missing) +2. Open persistence (PG or SQLite); migrate! (versioned Step list) +3. Build Envelope from KEK; build optional Ed25519Signer from CRE_SIGNING_KEY_HEX +4. Load compiled-in policies (REGISTRY) + register rotators from env vars 5. Start EventBus.run (dispatcher fiber) -6. Start subscribers: audit, log, telegram, metrics -7. Start Scheduler (publishes SchedulerTick on tick) -8. Start PolicyEvaluator (subscribes to ticks + credential events) -9. Optionally start TUI (cre watch) -10. Block on signal: SIGTERM/SIGINT triggers graceful drain +6. Start subscribers: AuditSubscriber, LogNotifier, RotationWorker, PolicyEvaluator +7. Start Scheduler (SchedulerTick every CRE_TICK_SECONDS) +8. Start BatchSealerScheduler if signer present (default every 5min) +9. Wire Telegram bot if TELEGRAM_TOKEN + chat IDs are set +10. Optionally start TUI + Snapshotter (cre watch) +11. Block on stop_signal channel; SIGINT triggers graceful drain ``` -Graceful shutdown: `engine.stop` publishes `ShutdownRequested`, gives subscribers ~50ms to flush, then closes the bus inbox and joins each subscriber fiber. +Graceful shutdown: `engine.stop` publishes `ShutdownRequested` and waits up to 2s on `audit_subscriber.await_drain` — the audit subscriber sends on a private completion channel as soon as it processes that event, which proves it has handled everything queued before it. Then the bus closes its inbox and subscriber channels; each subscriber fiber exits on `Channel::ClosedError`. diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/learn/03-IMPLEMENTATION.md b/PROJECTS/intermediate/credential-rotation-enforcer/learn/03-IMPLEMENTATION.md index 31d8c9a5..e12a1d09 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/learn/03-IMPLEMENTATION.md +++ b/PROJECTS/intermediate/credential-rotation-enforcer/learn/03-IMPLEMENTATION.md @@ -7,27 +7,36 @@ This document points you at the most important code paths. Read it with `tree src/` open in another window. -## The Policy DSL (compile-time validation) +## The Policy DSL (statically- and registration-time validated) -`src/cre/policy/dsl.cr` defines a top-level `policy` method that uses Crystal's `with builder yield` so every Builder method is callable receiver-less inside the block: +`src/cre/policy/dsl.cr` declares the DSL inside `module CRE::Policy::Dsl`. Consumers opt in explicitly: ```crystal -def policy(name : String, &block) - builder = CRE::Policy::Builder.new(name) - with builder yield - CRE::Policy::REGISTRY << builder.build +require "cre/policy/dsl" +include CRE::Policy::Dsl + +policy "production-aws-secrets" do + match { |c| c.kind.aws_secretsmgr? && c.tag(:env) == "prod" } + max_age 30.days + enforce :rotate_immediately + notify_via :telegram, :structured_log end ``` -The Builder methods (in `src/cre/policy/builder.cr`) take typed enum parameters - so `enforce :rotate_immediately` autocasts the symbol to `Action::RotateImmediately` at compile time. Typo'd `:rotate_immediatly` fails the build with `expected Action, got :rotate_immediatly`. The `match {}` block is a real `Proc(Credential, Bool)` so `c.kund` (typo) breaks compilation pointing at the policy file. +`with builder yield` makes every Builder method callable receiver-less inside the block. Two flavors of typo-detection apply: -`Builder#build` validates required fields (`matcher`, `max_age`, `enforce_action`) and raises `BuilderError` if any are missing. This makes a policy literally unable to ship to production in a misformed state. +- **Compile time** — `enforce :rotate_immediatly` (single Symbol arg → `Action` enum) is rejected by Crystal's autocast. `match { |c| c.kund }` (typo on a Credential getter) breaks compilation pointing at the policy file. +- **Registration time** — `notify_via :telegrm, :slak` uses a splat-Symbol overload that runs `Channel.parse?` on each value and raises `BuilderError("unknown channel 'telegrm' in policy ''")` when the file is loaded. Splat autocast doesn't reach into Symbols, so this layer is the next-best thing. + +`Builder#build` also raises `BuilderError` on missing required fields (`matcher`, `max_age`, `enforce_action`). Either way, a misformed policy never reaches a running daemon. ## The Event Bus (fanout via Crystal channels) -`src/cre/engine/event_bus.cr` exposes `subscribe(buffer:, overflow:)` returning a `Channel(Event)`. The `run` method spawns a single dispatcher fiber that reads from `@inbox` and forwards to each subscriber's channel. Per-subscriber overflow policy (`Block` or `Drop`) drives whether a slow consumer pauses the dispatcher or quietly loses events. +`src/cre/engine/event_bus.cr` exposes `subscribe(buffer:, overflow:)` returning a `Channel(Event)`. The `run` method spawns a single dispatcher fiber that reads from `@inbox` and forwards to each subscriber's channel. -The `dispatch` method uses Crystal's `select` to attempt a non-blocking send for `Drop` subscribers and logs a warning when full. `Block` subscribers get `send` directly. +`dispatch` uses Crystal's `select` for both overflow modes: +- `Drop` — `select … else` drops the event when the buffer is full and logs a warn. +- `Block` — `select … when timeout(@block_send_timeout)` waits up to `@block_send_timeout` (default 5s) and only then drops, logging the stall. This isolates the bus from a stuck subscriber: head-of-line blocking is bounded. ## The Rotator Plugin Registration @@ -50,15 +59,27 @@ When a file like `src/cre/rotators/aws_secrets.cr` is required, the `register_as `src/cre/engine/rotation_orchestrator.cr` runs the contract: ``` -generate -> persist pending version +generate -> rotator-specific (often produces the new value + cloud version_id) apply -> rotator-specific (often no-op for cloud rotators where generate already exposed) verify -> read back, byte-equal check commit -> promote new -> AWSCURRENT, demote old -> AWSPREVIOUS ``` -Each step publishes `RotationStepStarted` and either `RotationStepCompleted` or `RotationStepFailed` to the bus. On any exception during apply/verify, `rollback_apply` is invoked and `RotationFailed` is published. `RotationCompleted` is the success terminal. +Each step publishes `RotationStepStarted` and either `RotationStepCompleted` or `RotationStepFailed` to the bus. -The orchestrator never directly calls audit. Audit happens automatically because `AuditSubscriber` is on the bus listening for these exact event types - the orchestrator can't forget to log. +Failure handling has two regimes: + +- **apply or verify fails** — `rotator.rollback_apply(c, new_secret)` reverses the cloud-side mutation; rotation moves to `Failed`; bus emits `RotationStepFailed` + `RotationFailed`. +- **commit fails** — partial cross-call commit sequences (e.g., AWS `UpdateSecretVersionStage` 5xx half-way through) cannot be reliably reversed client-side. The rotation transitions to `Inconsistent` (a terminal state distinct from `Failed`), and the orchestrator emits a critical `AlertRaised` so operators know to intervene. + +Success path is now the heavyweight one: when the four steps complete, the orchestrator +1. seals `new_secret.ciphertext` with the optional `Crypto::Envelope` (AES-256-GCM, AAD = `cred=|kind=`), +2. inserts a `credential_versions` row with the wrapped DEK + KEK version, +3. updates the credential row to bump `last_rotated_at`, set `current_version_id` to the new version's id, and demote the old one to `previous_version_id`. + +That last step is what stops the policy evaluator from re-scheduling the same rotation on every tick — `Policy#overdue?` keys on `c.rotation_anchor` (which is `last_rotated_at || created_at`), not on `updated_at`. + +The orchestrator never directly calls audit. Audit happens automatically because `AuditSubscriber` is on the bus listening for these exact event types — the orchestrator can't forget to log. And because the orchestrator's path is the only path that runs `versions.insert` + `credentials.update`, persistence-side state stays consistent with the audit-log narrative. ## SigV4 Signer (the AWS-flavored work) @@ -75,20 +96,30 @@ signature = HMAC(signing_key, string_to_sign) The `Authorization` header is built from `algorithm + Credential=... + SignedHeaders=... + Signature=...`. Includes `X-Amz-Security-Token` when an STS session token is supplied. -Tested against AWS canonical examples in `spec/unit/aws/signer_spec.cr` for idempotence and format conformance. +Two test files cover the signer: +- `spec/unit/aws/signer_spec.cr` — idempotence + Authorization-header regex shape. +- `spec/unit/aws/signer_aws_vector_spec.cr` — uses the AWS reference suite's `get-vanilla` inputs (access key, secret, region, service, fixed timestamp) and locks in a regression vector for the exact signature our signer produces. Because we always emit `X-Amz-Content-SHA256` (required by Secrets Manager and other JSON-protocol services), the signed-headers list is `host;x-amz-content-sha256;x-amz-date` — slightly different from AWS's vanilla vector, so we lock in our own bytes rather than match theirs. Any future change to canonicalization, key derivation, or header ordering trips the test. ## Audit Log Integrity (three-layer) -`src/cre/audit/audit_log.cr` orchestrates Layer 1 + 2: +`src/cre/audit/audit_log.cr` writes Layers 1 + 2 on every `append`: - `latest_hash` from the DB (genesis = 32 zero bytes for an empty log) - `content_hash = HashChain.next_hash(prev_hash, canonical_payload)` - `hmac = HmacRatchet#sign(content_hash)`; ratchet rolls every 1024 rows +- All three columns plus `hmac_key_version` get persisted in one `INSERT` per row -`src/cre/audit/batch_sealer.cr` builds Layer 3: +Verification is split into three independently-callable methods: +- `verify_hash_chain` — walks every entry, recomputes `SHA256(prev_hash || payload)`, compares against `content_hash`. +- `verify_hmac_ratchet(seed_key)` — replays the ratchet from the seed `CRE_HMAC_KEY_HEX`, recomputes each row's HMAC against `content_hash`, and checks `hmac_key_version` matches the ratchet's view of where rotation should be. Catches an attacker who fixed up the hash chain but doesn't have the seed. +- `verify_batches(verifier)` — for every row in `audit_batches`, refetch the corresponding `content_hash` leaves, recompute the Merkle root, then verify the Ed25519 signature over `(start_seq, end_seq, root)`. + +`src/cre/audit/batch_sealer.cr` builds Layer 3 entries: - Walk new audit_events since `last_sealed_seq` - Build a Merkle tree (`Merkle.root`) over each row's `content_hash` - Sign `(start_seq, end_seq, root)` with Ed25519 via `Signing::Ed25519Signer` -- Store the signed batch in `audit_batches` +- Insert into `audit_batches` + +`src/cre/audit/batch_sealer_scheduler.cr` is the fiber that actually drives the sealer in `cre run` / `cre watch`: it calls `seal_pending` once on start, again every `CRE_SEAL_INTERVAL_SECONDS` (default 300s), and once more on shutdown. Every successful seal publishes a typed `AuditBatchSealed` event, which the audit subscriber writes back into the audit log under `audit.batch.sealed` — closing the loop with the SOC 2 / PCI-DSS / ISO / HIPAA control mapping that already keys on that event type. Crystal's stdlib OpenSSL doesn't expose Ed25519 high-level wrappers, so `src/cre/audit/signing.cr` reaches into LibCrypto via FFI: `EVP_PKEY_new_raw_private_key`, `EVP_DigestSign`, etc. Public-key verification is symmetrical: `Ed25519Verifier#verify(message, signature)`. @@ -108,14 +139,20 @@ Decrypting requires the KEK to unwrap the DEK, then the DEK + AAD to decrypt the ## Telegram Bot -`src/cre/notifiers/telegram.cr` is a thin HTTP::Client wrapper for the Telegram Bot API (no tourmaline dependency for the notification path). +`src/cre/notifiers/telegram.cr` is a thin HTTP::Client wrapper for the Telegram Bot API (no tourmaline dependency for the notification path). Errors get the bot token redacted before they hit logs — Telegram requires the token in the URL path, so the redaction is best-effort, but it stops the obvious leak. -`src/cre/notifiers/telegram_bot.cr` does long-poll `getUpdates` and dispatches commands. Auth is by chat-ID allowlist; viewer tier vs operator tier separates `/status` from `/rotate`. `/rotate` publishes `RotationScheduled` to the bus, where the orchestrator picks it up. +`src/cre/notifiers/telegram_bot.cr` does long-poll `getUpdates` and dispatches commands. Auth is by chat-ID allowlist; viewer tier (`/status`, `/queue`, `/history`, `/alerts`) is read-only; operator tier adds `/rotate`. `/rotate ` publishes `RotationScheduled` to the bus, which the `RotationWorker` consumes (see `src/cre/engine/rotation_worker.cr`) — the worker resolves the credential, looks up the right `Rotator` from the env-driven dispatch table, checks `rotations.in_flight` to dedupe, and hands off to `RotationOrchestrator`. ## Persistence Layer Shape -`src/cre/persistence/repos.cr` declares the abstract repos (`CredentialsRepo`, `VersionsRepo`, `RotationsRepo`, `AuditRepo`) and the record types (`RotationRecord`, `AuditEntry`, `AuditBatch`, plus the `RotatorKind` and `RotationState` enums). +`src/cre/persistence/repos.cr` declares the abstract repos (`CredentialsRepo`, `VersionsRepo`, `RotationsRepo`, `AuditRepo`) and the record types (`RotationRecord`, `AuditEntry`, `AuditBatch`, plus the `RotatorKind` and `RotationState` enums). `RotationState::Inconsistent` is included in `TERMINAL_STATES` alongside `Completed` / `Failed` / `Aborted`. -`src/cre/persistence/sqlite/` and `src/cre/persistence/postgres/` mirror each other under the same interface. PG uses `$1, $2` placeholders, BYTEA + JSONB native types, BIGSERIAL audit; SQLite uses `?` placeholders, BLOB + TEXT (with JSON helpers). +Both adapters apply schema changes through `Migrations::Step` records keyed on a monotonic `version`. The `schema_migrations` table tracks which versions have run; new alterations land as new `Step.new(N, ["ALTER TABLE ..."])` entries instead of editing the soup of `IF NOT EXISTS` statements. -`audit_events` is the most carefully guarded table in the schema - PG triggers refuse `UPDATE`, `DELETE`, `TRUNCATE` and the application role doesn't have those grants either. Two independent locks; both must be subverted to forge history. +`audit_events` is the most carefully guarded table in the schema: +- Postgres has the original `audit_events_no_update` trigger (raises `audit_events is append-only` on UPDATE/DELETE/TRUNCATE). +- SQLite gets parity via two `BEFORE UPDATE` / `BEFORE DELETE` triggers using `RAISE(ABORT, '...')`. +- The repo's `INSERT` no longer uses `OR IGNORE` / `ON CONFLICT … DO NOTHING`, so a constraint failure raises into the application instead of silently dropping. +- The audit subscriber's rescue path publishes a critical `AlertRaised` and (by default) panics the process via `CRE_AUDIT_FAILURE_MODE=panic`. + +Two independent layers; both must be subverted to forge history. diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/learn/04-CHALLENGES.md b/PROJECTS/intermediate/credential-rotation-enforcer/learn/04-CHALLENGES.md index ad62acc6..845605e4 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/learn/04-CHALLENGES.md +++ b/PROJECTS/intermediate/credential-rotation-enforcer/learn/04-CHALLENGES.md @@ -35,10 +35,10 @@ Anchor each `audit_batches` Merkle root to the Bitcoin blockchain via OpenTimest Add a rotator that *replaces* static credentials with SPIFFE SVIDs (X.509 + JWT). Demonstrates the post-2024 industry shift away from rotation entirely toward attestation-based ephemeral identity. Touch points: new client in `src/cre/spiffe/`, new rotator in `src/cre/rotators/spiffe.cr`, new credential kind in `src/cre/domain/credential.cr`. ### 8. Crash recovery state machine -Implement the recovery protocol described in the spec: on boot, scan `rotations` table for non-terminal states; for each, decide whether to rollback, retry, or mark `inconsistent`. The current orchestrator publishes events but doesn't recover from a daemon crash mid-rotation. Add `src/cre/engine/recovery.cr` with explicit state-machine semantics for each `(rotator_kind, last_step)` pair. +The orchestrator already marks commit-step failures as `Inconsistent` and emits a critical alert, so single-step partial failures surface loudly. What's still missing is the *boot-time* recovery sweep: if the daemon was killed between, say, a successful `apply` and the start of `verify`, the `rotations` row is left in `Verifying` state with no fiber driving it forward. Implement the recovery protocol: on boot, scan `rotations` for non-terminal states; for each, decide based on `(rotator_kind, last_step)` whether to invoke `rollback_apply`, retry from the failed step, or transition to `Inconsistent`. Add `src/cre/engine/recovery.cr` with explicit state-machine semantics, wire it into `Engine#start`, and add a SQL fixture spec in `spec/unit/engine/recovery_spec.cr` that builds each kind of "stale" row and asserts the right outcome. ### 9. Multi-tenant support -Wire `tenant_id` through the schema (already reserved as a column placeholder), the AAD construction, the policy matchers (allow `c.tenant == "tenant-x"`), and the API surface. Postgres row-level security policies enforce isolation at the DB level. The biggest design decision: per-tenant KEKs vs shared KEK with per-tenant DEKs. +Wire `tenant_id` end-to-end: a new migration adds the column on `credentials` / `credential_versions` / `rotations` / `audit_events`; the AAD construction in `RotationOrchestrator#persist_credential_version` extends to `cred=|kind=|tenant=`; policy matchers gain `c.tenant == "tenant-x"`; and Postgres row-level security policies enforce isolation at the DB level. The biggest design decision: per-tenant KEKs (each tenant rotates independently, but bootstrap juggles N keys) vs shared KEK with tenant-bound DEKs (simpler config, the AAD does the isolation work). ### 10. JIT credential broker Replace the rotation contract entirely for some credential types: instead of rotating, *issue* a fresh ephemeral credential on each access (5-15 minute TTL). Requires a new `Broker` abstraction alongside `Rotator`, integration with AWS STS / GCP service account impersonation / Vault dynamic, and consumer-side token refresh logic in the example apps. diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/audit/audit_log_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/audit/audit_log_spec.cr index 690c08e9..ea09a298 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/audit/audit_log_spec.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/audit/audit_log_spec.cr @@ -30,6 +30,10 @@ describe CRE::Audit::AuditLog do log.append("a", "s", nil, {"k" => "v"}) log.append("b", "s", nil, {"k" => "v2"}) + # The append-only trigger blocks tampering in the live DB; drop it + # for this test so we can exercise the verify_chain code path against + # a forced inconsistency. + persist.db.exec("DROP TRIGGER audit_events_no_update") persist.db.exec("UPDATE audit_events SET payload = ? WHERE seq = 2", %({"event_type":"b","actor":"s","target_id":null,"payload":{"k":"BAD"}})) log.verify_chain.should be_false @@ -37,6 +41,23 @@ describe CRE::Audit::AuditLog do persist.try(&.close) end + it "append-only triggers exist on audit_events (SQLite)" do + # The crystal-sqlite3 driver caches prepared statements at the connection + # level; a failed UPDATE leaves the cached statement in error state and the + # error re-surfaces at connection close, which conflates 'expect_raises' + # bookkeeping. We verify the trigger by inspecting sqlite_master directly. + persist = CRE::Persistence::Sqlite::SqlitePersistence.new(":memory:") + persist.migrate! + triggers = persist.db.query_all( + "SELECT name FROM sqlite_master WHERE type='trigger' AND tbl_name='audit_events'", + as: String, + ) + triggers.should contain "audit_events_no_update" + triggers.should contain "audit_events_no_delete" + ensure + persist.try(&.close) + end + it "verify_chain returns true on empty log" do persist = CRE::Persistence::Sqlite::SqlitePersistence.new(":memory:") persist.migrate! diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/aws/signer_aws_vector_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/aws/signer_aws_vector_spec.cr new file mode 100644 index 00000000..ab2bf137 --- /dev/null +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/aws/signer_aws_vector_spec.cr @@ -0,0 +1,123 @@ +# =================== +# ©AngelaMos | 2026 +# signer_aws_vector_spec.cr +# =================== + +require "../../spec_helper" +require "../../../src/cre/aws/signer" + +# AWS publishes a reference SigV4 test suite at +# https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html with +# golden values for canonical request, string-to-sign, and signature. +# +# Our signer always emits X-Amz-Content-SHA256 (required by Secrets Manager +# and all the JSON-protocol services we target), which AWS's "vanilla" +# reference vectors deliberately omit. So instead of matching the vanilla +# vector byte-for-byte, we lock in: +# 1. The exact AWS-spec credential-scope and signed-headers list, +# 2. A regression-stable signature for a known input set with our +# always-on X-Amz-Content-SHA256 header. +# Any change to canonicalization, key derivation, or header ordering +# breaks the regression vector — which is the failure mode we care about. +# +# Inputs match the published reference suite for everything except the +# extra signed header: +# access_key: AKIDEXAMPLE +# secret_key: wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY +# region: us-east-1 +# service: service +# date: 20150830T123600Z (UTC) +describe CRE::Aws::SigV4 do + it "produces the AWS-spec credential-scope and signed-headers list" do + signer = CRE::Aws::SigV4.new( + access_key_id: "AKIDEXAMPLE", + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region: "us-east-1", + service: "service", + ) + + headers = HTTP::Headers.new + uri = URI.parse("https://example.amazonaws.com/") + fixed_time = Time.utc(2015, 8, 30, 12, 36, 0) + + signer.sign("GET", uri, headers, "", fixed_time) + + headers["X-Amz-Date"].should eq "20150830T123600Z" + headers["Host"].should eq "example.amazonaws.com" + + auth = headers["Authorization"] + auth.should start_with "AWS4-HMAC-SHA256 " + auth.should contain "Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request" + auth.should contain "SignedHeaders=host;x-amz-content-sha256;x-amz-date" + auth.should match(/Signature=[a-f0-9]{64}\z/) + end + + it "regression vector: locks in the bytewise signature for a fixed input" do + signer = CRE::Aws::SigV4.new( + access_key_id: "AKIDEXAMPLE", + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region: "us-east-1", + service: "service", + ) + headers = HTTP::Headers.new + uri = URI.parse("https://example.amazonaws.com/") + fixed_time = Time.utc(2015, 8, 30, 12, 36, 0) + + signer.sign("GET", uri, headers, "", fixed_time) + + # If any of canonicalization / key derivation / signed-headers + # ordering / content-sha256 logic changes, this assertion catches it. + expected = "726c5c4879a6b4ccbbd3b24edbd6b8826d34f87450fbbf4e85546fc7ba9c1642" + headers["Authorization"].should contain "Signature=#{expected}" + end + + it "matches a POST with body — content-sha256 changes the signature" do + signer = CRE::Aws::SigV4.new( + access_key_id: "AKIDEXAMPLE", + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region: "us-east-1", + service: "service", + ) + + fixed_time = Time.utc(2015, 8, 30, 12, 36, 0) + body = "Param1=value1" + + h_a = HTTP::Headers{"Content-Type" => "application/x-www-form-urlencoded"} + h_b = HTTP::Headers{"Content-Type" => "application/x-www-form-urlencoded"} + + signer.sign("POST", URI.parse("https://example.amazonaws.com/"), h_a, body, fixed_time) + signer.sign("POST", URI.parse("https://example.amazonaws.com/"), h_b, "different-body", fixed_time) + + # Two bodies, two different content-sha256 inputs, two different + # signatures — proves the body actually flows into the signature. + h_a["X-Amz-Content-SHA256"].should_not eq h_b["X-Amz-Content-SHA256"] + h_a["Authorization"].should_not eq h_b["Authorization"] + end + + it "different regions produce different signing keys (and signatures)" do + east = CRE::Aws::SigV4.new("AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "us-east-1", "service") + west = CRE::Aws::SigV4.new("AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "us-west-2", "service") + fixed = Time.utc(2015, 8, 30, 12, 36, 0) + + h_e = HTTP::Headers.new + h_w = HTTP::Headers.new + east.sign("GET", URI.parse("https://example.amazonaws.com/"), h_e, "", fixed) + west.sign("GET", URI.parse("https://example.amazonaws.com/"), h_w, "", fixed) + + h_e["Authorization"].should_not eq h_w["Authorization"] + end + + it "session token participates in the signed-headers list" do + signer = CRE::Aws::SigV4.new( + access_key_id: "AKIDEXAMPLE", + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region: "us-east-1", + service: "service", + session_token: "TOKENVALUE", + ) + h = HTTP::Headers.new + fixed = Time.utc(2015, 8, 30, 12, 36, 0) + signer.sign("GET", URI.parse("https://example.amazonaws.com/"), h, "", fixed) + h["Authorization"].should contain "x-amz-security-token" + end +end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/domain/credential_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/domain/credential_spec.cr index 593f5e57..0cefa391 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/domain/credential_spec.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/domain/credential_spec.cr @@ -39,7 +39,7 @@ describe CRE::Domain::Credential do tags: {} of String => String, ) c.kind.github_pat?.should be_true - c.kind.aws_iam_key?.should be_false + c.kind.env_file?.should be_false end it "tag() accepts both string and symbol keys" do @@ -51,4 +51,31 @@ describe CRE::Domain::Credential do c.tag("foo").should eq "bar" c.tag(:foo).should eq "bar" end + + it "rotation_anchor falls back to created_at when never rotated" do + created = Time.utc - 3.days + c = CRE::Domain::Credential.new( + id: UUID.random, + external_id: "x", + kind: CRE::Domain::CredentialKind::EnvFile, + name: "n", + tags: {} of String => String, + created_at: created, + ) + c.rotation_anchor.should eq created + end + + it "rotation_anchor uses last_rotated_at once set" do + rotated = Time.utc - 1.hour + c = CRE::Domain::Credential.new( + id: UUID.random, + external_id: "x", + kind: CRE::Domain::CredentialKind::EnvFile, + name: "n", + tags: {} of String => String, + created_at: Time.utc - 30.days, + last_rotated_at: rotated, + ) + c.rotation_anchor.should eq rotated + end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_orchestrator_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_orchestrator_spec.cr index 6b98dd1c..6b14d618 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_orchestrator_spec.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_orchestrator_spec.cr @@ -7,6 +7,8 @@ require "../../spec_helper" require "../../../src/cre/engine/rotation_orchestrator" require "../../../src/cre/persistence/sqlite/sqlite_persistence" require "../../../src/cre/rotators/env_file" +require "../../../src/cre/crypto/envelope" +require "../../../src/cre/crypto/kek" private def drain(ch : ::Channel(CRE::Events::Event)) : Array(CRE::Events::Event) out = [] of CRE::Events::Event @@ -91,6 +93,123 @@ describe CRE::Engine::RotationOrchestrator do persist.try(&.close) tmp.try(&.delete) end + + it "bumps credential.last_rotated_at on successful rotation" do + tmp = File.tempfile("cre_rot_anchor_") { |f| f << "K=v\n" } + cred = env_credential(tmp.path, "K") + persist = CRE::Persistence::Sqlite::SqlitePersistence.new(":memory:") + persist.migrate! + persist.credentials.insert(cred) + + bus = CRE::Engine::EventBus.new + bus.run + + floor = Time.utc.at_beginning_of_second + state = CRE::Engine::RotationOrchestrator.new(bus, persist).run(cred, CRE::Rotators::EnvFileRotator.new) + sleep 0.05.seconds + state.completed?.should be_true + + refreshed = persist.credentials.find(cred.id).not_nil! + refreshed.last_rotated_at.not_nil!.should be >= floor + refreshed.rotation_anchor.should be >= floor + ensure + bus.try(&.stop) + persist.try(&.close) + tmp.try(&.delete) + end + + it "writes an encrypted credential_version when an Envelope is configured" do + ENV["TEST_KEK_ROT"] = "0" * 64 + kek = CRE::Crypto::Kek::EnvKek.new("TEST_KEK_ROT", version: 1) + envelope = CRE::Crypto::Envelope.new(kek) + + tmp = File.tempfile("cre_rot_env_") { |f| f << "K=v\n" } + cred = env_credential(tmp.path, "K") + persist = CRE::Persistence::Sqlite::SqlitePersistence.new(":memory:") + persist.migrate! + persist.credentials.insert(cred) + + bus = CRE::Engine::EventBus.new + bus.run + + state = CRE::Engine::RotationOrchestrator.new(bus, persist, envelope).run(cred, CRE::Rotators::EnvFileRotator.new) + sleep 0.05.seconds + state.completed?.should be_true + + versions = persist.versions.for_credential(cred.id) + versions.size.should eq 1 + v = versions.first + v.kek_version.should eq 1 + v.algorithm_id.should eq CRE::Crypto::ALGORITHM_AES_256_GCM + v.ciphertext.size.should be > 0 + + sealed = CRE::Crypto::SealedSecret.new(v.ciphertext, v.dek_wrapped, v.kek_version, v.algorithm_id) + plaintext = envelope.open(sealed, "cred=#{cred.id}|kind=#{cred.kind}".to_slice) + plaintext.size.should be > 0 + + refreshed = persist.credentials.find(cred.id).not_nil! + refreshed.current_version_id.should eq v.id + ensure + ENV.delete("TEST_KEK_ROT") + bus.try(&.stop) + persist.try(&.close) + tmp.try(&.delete) + end + + it "marks rotation Inconsistent when commit step fails" do + tmp = File.tempfile("cre_rot_commit_fail_") { |f| f << "K=v\n" } + cred = env_credential(tmp.path, "K") + persist = CRE::Persistence::Sqlite::SqlitePersistence.new(":memory:") + persist.migrate! + persist.credentials.insert(cred) + + bus = CRE::Engine::EventBus.new + ch = bus.subscribe + bus.run + + state = CRE::Engine::RotationOrchestrator.new(bus, persist).run(cred, CommitFailingRotator.new) + sleep 0.05.seconds + + state.should eq CRE::Persistence::RotationState::Inconsistent + persist.rotations.in_flight.size.should eq 0 + + types = drain(ch).map(&.class.name) + types.should contain "CRE::Events::AlertRaised" + ensure + bus.try(&.stop) + persist.try(&.close) + tmp.try(&.delete) + end +end + +class CommitFailingRotator < CRE::Rotators::Rotator + def kind : Symbol + :env_file + end + + def can_rotate?(c : CRE::Domain::Credential) : Bool + _ = c + true + end + + def generate(c : CRE::Domain::Credential) : CRE::Domain::NewSecret + _ = c + CRE::Domain::NewSecret.new(ciphertext: "x".to_slice) + end + + def apply(c : CRE::Domain::Credential, s : CRE::Domain::NewSecret) : Nil + _ = {c, s} + end + + def verify(c : CRE::Domain::Credential, s : CRE::Domain::NewSecret) : Bool + _ = {c, s} + true + end + + def commit(c : CRE::Domain::Credential, s : CRE::Domain::NewSecret) : Nil + _ = {c, s} + raise CRE::Rotators::RotatorError.new("commit network 503") + end end class FailingRotator < CRE::Rotators::Rotator diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_worker_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_worker_spec.cr new file mode 100644 index 00000000..aa4f5b45 --- /dev/null +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/engine/rotation_worker_spec.cr @@ -0,0 +1,197 @@ +# =================== +# ©AngelaMos | 2026 +# rotation_worker_spec.cr +# =================== + +require "../../spec_helper" +require "../../../src/cre/engine/event_bus" +require "../../../src/cre/engine/rotation_worker" +require "../../../src/cre/engine/rotation_orchestrator" +require "../../../src/cre/persistence/sqlite/sqlite_persistence" +require "../../../src/cre/rotators/env_file" + +private def env_credential(path : String, key : String) : CRE::Domain::Credential + CRE::Domain::Credential.new( + id: UUID.random, + external_id: "#{path}::#{key}", + kind: CRE::Domain::CredentialKind::EnvFile, + name: key, + tags: {"path" => path, "key" => key} of String => String, + ) +end + +private def setup_worker + persist = CRE::Persistence::Sqlite::SqlitePersistence.new(":memory:") + persist.migrate! + bus = CRE::Engine::EventBus.new + bus.run + orchestrator = CRE::Engine::RotationOrchestrator.new(bus, persist) + worker = CRE::Engine::RotationWorker.new(bus, orchestrator, persist) + {persist, bus, worker} +end + +class StubRotator < CRE::Rotators::Rotator + property generate_count = 0 + property apply_count = 0 + property commit_count = 0 + property declined_credentials = [] of UUID + + def initialize(@kind_sym : Symbol = :env_file, @can_rotate : Bool = true) + end + + def kind : Symbol + @kind_sym + end + + def can_rotate?(c : CRE::Domain::Credential) : Bool + @declined_credentials << c.id unless @can_rotate + @can_rotate + end + + def generate(c : CRE::Domain::Credential) : CRE::Domain::NewSecret + _ = c + @generate_count += 1 + CRE::Domain::NewSecret.new(ciphertext: "x".to_slice) + end + + def apply(c : CRE::Domain::Credential, s : CRE::Domain::NewSecret) : Nil + _ = {c, s} + @apply_count += 1 + end + + def verify(c : CRE::Domain::Credential, s : CRE::Domain::NewSecret) : Bool + _ = {c, s} + true + end + + def commit(c : CRE::Domain::Credential, s : CRE::Domain::NewSecret) : Nil + _ = {c, s} + @commit_count += 1 + end +end + +describe CRE::Engine::RotationWorker do + it "dispatches RotationScheduled to the registered rotator" do + persist, bus, worker = setup_worker + persist.credentials.insert(env_credential("/tmp/x.env", "K")) + rotator = StubRotator.new + worker.register(:env_file, rotator) + worker.start + + cred = persist.credentials.all.first + bus.publish CRE::Events::RotationScheduled.new(cred.id, "env_file") + sleep 0.15.seconds + + rotator.generate_count.should eq 1 + ensure + worker.try(&.stop) + bus.try(&.stop) + persist.try(&.close) + end + + it "skips when no rotator is registered for the credential's kind" do + persist, bus, worker = setup_worker + persist.credentials.insert(env_credential("/tmp/y.env", "K")) + worker.start + + cred = persist.credentials.all.first + bus.publish CRE::Events::RotationScheduled.new(cred.id, "env_file") + sleep 0.1.seconds + + persist.rotations.in_flight.size.should eq 0 + ensure + worker.try(&.stop) + bus.try(&.stop) + persist.try(&.close) + end + + it "skips when rotator.can_rotate? returns false" do + persist, bus, worker = setup_worker + persist.credentials.insert(env_credential("/tmp/z.env", "K")) + rotator = StubRotator.new(can_rotate: false) + worker.register(:env_file, rotator) + worker.start + + cred = persist.credentials.all.first + bus.publish CRE::Events::RotationScheduled.new(cred.id, "env_file") + sleep 0.1.seconds + + rotator.generate_count.should eq 0 + rotator.declined_credentials.should contain(cred.id) + ensure + worker.try(&.stop) + bus.try(&.stop) + persist.try(&.close) + end + + it "ignores events that aren't RotationScheduled" do + persist, bus, worker = setup_worker + persist.credentials.insert(env_credential("/tmp/w.env", "K")) + rotator = StubRotator.new + worker.register(:env_file, rotator) + worker.start + + cred = persist.credentials.all.first + bus.publish CRE::Events::RotationCompleted.new(cred.id, UUID.random) + sleep 0.1.seconds + + rotator.generate_count.should eq 0 + ensure + worker.try(&.stop) + bus.try(&.stop) + persist.try(&.close) + end + + it "deduplicates: a duplicate schedule while one is in_flight is dropped" do + persist, bus, worker = setup_worker + persist.credentials.insert(env_credential("/tmp/dedup.env", "K")) + cred = persist.credentials.all.first + + record = CRE::Persistence::RotationRecord.new( + id: UUID.random, + credential_id: cred.id, + rotator_kind: CRE::Persistence::RotatorKind::EnvFile, + state: CRE::Persistence::RotationState::Generating, + started_at: Time.utc, + completed_at: nil, + failure_reason: nil, + ) + persist.rotations.insert(record) + + rotator = StubRotator.new + worker.register(:env_file, rotator) + worker.start + + bus.publish CRE::Events::RotationScheduled.new(cred.id, "env_file") + sleep 0.1.seconds + + rotator.generate_count.should eq 0 # blocked by in_flight check + ensure + worker.try(&.stop) + bus.try(&.stop) + persist.try(&.close) + end + + it "warns and skips when credential is missing from persistence" do + persist, bus, worker = setup_worker + rotator = StubRotator.new + worker.register(:env_file, rotator) + worker.start + + bus.publish CRE::Events::RotationScheduled.new(UUID.random, "env_file") + sleep 0.1.seconds + + rotator.generate_count.should eq 0 + ensure + worker.try(&.stop) + bus.try(&.stop) + persist.try(&.close) + end + + it "rotator_for_kind returns nil for unregistered kinds" do + _, _, worker = setup_worker + worker.register(:env_file, StubRotator.new) + worker.rotator_for_kind(CRE::Domain::CredentialKind::EnvFile).should_not be_nil + worker.rotator_for_kind(CRE::Domain::CredentialKind::AwsSecretsmgr).should be_nil + end +end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/dsl_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/dsl_spec.cr index 1a2c29a1..7164eb35 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/dsl_spec.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/dsl_spec.cr @@ -6,13 +6,15 @@ require "../../spec_helper" require "../../../src/cre/policy/dsl" +include CRE::Policy::Dsl + describe "Policy DSL" do before_each { CRE::Policy.clear_registry! } it "registers a policy with full DSL syntax" do - policy "production-databases" do - description "Prod DB rotation" - match { |c| c.kind.database? && c.tag(:env) == "prod" } + policy "production-aws-secrets" do + description "Prod AWS secret rotation" + match { |c| c.kind.aws_secretsmgr? && c.tag(:env) == "prod" } max_age 30.days warn_at 25.days enforce :rotate_immediately @@ -22,8 +24,8 @@ describe "Policy DSL" do CRE::Policy.registry.size.should eq 1 p = CRE::Policy.registry.first - p.name.should eq "production-databases" - p.description.should eq "Prod DB rotation" + p.name.should eq "production-aws-secrets" + p.description.should eq "Prod AWS secret rotation" p.max_age.should eq 30.days p.warn_at.should eq 25.days p.enforce_action.should eq CRE::Policy::Action::RotateImmediately diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/evaluator_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/evaluator_spec.cr index 29f291bc..f76ba920 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/evaluator_spec.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/evaluator_spec.cr @@ -8,6 +8,8 @@ require "../../../src/cre/policy/evaluator" require "../../../src/cre/policy/dsl" require "../../../src/cre/persistence/sqlite/sqlite_persistence" +include CRE::Policy::Dsl + private def fresh_persistence persist = CRE::Persistence::Sqlite::SqlitePersistence.new(":memory:") persist.migrate! diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/policy_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/policy_spec.cr index 598f8f97..a4fbba3f 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/policy_spec.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/policy/policy_spec.cr @@ -32,7 +32,7 @@ describe CRE::Policy::Policy do p.matches?(other).should be_false end - it "detects overdue based on updated_at + max_age" do + it "detects overdue based on rotation_anchor + max_age" do p = CRE::Policy::Policy.new( name: "p", description: nil, matcher: ->(_c : CRE::Domain::Credential) { true }, @@ -46,19 +46,59 @@ describe CRE::Policy::Policy do id: UUID.random, external_id: "f", kind: CRE::Domain::CredentialKind::EnvFile, name: "n", tags: {} of String => String, - updated_at: Time.utc - 1.day, + last_rotated_at: Time.utc - 1.day, ) stale = CRE::Domain::Credential.new( id: UUID.random, external_id: "s", kind: CRE::Domain::CredentialKind::EnvFile, name: "n", tags: {} of String => String, - updated_at: Time.utc - 30.days, + last_rotated_at: Time.utc - 30.days, ) p.overdue?(fresh).should be_false p.overdue?(stale).should be_true end + it "treats never-rotated credentials by created_at" do + p = CRE::Policy::Policy.new( + name: "p", description: nil, + matcher: ->(_c : CRE::Domain::Credential) { true }, + max_age: 7.days, warn_at: nil, + enforce_action: CRE::Policy::Action::NotifyOnly, + notify_channels: [] of CRE::Policy::Channel, + triggers: {} of CRE::Policy::Trigger => CRE::Policy::Action, + ) + + aged = CRE::Domain::Credential.new( + id: UUID.random, external_id: "a", + kind: CRE::Domain::CredentialKind::EnvFile, + name: "n", tags: {} of String => String, + created_at: Time.utc - 30.days, + ) + p.overdue?(aged).should be_true + end + + it "ignores updated_at (renaming a credential does not reset rotation clock)" do + p = CRE::Policy::Policy.new( + name: "p", description: nil, + matcher: ->(_c : CRE::Domain::Credential) { true }, + max_age: 7.days, warn_at: nil, + enforce_action: CRE::Policy::Action::NotifyOnly, + notify_channels: [] of CRE::Policy::Channel, + triggers: {} of CRE::Policy::Trigger => CRE::Policy::Action, + ) + + just_renamed = CRE::Domain::Credential.new( + id: UUID.random, external_id: "x", + kind: CRE::Domain::CredentialKind::EnvFile, + name: "n", tags: {} of String => String, + created_at: Time.utc - 30.days, + updated_at: Time.utc, # tag/name was just edited + last_rotated_at: Time.utc - 30.days, + ) + p.overdue?(just_renamed).should be_true + end + it "computes warning window" do p = CRE::Policy::Policy.new( name: "p", description: nil, @@ -72,17 +112,17 @@ describe CRE::Policy::Policy do young = CRE::Domain::Credential.new( id: UUID.random, external_id: "y", kind: CRE::Domain::CredentialKind::EnvFile, name: "n", tags: {} of String => String, - updated_at: Time.utc - 10.days, + last_rotated_at: Time.utc - 10.days, ) warning = CRE::Domain::Credential.new( id: UUID.random, external_id: "w", kind: CRE::Domain::CredentialKind::EnvFile, name: "n", tags: {} of String => String, - updated_at: Time.utc - 27.days, + last_rotated_at: Time.utc - 27.days, ) overdue = CRE::Domain::Credential.new( id: UUID.random, external_id: "o", kind: CRE::Domain::CredentialKind::EnvFile, name: "n", tags: {} of String => String, - updated_at: Time.utc - 31.days, + last_rotated_at: Time.utc - 31.days, ) p.in_warning_window?(young).should be_false diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/rotators/env_file_spec.cr b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/rotators/env_file_spec.cr index 5b286a98..7c9eaee0 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/rotators/env_file_spec.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/spec/unit/rotators/env_file_spec.cr @@ -32,11 +32,11 @@ describe CRE::Rotators::EnvFileRotator do new_secret.ciphertext.size.should be > 0 rotator.apply(cred, new_secret) - File.exists?("#{path}.pending").should be_true + File.exists?("#{path}.pending.#{Process.pid}").should be_true rotator.verify(cred, new_secret).should be_true rotator.commit(cred, new_secret) - File.exists?("#{path}.pending").should be_false + File.exists?("#{path}.pending.#{Process.pid}").should be_false final = File.read(path) new_value = String.new(new_secret.ciphertext) @@ -56,10 +56,10 @@ describe CRE::Rotators::EnvFileRotator do s = rotator.generate(cred) rotator.apply(cred, s) - File.exists?("#{tmp.path}.pending").should be_true + File.exists?("#{tmp.path}.pending.#{Process.pid}").should be_true rotator.rollback_apply(cred, s) - File.exists?("#{tmp.path}.pending").should be_false + File.exists?("#{tmp.path}.pending.#{Process.pid}").should be_false File.read(tmp.path).should eq "K=v\n" ensure tmp.try(&.delete) diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/audit_log.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/audit_log.cr index bbca72b3..94103d13 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/audit_log.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/audit_log.cr @@ -5,17 +5,39 @@ require "json" require "uuid" +require "openssl/hmac" require "./hash_chain" require "./hmac_ratchet" +require "./merkle" +require "./signing" +require "../crypto/random" require "../persistence/persistence" require "../persistence/repos" module CRE::Audit + # AuditLog is the append-only, tamper-evident write API used by the + # AuditSubscriber. Verification is split across three layers, each + # callable independently: + # + # verify_hash_chain SHA-256 chain over (prev_hash || payload) + # verify_hmac_ratchet HMAC-SHA256 of every content_hash, with + # the ratcheting key replayed from the + # initial seed + # verify_batches Ed25519-signed Merkle-root batches + # + # Together they answer 'has this log been mutated since it was written': + # - hash chain catches edits to any single row (recompute everything), + # - HMAC ratchet catches edits an attacker who recomputed hashes might + # have made (they don't have the seed key), + # - Merkle batches give an external auditor an O(1) commitment to a + # range of entries that they can verify offline with a public key. class AuditLog @ratchet : HmacRatchet @mutex : Mutex + @initial_hmac_key : Bytes def initialize(@persistence : Persistence::Persistence, initial_hmac_key : Bytes, @hmac_version : Int32, @ratchet_every : Int32) + @initial_hmac_key = initial_hmac_key.dup @ratchet = HmacRatchet.new(initial_hmac_key, @hmac_version, @ratchet_every) @mutex = Mutex.new end @@ -45,7 +67,15 @@ module CRE::Audit end end + # Backwards-compatible alias: returns true iff hash chain + HMAC ratchet + # both verify against the seed key the log was constructed with. def verify_chain : Bool + verify_hash_chain && verify_hmac_ratchet(@initial_hmac_key) + end + + # Verify only the SHA-256 chain. Catches tampering when an attacker + # didn't recompute hashes; doesn't catch tampering when they did. + def verify_hash_chain : Bool latest = @persistence.audit.latest_seq return true if latest == 0 entries = @persistence.audit.range(1_i64, latest) @@ -55,6 +85,47 @@ module CRE::Audit HashChain.verify(pairs, payloads) end + # Verify the HMAC ratchet by replaying it from the seed key. An + # attacker who modified rows AND recomputed hashes still doesn't have + # the seed key, so any HMAC mismatch is dispositive evidence of + # tampering. + def verify_hmac_ratchet(seed_key : Bytes) : Bool + raise ArgumentError.new("seed key must be 32 bytes") unless seed_key.size == 32 + latest = @persistence.audit.latest_seq + return true if latest == 0 + + ratchet = HmacRatchet.new(seed_key, version: 1, ratchet_every: @ratchet_every) + entries = @persistence.audit.range(1_i64, latest) + + entries.each do |entry| + return false unless entry.hmac_key_version == ratchet.version + expected = OpenSSL::HMAC.digest(:sha256, ratchet.current_key, entry.content_hash) + return false unless CRE::Crypto::Random.constant_time_equal?(expected, entry.hmac) + ratchet.sign(entry.content_hash) # advance counter; trigger rotation at threshold + end + true + end + + # Verify all sealed Merkle batches against a public key. Each batch + # commits to a Merkle root over content_hashes from start_seq..end_seq; + # we re-derive the root from the live entries and check the signature. + def verify_batches(verifier : Signing::Ed25519Verifier) : Bool + batches = @persistence.audit.all_batches + return true if batches.empty? + + batches.each do |batch| + entries = @persistence.audit.range(batch.start_seq, batch.end_seq) + return false if entries.size != (batch.end_seq - batch.start_seq + 1) + leaves = entries.map(&.content_hash) + recomputed_root = Merkle.root(leaves) + return false unless CRE::Crypto::Random.constant_time_equal?(recomputed_root, batch.merkle_root) + + msg = BatchSealer.pack_message(batch.start_seq, batch.end_seq, batch.merkle_root) + return false unless verifier.verify(msg, batch.signature) + end + true + end + def ratchet_version : Int32 @ratchet.version end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/batch_sealer_scheduler.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/batch_sealer_scheduler.cr new file mode 100644 index 00000000..006d4571 --- /dev/null +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/audit/batch_sealer_scheduler.cr @@ -0,0 +1,64 @@ +# =================== +# ©AngelaMos | 2026 +# batch_sealer_scheduler.cr +# =================== + +require "log" +require "./batch_sealer" +require "../engine/event_bus" +require "../events/credential_events" +require "../events/system_events" + +module CRE::Audit + # BatchSealerScheduler runs an idle fiber that periodically calls + # BatchSealer.seal_pending. Without this fiber the audit_batches table + # never grows and 'cre audit verify --merkle' has nothing to verify. + # + # On each successful seal we publish AlertRaised(:info) with the sealed + # range; the AuditSubscriber then writes a 'audit.batch.sealed' event + # into the audit log itself, which closes the loop for compliance + # frameworks that key on that event_type. + class BatchSealerScheduler + Log = ::Log.for("cre.batch_sealer") + + @running : Bool + + def initialize(@bus : Engine::EventBus, @sealer : BatchSealer, @interval : Time::Span = 5.minutes) + @running = false + end + + def start : Nil + @running = true + spawn(name: "batch-sealer") do + seal_once + while @running + sleep @interval + break unless @running + seal_once + end + end + end + + def stop : Nil + @running = false + seal_once # final seal on shutdown + end + + def seal_once : Nil + batch = @sealer.seal_pending + return if batch.nil? + + @bus.publish Events::AuditBatchSealed.new( + start_seq: batch.start_seq, + end_seq: batch.end_seq, + signing_key_version: batch.signing_key_version, + ) + rescue ex + Log.error(exception: ex) { "batch_sealer.seal_pending failed" } + @bus.publish(Events::AlertRaised.new( + severity: Events::Severity::Critical, + message: "audit batch sealing failed: #{ex.message}", + )) rescue nil + end + end +end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/aws/secrets_client.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/aws/secrets_client.cr index c106ca21..1984516d 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/aws/secrets_client.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/aws/secrets_client.cr @@ -7,6 +7,7 @@ require "http/client" require "json" require "uuid" require "./signer" +require "../http/retry" module CRE::Aws class AwsApiError < Exception @@ -77,7 +78,7 @@ module CRE::Aws } @signer.sign("POST", uri, headers, body) - response = HTTP::Client.post(uri.to_s, headers: headers, body: body) + response = CRE::Http.request("POST", uri.to_s, headers, body, label: "aws.#{action}") raise AwsApiError.new(error_message(response), response.status_code, error_code(response)) unless response.status_code < 300 response.body.empty? ? JSON::Any.new(Hash(String, JSON::Any).new) : JSON.parse(response.body) diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/bootstrap.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/bootstrap.cr new file mode 100644 index 00000000..0bd27491 --- /dev/null +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/bootstrap.cr @@ -0,0 +1,122 @@ +# =================== +# ©AngelaMos | 2026 +# bootstrap.cr +# =================== + +require "../crypto/kek" +require "../crypto/envelope" +require "../audit/signing" +require "../persistence/persistence" +require "../persistence/sqlite/sqlite_persistence" +require "../persistence/postgres/postgres_persistence" +require "../engine/rotation_worker" +require "../rotators/env_file" +require "../rotators/aws_secrets" +require "../rotators/vault_dynamic" +require "../rotators/github_pat" +require "../aws/secrets_client" +require "../vault/client" +require "../github/client" + +module CRE::Cli::Bootstrap + HMAC_KEY_VAR = "CRE_HMAC_KEY_HEX" + KEK_HEX_VAR = "CRE_KEK_HEX" + KEK_VERSION_VAR = "CRE_KEK_VERSION" + SIGNING_KEY_VAR = "CRE_SIGNING_KEY_HEX" + SEAL_INTERVAL_VAR = "CRE_SEAL_INTERVAL_SECONDS" + + class ConfigError < Exception; end + + # Loads the 32-byte HMAC seed key from CRE_HMAC_KEY_HEX. Hard-fails when + # missing; the prior all-zero default left audit logs trivially forgeable + # by anyone with read access to the source. + def self.require_hmac_key : Bytes + hex = ENV[HMAC_KEY_VAR]? + if hex.nil? || hex.empty? + raise ConfigError.new( + "#{HMAC_KEY_VAR} is required for cre to start. Generate one with:\n openssl rand -hex 32\nThen export it before invoking cre.", + ) + end + if hex.size != 64 + raise ConfigError.new("#{HMAC_KEY_VAR} must be 64 hex chars (32 bytes); got #{hex.size}") + end + hex.hexbytes + end + + # Returns an Envelope when CRE_KEK_HEX is set; nil otherwise. nil disables + # at-rest encryption — appropriate for the demo path, but cre run/watch + # should refuse to start without it. + def self.envelope : Crypto::Envelope? + hex = ENV[KEK_HEX_VAR]? + return nil if hex.nil? || hex.empty? + raise ConfigError.new("#{KEK_HEX_VAR} must be 64 hex chars (32 bytes); got #{hex.size}") unless hex.size == 64 + version = (ENV[KEK_VERSION_VAR]? || "1").to_i + kek = Crypto::Kek::EnvKek.new(KEK_HEX_VAR, version) + Crypto::Envelope.new(kek) + end + + def self.require_envelope : Crypto::Envelope + env = envelope + return env unless env.nil? + raise ConfigError.new( + "#{KEK_HEX_VAR} is required for cre run/watch. Generate with:\n openssl rand -hex 32\nKEK rotation: bump #{KEK_VERSION_VAR}.", + ) + end + + # Returns an Ed25519 signer when CRE_SIGNING_KEY_HEX is set, nil otherwise. + # When nil, batch sealing is disabled and 'cre audit verify' will skip the + # Merkle layer. + def self.signer : Audit::Signing::Ed25519Signer? + hex = ENV[SIGNING_KEY_VAR]? + return nil if hex.nil? || hex.empty? + raise ConfigError.new("#{SIGNING_KEY_VAR} must be 64 hex chars (32 bytes); got #{hex.size}") unless hex.size == 64 + Audit::Signing::Ed25519Signer.new(hex.hexbytes, version: 1) + end + + def self.seal_interval : Time::Span + seconds = (ENV[SEAL_INTERVAL_VAR]? || "300").to_i + seconds.seconds + end + + def self.build_persistence(url : String) : Persistence::Persistence + if url.starts_with?("sqlite:") + Persistence::Sqlite::SqlitePersistence.new(url.lchop("sqlite:")) + elsif url.starts_with?("postgres://") || url.starts_with?("postgresql://") + Persistence::Postgres::PostgresPersistence.new(url) + else + raise ConfigError.new("unknown database URL: #{url} (expected sqlite:PATH or postgres://...)") + end + end + + def self.register_rotators(worker : Engine::RotationWorker, io : IO) : Nil + worker.register(:env_file, Rotators::EnvFileRotator.new) + + if (aws_id = ENV["AWS_ACCESS_KEY_ID"]?) && (aws_secret = ENV["AWS_SECRET_ACCESS_KEY"]?) + client = Aws::SecretsManagerClient.new( + access_key_id: aws_id, + secret_access_key: aws_secret, + region: ENV["AWS_REGION"]? || "us-east-1", + endpoint: ENV["AWS_ENDPOINT"]?, + session_token: ENV["AWS_SESSION_TOKEN"]?, + ) + worker.register(:aws_secretsmgr, Rotators::AwsSecretsRotator.new(client)) + end + + if (vault_addr = ENV["VAULT_ADDR"]?) && (vault_token = ENV["VAULT_TOKEN"]?) + client = Vault::Client.new(addr: vault_addr, token: vault_token) + worker.register(:vault_dynamic, Rotators::VaultDynamicRotator.new(client)) + end + + if gh_token = ENV["GITHUB_TOKEN"]? + api = ENV["GITHUB_API_BASE"]? || "https://api.github.com" + client = Github::Client.new(token: gh_token, api_base: api) + worker.register(:github_pat, Rotators::GithubPatRotator.new(client)) + end + rescue ex + io.puts "warning: rotator wiring failed: #{ex.message}" + end + + def self.redact_db_url(url : String) : String + url.gsub(/:\/\/[^:]+:[^@]+@/) { |_| "://****:****@" } + end +end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/cli.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/cli.cr index 3d7b60a7..d2e01b9a 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/cli.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/cli.cr @@ -23,6 +23,7 @@ module CRE::Cli policy show inspect one policy export --framework= generate signed compliance evidence bundle audit verify verify hash chain + HMAC ratchet + Merkle batches + verify-bundle verify a compliance evidence ZIP offline demo tier-1 zero-deps demo (SQLite + .env rotator) tui-demo 8-second TUI preview with synthetic events version print version @@ -44,15 +45,16 @@ module CRE::Cli when "version" io.puts CRE::VERSION 0 - when "run" then Commands::Run.new.execute(argv, io) - when "watch" then Commands::Watch.new.execute(argv, io) - when "check" then Commands::Check.new.execute(argv, io) - when "rotate" then Commands::Rotate.new.execute(argv, io) - when "policy" then Commands::Policy.new.execute(argv, io) - when "export" then Commands::Export.new.execute(argv, io) - when "audit" then Commands::Audit.new.execute(argv, io) - when "demo" then Commands::Demo.new.execute(argv, io) - when "tui-demo" then Commands::TuiDemo.new.execute(argv, io) + when "run" then Commands::Run.new.execute(argv, io) + when "watch" then Commands::Watch.new.execute(argv, io) + when "check" then Commands::Check.new.execute(argv, io) + when "rotate" then Commands::Rotate.new.execute(argv, io) + when "policy" then Commands::Policy.new.execute(argv, io) + when "export" then Commands::Export.new.execute(argv, io) + when "audit" then Commands::Audit.new.execute(argv, io) + when "verify-bundle" then Commands::VerifyBundle.new.execute(argv, io) + when "demo" then Commands::Demo.new.execute(argv, io) + when "tui-demo" then Commands::TuiDemo.new.execute(argv, io) else io.puts "unknown subcommand: #{subcommand}" io.puts USAGE diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands.cr index 1f8d69b4..01946114 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands.cr @@ -10,6 +10,7 @@ require "./commands/rotate" require "./commands/policy" require "./commands/export" require "./commands/audit" +require "./commands/verify_bundle" require "./commands/demo" require "./commands/tui_demo" require "./commands/version" diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/audit.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/audit.cr index e4371243..fd534da9 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/audit.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/audit.cr @@ -4,7 +4,9 @@ # =================== require "../../audit/audit_log" +require "../../audit/signing" require "../../persistence/sqlite/sqlite_persistence" +require "../bootstrap" module CRE::Cli::Commands class Audit @@ -13,7 +15,15 @@ module CRE::Cli::Commands case sub when "verify" then verify(argv, io) when nil, "--help", "-h" - io.puts "Usage: cre audit verify [--db=PATH]" + io.puts <<-USAGE + Usage: cre audit verify [--db=PATH] [--public-key=PATH] + + Verifies the local audit log in three layers: + - hash chain (always) + - HMAC ratchet (always; requires CRE_HMAC_KEY_HEX) + - Merkle batch signatures (when --public-key=PATH or + CRE_AUDIT_PUBLIC_KEY_HEX is set) + USAGE 0 else io.puts "unknown audit subcommand: #{sub}" @@ -23,21 +33,53 @@ module CRE::Cli::Commands private def verify(argv : Array(String), io : IO) : Int32 db_path = ENV["CRE_DB_PATH"]? || "cre.db" - hmac_hex = ENV["CRE_HMAC_KEY_HEX"]? || "0" * 64 + public_key_hex = ENV["CRE_AUDIT_PUBLIC_KEY_HEX"]? + public_key_path = nil OptionParser.parse(argv) do |parser| parser.on("--db=PATH", "") { |p| db_path = p } + parser.on("--public-key=PATH", "Ed25519 public key (32 bytes hex) for batch signature verification") { |p| public_key_path = p } end + hmac_key = Bootstrap.require_hmac_key + persist = CRE::Persistence::Sqlite::SqlitePersistence.new(db_path) persist.migrate! - log = CRE::Audit::AuditLog.new(persist, hmac_hex.hexbytes, 1, 1024) - ok = log.verify_chain + log = CRE::Audit::AuditLog.new(persist, hmac_key, 1, 1024) + + hash_ok = log.verify_hash_chain + hmac_ok = log.verify_hmac_ratchet(hmac_key) latest_seq = persist.audit.latest_seq + + verifier_pem_hex = if path = public_key_path + File.read(path).strip + else + public_key_hex + end + + batches_ok = true + if (hex = verifier_pem_hex) && !hex.empty? + if hex.size != 64 + io.puts "✗ public key must be 64 hex chars (32 bytes); got #{hex.size}" + persist.close + return 2 + end + verifier = CRE::Audit::Signing::Ed25519Verifier.new(hex.hexbytes) + batches_ok = log.verify_batches(verifier) + end + persist.close - if ok + print_result(io, "hash chain", hash_ok) + print_result(io, "HMAC ratchet", hmac_ok) + if verifier_pem_hex + print_result(io, "Merkle batches", batches_ok) + else + io.puts " - Merkle batches not checked (no public key supplied)" + end + + if hash_ok && hmac_ok && batches_ok io.puts "✓ audit chain valid: #{latest_seq} entries" 0 else @@ -45,5 +87,9 @@ module CRE::Cli::Commands 2 end end + + private def print_result(io : IO, label : String, ok : Bool) : Nil + io.puts " #{ok ? "✓" : "✗"} #{label}: #{ok ? "OK" : "FAILED"}" + end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/rotate.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/rotate.cr index a52e7fcd..d57ae558 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/rotate.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/rotate.cr @@ -3,10 +3,10 @@ # rotate.cr # =================== +require "../bootstrap" require "../../engine/event_bus" require "../../engine/rotation_orchestrator" -require "../../persistence/sqlite/sqlite_persistence" -require "../../rotators/env_file" +require "../../engine/rotation_worker" module CRE::Cli::Commands class Rotate @@ -17,7 +17,7 @@ module CRE::Cli::Commands OptionParser.parse(argv) do |parser| parser.banner = "Usage: cre rotate [options]" - parser.on("--db=URL", "") { |u| db_url = u } + parser.on("--db=URL", "database URL (sqlite:path or postgres://...)") { |u| db_url = u } parser.on("-h", "--help") { _help_requested = true; io.puts parser } parser.unknown_args { |args| cred_id_str = args.first? } end @@ -34,34 +34,31 @@ module CRE::Cli::Commands return 64 end - persist = if db_url.starts_with?("sqlite:") - CRE::Persistence::Sqlite::SqlitePersistence.new(db_url.lchop("sqlite:")) - else - raise "rotate currently supports SQLite only via CLI shortcut" - end + envelope = CRE::Cli::Bootstrap.envelope + + persist = CRE::Cli::Bootstrap.build_persistence(db_url) persist.migrate! cred = persist.credentials.find(cred_id) if cred.nil? io.puts "credential not found: #{cred_id}" - return 1 - end - - rotator_class = CRE::Rotators::Rotator.for(rotator_kind_for(cred.kind)) - if rotator_class.nil? - io.puts "no rotator registered for kind=#{cred.kind}" + persist.close return 1 end bus = CRE::Engine::EventBus.new bus.run - orchestrator = CRE::Engine::RotationOrchestrator.new(bus, persist) + orchestrator = CRE::Engine::RotationOrchestrator.new(bus, persist, envelope) + worker = CRE::Engine::RotationWorker.new(bus, orchestrator, persist) + CRE::Cli::Bootstrap.register_rotators(worker, io) - rotator = case cred.kind - when CRE::Domain::CredentialKind::EnvFile then CRE::Rotators::EnvFileRotator.new - else - raise "this CLI shortcut only supports env_file via direct rotation; cloud rotators need full daemon config" - end + rotator = worker.rotator_for_kind(cred.kind) + if rotator.nil? + io.puts "no rotator registered for #{cred.kind} (set the matching env vars; see README)" + bus.stop + persist.close + return 1 + end io.puts "Rotating #{cred.name} (#{cred.id}) via #{rotator.kind}..." state = orchestrator.run(cred, rotator) @@ -70,21 +67,14 @@ module CRE::Cli::Commands persist.close case state - when CRE::Persistence::RotationState::Completed then io.puts "✓ rotation completed"; 0 - when CRE::Persistence::RotationState::Failed then io.puts "✗ rotation failed"; 1 - else io.puts "rotation ended in unexpected state #{state}"; 2 - end - end - - private def rotator_kind_for(kind : CRE::Domain::CredentialKind) : Symbol - case kind - in .aws_secretsmgr? then :aws_secretsmgr - in .vault_dynamic? then :vault_dynamic - in .github_pat? then :github_pat - in .env_file? then :env_file - in .aws_iam_key? then :aws_iam_key - in .database? then :database + when CRE::Persistence::RotationState::Completed then io.puts "✓ rotation completed"; 0 + when CRE::Persistence::RotationState::Failed then io.puts "✗ rotation failed"; 1 + when CRE::Persistence::RotationState::Inconsistent then io.puts "✗ rotation INCONSISTENT — manual intervention required"; 2 + else io.puts "rotation ended in unexpected state #{state}"; 2 end + rescue ex : CRE::Cli::Bootstrap::ConfigError + io.puts "configuration error: #{ex.message}" + 78 end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/run.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/run.cr index d1f20685..62e4ca7a 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/run.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/run.cr @@ -3,24 +3,18 @@ # run.cr # =================== +require "../bootstrap" require "../../engine/engine" require "../../engine/scheduler" require "../../engine/rotation_orchestrator" require "../../engine/rotation_worker" -require "../../persistence/sqlite/sqlite_persistence" -require "../../persistence/postgres/postgres_persistence" +require "../../audit/batch_sealer" +require "../../audit/batch_sealer_scheduler" require "../../policy/evaluator" require "../../notifiers/log_notifier" require "../../notifiers/telegram" require "../../notifiers/telegram_subscriber" require "../../notifiers/telegram_bot" -require "../../rotators/env_file" -require "../../rotators/aws_secrets" -require "../../rotators/vault_dynamic" -require "../../rotators/github_pat" -require "../../aws/secrets_client" -require "../../vault/client" -require "../../github/client" module CRE::Cli::Commands class Run @@ -40,7 +34,6 @@ module CRE::Cli::Commands def execute(argv : Array(String), io : IO) : Int32 _help_requested = false db_url = ENV["DATABASE_URL"]? || "sqlite:cre.db" - hmac_hex = ENV["CRE_HMAC_KEY_HEX"]? || "0" * 64 interval = (ENV["CRE_TICK_SECONDS"]? || "60").to_i OptionParser.parse(argv) do |parser| @@ -51,17 +44,26 @@ module CRE::Cli::Commands end return 0 if _help_requested - persist = build_persistence(db_url) + hmac_key = Bootstrap.require_hmac_key + envelope = Bootstrap.require_envelope + signer = Bootstrap.signer + + persist = Bootstrap.build_persistence(db_url) persist.migrate! - engine = CRE::Engine::Engine.new(persist, hmac_hex.hexbytes) + engine = CRE::Engine::Engine.new(persist, hmac_key) log_notifier = CRE::Notifiers::LogNotifier.new(engine.bus) evaluator = CRE::Policy::Evaluator.new(engine.bus, persist) scheduler = CRE::Engine::Scheduler.new(engine.bus, interval.seconds) - orchestrator = CRE::Engine::RotationOrchestrator.new(engine.bus, persist) + orchestrator = CRE::Engine::RotationOrchestrator.new(engine.bus, persist, envelope) worker = CRE::Engine::RotationWorker.new(engine.bus, orchestrator, persist) - register_rotators(worker, io) + Bootstrap.register_rotators(worker, io) + + sealer_scheduler = if signer_obj = signer + sealer = CRE::Audit::BatchSealer.new(persist, signer_obj) + CRE::Audit::BatchSealerScheduler.new(engine.bus, sealer, Bootstrap.seal_interval) + end telegram_pieces = wire_telegram(engine.bus, persist, io) @@ -70,10 +72,13 @@ module CRE::Cli::Commands worker.start evaluator.start scheduler.start + sealer_scheduler.try(&.start) telegram_pieces.each(&.start) - io.puts "cre running. PID #{Process.pid}, tick #{interval}s, db #{redact(db_url)}" + io.puts "cre running. PID #{Process.pid}, tick #{interval}s, db #{Bootstrap.redact_db_url(db_url)}" io.puts "rotators: #{worker.kinds.map(&.to_s).join(", ")}" + io.puts "envelope: AES-256-GCM (KEK v#{ENV[Bootstrap::KEK_VERSION_VAR]? || "1"})" + io.puts "audit batches: #{sealer_scheduler.nil? ? "(disabled — set #{Bootstrap::SIGNING_KEY_VAR})" : "every #{Bootstrap.seal_interval.total_seconds.to_i}s"}" io.puts "telegram: #{telegram_pieces.empty? ? "(disabled)" : "enabled"}" stop_signal = Channel(Nil).new @@ -82,6 +87,7 @@ module CRE::Cli::Commands scheduler.stop evaluator.stop worker.stop + sealer_scheduler.try(&.stop) log_notifier.stop telegram_pieces.each(&.stop) engine.stop @@ -91,44 +97,9 @@ module CRE::Cli::Commands stop_signal.receive 0 - end - - private def build_persistence(url : String) : CRE::Persistence::Persistence - if url.starts_with?("sqlite:") - CRE::Persistence::Sqlite::SqlitePersistence.new(url.lchop("sqlite:")) - elsif url.starts_with?("postgres://") || url.starts_with?("postgresql://") - CRE::Persistence::Postgres::PostgresPersistence.new(url) - else - raise "unknown database URL: #{url}" - end - end - - private def register_rotators(worker : CRE::Engine::RotationWorker, io : IO) : Nil - worker.register(:env_file, CRE::Rotators::EnvFileRotator.new) - - if (aws_id = ENV["AWS_ACCESS_KEY_ID"]?) && (aws_secret = ENV["AWS_SECRET_ACCESS_KEY"]?) - client = CRE::Aws::SecretsManagerClient.new( - access_key_id: aws_id, - secret_access_key: aws_secret, - region: ENV["AWS_REGION"]? || "us-east-1", - endpoint: ENV["AWS_ENDPOINT"]?, - session_token: ENV["AWS_SESSION_TOKEN"]?, - ) - worker.register(:aws_secretsmgr, CRE::Rotators::AwsSecretsRotator.new(client)) - end - - if (vault_addr = ENV["VAULT_ADDR"]?) && (vault_token = ENV["VAULT_TOKEN"]?) - client = CRE::Vault::Client.new(addr: vault_addr, token: vault_token) - worker.register(:vault_dynamic, CRE::Rotators::VaultDynamicRotator.new(client)) - end - - if gh_token = ENV["GITHUB_TOKEN"]? - api = ENV["GITHUB_API_BASE"]? || "https://api.github.com" - client = CRE::Github::Client.new(token: gh_token, api_base: api) - worker.register(:github_pat, CRE::Rotators::GithubPatRotator.new(client)) - end - rescue ex - io.puts "warning: rotator wiring failed: #{ex.message}" + rescue ex : CRE::Cli::Bootstrap::ConfigError + io.puts "configuration error: #{ex.message}" + 78 # EX_CONFIG end private def wire_telegram(bus : CRE::Engine::EventBus, persist : CRE::Persistence::Persistence, io : IO) : Array(StartStop) @@ -153,8 +124,8 @@ module CRE::Cli::Commands viewer_chats: viewer_chats, operator_chats: operator_chats, ) - pieces << StartStop.new(start_proc: ->{ sub.start }, stop_proc: ->{ sub.stop }) - pieces << StartStop.new(start_proc: ->{ bot.start }, stop_proc: ->{ bot.stop }) + pieces << StartStop.new(start_proc: -> { sub.start }, stop_proc: -> { sub.stop }) + pieces << StartStop.new(start_proc: -> { bot.start }, stop_proc: -> { bot.stop }) pieces end @@ -162,9 +133,5 @@ module CRE::Cli::Commands return [] of Int64 if raw.nil? || raw.empty? raw.split(',').map(&.strip).reject(&.empty?).map(&.to_i64) end - - private def redact(url : String) : String - url.gsub(/:\/\/[^:]+:[^@]+@/) { |_| "://****:****@" } - end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/verify_bundle.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/verify_bundle.cr new file mode 100644 index 00000000..1642fa4e --- /dev/null +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/verify_bundle.cr @@ -0,0 +1,218 @@ +# =================== +# ©AngelaMos | 2026 +# verify_bundle.cr +# =================== + +require "compress/zip" +require "json" +require "openssl/digest" +require "../../audit/hash_chain" +require "../../audit/signing" +require "../../audit/merkle" + +module CRE::Cli::Commands + # VerifyBundle reads a compliance evidence ZIP produced by 'cre export' + # and re-runs every check the bundle's README documents: + # + # 1. Per-file SHA-256 vs manifest.json + # 2. Manifest Ed25519 signature vs public_key.pem (if both present) + # 3. Audit log hash chain reconstruction + # 4. Audit batch Merkle-root + signature reconstruction + # + # Exit codes: + # 0 - all checks passed + # 2 - one or more checks failed + # 64 - usage error + class VerifyBundle + def execute(argv : Array(String), io : IO) : Int32 + _help_requested = false + bundle_path = nil + + OptionParser.parse(argv) do |parser| + parser.banner = "Usage: cre verify-bundle " + parser.on("-h", "--help") { _help_requested = true; io.puts parser } + parser.unknown_args { |args| bundle_path = args.first? } + end + return 0 if _help_requested + + if bundle_path.nil? + io.puts "usage: cre verify-bundle " + return 64 + end + bundle = bundle_path.not_nil! + + unless File.exists?(bundle) + io.puts "bundle not found: #{bundle}" + return 2 + end + + contents = read_zip(bundle) + manifest_text = contents["manifest.json"]? + if manifest_text.nil? + io.puts "✗ bundle is missing manifest.json" + return 2 + end + manifest = JSON.parse(manifest_text) + + checks_passed = true + checks_passed &= verify_checksums(io, contents, manifest) + checks_passed &= verify_manifest_signature(io, contents, manifest_text) + checks_passed &= verify_hash_chain(io, contents) + checks_passed &= verify_batches(io, contents) + + if checks_passed + io.puts "✓ bundle valid" + 0 + else + io.puts "✗ bundle FAILED verification" + 2 + end + end + + private def read_zip(path : String) : Hash(String, String) + out = {} of String => String + Compress::Zip::File.open(path) do |zip| + zip.entries.each do |entry| + out[entry.filename] = entry.open(&.gets_to_end) + end + end + out + end + + private def verify_checksums(io : IO, contents : Hash(String, String), manifest : JSON::Any) : Bool + ok = true + manifest["files"].as_a.each do |entry| + name = entry["name"].as_s + expected = entry["sha256"].as_s + body = contents[name]? + if body.nil? + io.puts " ✗ checksum: missing #{name}" + ok = false + next + end + actual = sha256_hex(body) + if actual != expected + io.puts " ✗ checksum mismatch on #{name}: expected #{expected}, got #{actual}" + ok = false + end + end + io.puts " ✓ #{manifest["files"].as_a.size} file checksums match" if ok + ok + end + + private def verify_manifest_signature(io : IO, contents : Hash(String, String), manifest_text : String) : Bool + sig_b64 = contents["manifest.sig"]? + pubkey_hex = contents["public_key.pem"]? + if sig_b64.nil? || pubkey_hex.nil? + io.puts " - manifest.sig / public_key.pem absent — signature step skipped" + return true + end + + sig = Base64.decode(sig_b64.strip) + key_bytes = pubkey_hex.strip.hexbytes + verifier = CRE::Audit::Signing::Ed25519Verifier.new(key_bytes) + if verifier.verify(manifest_text.to_slice, sig) + io.puts " ✓ manifest signature valid" + true + else + io.puts " ✗ manifest signature INVALID" + false + end + rescue ex + io.puts " ✗ manifest signature verification error: #{ex.message}" + false + end + + private def verify_hash_chain(io : IO, contents : Hash(String, String)) : Bool + ndjson = contents["audit_log.ndjson"]? + if ndjson.nil? || ndjson.empty? + io.puts " - audit_log.ndjson empty — hash chain step skipped" + return true + end + + prev = Bytes.new(32, 0_u8) + lineno = 0 + ndjson.each_line do |line| + lineno += 1 + next if line.empty? + row = JSON.parse(line) + # The exporter writes 'payload' as the canonical JSON the log used + # when computing content_hash, so we hash that string verbatim. + canonical = row["payload"].as_s + expected = CRE::Audit::HashChain.next_hash(prev, canonical.to_slice) + actual = row["content_hash_hex"].as_s.hexbytes + unless expected == actual + io.puts " ✗ hash chain broken at seq=#{row["seq"]?} (line #{lineno})" + return false + end + prev = actual + end + io.puts " ✓ hash chain valid (#{lineno} entries)" + true + end + + private def verify_batches(io : IO, contents : Hash(String, String)) : Bool + batches_json = contents["audit_batches.json"]? + pubkey_hex = contents["public_key.pem"]? + if batches_json.nil? || batches_json == "[]" + io.puts " - audit_batches.json empty — batch verify skipped" + return true + end + if pubkey_hex.nil? + io.puts " ✗ batches present but public_key.pem absent — cannot verify" + return false + end + + verifier = CRE::Audit::Signing::Ed25519Verifier.new(pubkey_hex.strip.hexbytes) + ndjson = contents["audit_log.ndjson"]? || "" + content_hashes_by_seq = {} of Int64 => Bytes + ndjson.each_line do |line| + next if line.empty? + row = JSON.parse(line) + content_hashes_by_seq[row["seq"].as_i64] = row["content_hash_hex"].as_s.hexbytes + end + + batches = JSON.parse(batches_json).as_a + batches.each do |b| + start_seq = b["start_seq"].as_i64 + end_seq = b["end_seq"].as_i64 + leaves = (start_seq..end_seq).map do |seq| + h = content_hashes_by_seq[seq]? + if h.nil? + io.puts " ✗ batch covers seq=#{seq} but ndjson is missing it" + return false + end + h + end + recomputed = CRE::Audit::Merkle.root(leaves) + stored = b["merkle_root_hex"].as_s.hexbytes + unless recomputed == stored + io.puts " ✗ Merkle root mismatch on batch [#{start_seq}..#{end_seq}]" + return false + end + msg = pack_batch_message(start_seq, end_seq, stored) + sig = b["signature_hex"].as_s.hexbytes + unless verifier.verify(msg, sig) + io.puts " ✗ batch signature invalid for [#{start_seq}..#{end_seq}]" + return false + end + end + io.puts " ✓ #{batches.size} audit batches verified" + true + end + + private def pack_batch_message(start_seq : Int64, end_seq : Int64, root : Bytes) : Bytes + io = IO::Memory.new + io.write_bytes(start_seq, IO::ByteFormat::BigEndian) + io.write_bytes(end_seq, IO::ByteFormat::BigEndian) + io.write(root) + io.to_slice + end + + private def sha256_hex(content : String) : String + d = OpenSSL::Digest.new("SHA256") + d.update(content) + d.hexfinal + end + end +end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/watch.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/watch.cr index 1c1118c4..e35e41f7 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/watch.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/cli/commands/watch.cr @@ -3,10 +3,13 @@ # watch.cr # =================== +require "../bootstrap" require "../../engine/engine" require "../../engine/scheduler" -require "../../persistence/sqlite/sqlite_persistence" -require "../../persistence/postgres/postgres_persistence" +require "../../engine/rotation_orchestrator" +require "../../engine/rotation_worker" +require "../../audit/batch_sealer" +require "../../audit/batch_sealer_scheduler" require "../../policy/evaluator" require "../../tui/tui" @@ -15,38 +18,55 @@ module CRE::Cli::Commands def execute(argv : Array(String), io : IO) : Int32 _help_requested = false db_url = ENV["DATABASE_URL"]? || "sqlite:cre.db" - hmac_hex = ENV["CRE_HMAC_KEY_HEX"]? || "0" * 64 + interval = (ENV["CRE_TICK_SECONDS"]? || "60").to_i OptionParser.parse(argv) do |parser| parser.banner = "Usage: cre watch [options]" - parser.on("--db=URL", "") { |u| db_url = u } + parser.on("--db=URL", "database URL (sqlite:path or postgres://...)") { |u| db_url = u } + parser.on("--interval=SECONDS", "scheduler tick interval") { |i| interval = i.to_i } parser.on("-h", "--help") { _help_requested = true; io.puts parser } end return 0 if _help_requested - persist = if db_url.starts_with?("sqlite:") - CRE::Persistence::Sqlite::SqlitePersistence.new(db_url.lchop("sqlite:")) - elsif db_url.starts_with?("postgres://") || db_url.starts_with?("postgresql://") - CRE::Persistence::Postgres::PostgresPersistence.new(db_url) - else - raise "unknown database URL" - end + hmac_key = Bootstrap.require_hmac_key + envelope = Bootstrap.require_envelope + signer = Bootstrap.signer + + persist = Bootstrap.build_persistence(db_url) persist.migrate! - engine = CRE::Engine::Engine.new(persist, hmac_hex.hexbytes) + engine = CRE::Engine::Engine.new(persist, hmac_key) evaluator = CRE::Policy::Evaluator.new(engine.bus, persist) - scheduler = CRE::Engine::Scheduler.new(engine.bus, 60.seconds) - tui = CRE::Tui::Tui.new(engine.bus) + scheduler = CRE::Engine::Scheduler.new(engine.bus, interval.seconds) + orchestrator = CRE::Engine::RotationOrchestrator.new(engine.bus, persist, envelope) + worker = CRE::Engine::RotationWorker.new(engine.bus, orchestrator, persist) + Bootstrap.register_rotators(worker, io) + + sealer_scheduler = if signer_obj = signer + sealer = CRE::Audit::BatchSealer.new(persist, signer_obj) + CRE::Audit::BatchSealerScheduler.new(engine.bus, sealer, Bootstrap.seal_interval) + end + + kek_version = (ENV[Bootstrap::KEK_VERSION_VAR]? || "1").to_i + tui_state = CRE::Tui::State.new(kek_version: kek_version) + tui_snapshotter = CRE::Tui::Snapshotter.new(tui_state, persist) + tui = CRE::Tui::Tui.new(engine.bus, tui_state) engine.start + worker.start evaluator.start scheduler.start + sealer_scheduler.try(&.start) + tui_snapshotter.start tui.start Signal::INT.trap do tui.stop + tui_snapshotter.stop scheduler.stop evaluator.stop + worker.stop + sealer_scheduler.try(&.stop) engine.stop persist.close exit 0 @@ -54,6 +74,9 @@ module CRE::Cli::Commands sleep 0 + rescue ex : CRE::Cli::Bootstrap::ConfigError + io.puts "configuration error: #{ex.message}" + 78 end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/compliance/bundle.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/compliance/bundle.cr index 23282a04..39df8aca 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/compliance/bundle.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/compliance/bundle.cr @@ -19,7 +19,9 @@ module CRE::Compliance # manifest.json - file checksums + signature # audit_log.ndjson - raw audit events with hash-chain fields # audit_batches.json - signed Merkle batch roots over the period - # public_key.pem - Ed25519 public key (for verification) + # public_key.pem - Ed25519 public key (32 hex bytes; SHA-256 + # covered by manifest so substitution shows + # up as a manifest checksum mismatch) # control_mapping.json - event_type -> framework controls class Bundle record FileEntry, name : String, sha256_hex : String, size : Int32 @@ -28,49 +30,53 @@ module CRE::Compliance @persistence : Persistence::Persistence, @framework : String, @signer : Audit::Signing::Ed25519Signer? = nil, - @public_key_pem : String? = nil, + @public_key_hex : String? = nil, ) end def write(path : String) : Nil File.open(path, "w") do |fp| Compress::Zip::Writer.open(fp) do |zip| - entries = [] of FileEntry - - add_file(zip, entries, "audit_log.ndjson", build_audit_log_ndjson) - add_file(zip, entries, "audit_batches.json", build_audit_batches_json) - add_file(zip, entries, "control_mapping.json", build_control_mapping_json) - add_file(zip, entries, "README.md", build_readme(entries)) - - manifest = build_manifest(entries) - add_file(zip, entries, "manifest.json", manifest) - - if pem = @public_key_pem - add_file(zip, entries, "public_key.pem", pem) + # 1) build raw bytes for every file we plan to ship + payload_files = [] of {String, String} + payload_files << {"audit_log.ndjson", build_audit_log_ndjson} + payload_files << {"audit_batches.json", build_audit_batches_json} + payload_files << {"control_mapping.json", build_control_mapping_json} + if pem = @public_key_hex + payload_files << {"public_key.pem", pem} end + # 2) compute checksums and build manifest covering ALL payload files + # (including public_key.pem so substitution invalidates the manifest sig) + payload_entries = payload_files.map do |(name, content)| + FileEntry.new(name: name, sha256_hex: sha256_hex(content), size: content.bytesize) + end + readme = build_readme(payload_entries) + payload_files << {"README.md", readme} + payload_entries << FileEntry.new(name: "README.md", sha256_hex: sha256_hex(readme), size: readme.bytesize) + + manifest = build_manifest(payload_entries) + + # 3) emit zip in dependency order + payload_files.each { |(name, content)| add_file(zip, name, content) } + add_file(zip, "manifest.json", manifest) if signer = @signer sig = signer.sign(manifest.to_slice) - add_file(zip, entries, "manifest.sig", Base64.encode(sig)) + add_file(zip, "manifest.sig", Base64.encode(sig)) end end end end - private def add_file(zip, entries : Array(FileEntry), name : String, content : String) : Nil + private def add_file(zip, name : String, content : String) : Nil zip.add(name) { |io| io << content } - entries << FileEntry.new( - name: name, - sha256_hex: sha256_hex(content), - size: content.bytesize, - ) end private def build_audit_log_ndjson : String latest = @persistence.audit.latest_seq return "" if latest == 0 io = IO::Memory.new - @persistence.audit.range(1_i64, latest).each do |entry| + @persistence.audit.each_in_range(1_i64, latest) do |entry| row = { "seq" => entry.seq, "event_id" => entry.event_id.to_s, @@ -91,12 +97,20 @@ module CRE::Compliance end private def build_audit_batches_json : String - sealed = @persistence.audit.last_sealed_seq - return "[]" if sealed == 0 - # We need a way to enumerate batches; AuditRepo currently exposes - # last_sealed_seq but not the batch list. For now write the most-recent - # batch metadata. Future work: add AuditRepo#all_batches. - "[]" + batches = @persistence.audit.all_batches + return "[]" if batches.empty? + rows = batches.map do |b| + { + "id" => b.id.to_s, + "start_seq" => b.start_seq, + "end_seq" => b.end_seq, + "merkle_root_hex" => b.merkle_root.hexstring, + "signature_hex" => b.signature.hexstring, + "signing_key_version" => b.signing_key_version, + "sealed_at" => b.sealed_at.to_rfc3339, + } + end + rows.to_json end private def build_control_mapping_json : String @@ -124,18 +138,28 @@ module CRE::Compliance - audit_log.ndjson raw audit events with hash-chain fields - audit_batches.json signed Merkle batches over the period - control_mapping.json event_type -> framework controls - - manifest.json per-file SHA-256 checksums + - public_key.pem Ed25519 public key (32 hex bytes) + - manifest.json per-file SHA-256 checksums (covers public_key.pem) - manifest.sig Ed25519 signature of manifest.json (if signed) - - public_key.pem Ed25519 public key (if signed) Verification: - 1. Recompute SHA-256 of each file and compare against manifest.json. - 2. If manifest.sig is present, verify with public_key.pem. - 3. Walk audit_log.ndjson and recompute the hash chain - each row's - content_hash should equal SHA256(prev_hash || canonical(payload+meta)). - - For automated verification, run: cre verify-bundle + + Manual verification: + 1. Recompute SHA-256 of every file listed in manifest.json and compare. + 2. Verify manifest.sig over manifest.json using public_key.pem. + 3. Walk audit_log.ndjson and recompute the hash chain - each row's + content_hash should equal SHA256(prev_hash || canonical(payload)). + 4. For each row in audit_batches.json, recompute the Merkle root over + content_hashes in [start_seq, end_seq] and verify signature_hex + with public_key.pem. + + Note on public_key.pem trust: + The bundled public key is fingerprint-protected by manifest.json's + SHA-256 entry. The manifest itself is Ed25519-signed; if an adversary + substitutes the key+sig pair, manifest.json's checksum no longer + matches the bundled file. Auditors should still verify the in-bundle + public key's SHA-256 against an out-of-band fingerprint they trust. MD end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/domain/credential.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/domain/credential.cr index a800bc59..f2d76eb2 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/domain/credential.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/domain/credential.cr @@ -8,11 +8,9 @@ require "uuid" module CRE::Domain enum CredentialKind AwsSecretsmgr - AwsIamKey VaultDynamic GithubPat EnvFile - Database end struct Credential @@ -26,6 +24,7 @@ module CRE::Domain getter previous_version_id : UUID? getter created_at : Time getter updated_at : Time + getter last_rotated_at : Time? def initialize( @id : UUID, @@ -38,11 +37,16 @@ module CRE::Domain @previous_version_id : UUID? = nil, @created_at : Time = Time.utc, @updated_at : Time = Time.utc, + @last_rotated_at : Time? = nil, ) end def tag(key : String | Symbol) : String? @tags[key.to_s]? end + + def rotation_anchor : Time + @last_rotated_at || @created_at + end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/engine.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/engine.cr index ba9b5c9e..0f8cf39f 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/engine.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/engine.cr @@ -39,7 +39,13 @@ module CRE::Engine return unless @started Log.info { "engine stopping" } @bus.publish(Events::ShutdownRequested.new) rescue nil - sleep 0.05.seconds # let subscribers see it + + # Wait until the audit subscriber has drained everything queued + # before ShutdownRequested. Bounded so a stuck subscriber can't + # hang shutdown forever. + drained = @audit_subscriber.await_drain(2.seconds) + Log.warn { "audit subscriber did not drain in time during shutdown" } unless drained + @audit_subscriber.stop @bus.stop @started = false diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/event_bus.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/event_bus.cr index 960e5ac4..e3627e88 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/event_bus.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/event_bus.cr @@ -22,7 +22,7 @@ module CRE::Engine @subs_mutex : Mutex @running : Bool - def initialize(inbox_capacity : Int32 = 1024) + def initialize(inbox_capacity : Int32 = 1024, @block_send_timeout : Time::Span = 5.seconds) @inbox = Channel(Events::Event).new(capacity: inbox_capacity) @subs = [] of Subscription @subs_mutex = Mutex.new @@ -61,10 +61,24 @@ module CRE::Engine end end + # Dispatch isolates each subscriber so a slow one cannot block the bus + # for everyone else (head-of-line blocking). + # + # Drop overflow uses a non-blocking select+default; rejections are + # logged. Block overflow uses a select with a timeout; if the + # subscriber's buffer stays full beyond the timeout, the event is + # logged as dropped and the bus moves on rather than freezing the + # entire pipeline. Operators tune the timeout up for slow downstreams + # they trust (DB writes) and down for unreliable ones (Telegram). private def dispatch(sub : Subscription, ev : Events::Event) : Nil case sub.overflow in Overflow::Block - sub.channel.send(ev) + select + when sub.channel.send(ev) + # delivered + when timeout(@block_send_timeout) + Log.warn { "subscriber stalled past #{@block_send_timeout.total_seconds}s; dropped: #{ev.class.name}" } + end in Overflow::Drop select when sub.channel.send(ev) diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_orchestrator.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_orchestrator.cr index 899315e8..9fa682d9 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_orchestrator.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_orchestrator.cr @@ -8,16 +8,33 @@ require "uuid" require "./event_bus" require "../events/credential_events" require "../rotators/rotator" +require "../crypto/envelope" require "../persistence/persistence" require "../persistence/repos" module CRE::Engine class VerifyFailed < Exception; end + # The orchestrator drives a credential through the four-step rotation + # contract (generate -> apply -> verify -> commit). On success it + # 1. seals the new secret with the optional Envelope and writes a + # credential_versions row, + # 2. bumps the credential's last_rotated_at and current/previous + # version pointers so the policy evaluator no longer sees it as + # overdue. + # + # Failures roll the credential back to its previous state. Failures in + # apply or verify call rotator.rollback_apply; commit failures additionally + # mark the rotation as Inconsistent because partial cloud-side stage + # transitions cannot always be undone client-side. class RotationOrchestrator Log = ::Log.for("cre.rotator") - def initialize(@bus : EventBus, @persistence : Persistence::Persistence) + def initialize( + @bus : EventBus, + @persistence : Persistence::Persistence, + @envelope : Crypto::Envelope? = nil, + ) end def run(c : Domain::Credential, rotator : Rotators::Rotator) : Persistence::RotationState @@ -58,28 +75,90 @@ module CRE::Engine current_step = :commit @bus.publish Events::RotationStepStarted.new(c.id, rotation_id, :commit) rotator.commit(c, new_secret) - @persistence.rotations.update_state(rotation_id, Persistence::RotationState::Completed) @bus.publish Events::RotationStepCompleted.new(c.id, rotation_id, :commit) - @bus.publish Events::RotationCompleted.new(c.id, rotation_id) + finalize_success(c, new_secret, rotation_id) Persistence::RotationState::Completed rescue ex - if ns = new_secret - if current_step == :apply || current_step == :verify - begin - rotator.rollback_apply(c, ns) - rescue rb - Log.error(exception: rb) { "rollback_apply failed for credential #{c.id}" } - end + if (ns = new_secret) && (current_step == :apply || current_step == :verify) + begin + rotator.rollback_apply(c, ns) + rescue rb + Log.error(exception: rb) { "rollback_apply failed for credential #{c.id}" } end end - @persistence.rotations.update_state(rotation_id, Persistence::RotationState::Failed, ex.message || ex.class.name) - @bus.publish Events::RotationStepFailed.new(c.id, rotation_id, current_step, ex.message || ex.class.name) - @bus.publish Events::RotationFailed.new(c.id, rotation_id, ex.message || ex.class.name) - Persistence::RotationState::Failed + finalize_failure(c, rotation_id, current_step, ex) end end + private def finalize_success(c : Domain::Credential, secret : Domain::NewSecret, rotation_id : UUID) : Nil + version_id = persist_credential_version(c, secret) + bump_credential(c, version_id) + @persistence.rotations.update_state(rotation_id, Persistence::RotationState::Completed) + @bus.publish Events::RotationCompleted.new(c.id, rotation_id) + end + + private def finalize_failure( + c : Domain::Credential, + rotation_id : UUID, + current_step : Symbol, + ex : Exception, + ) : Persistence::RotationState + reason = ex.message || ex.class.name + terminal_state = current_step == :commit ? Persistence::RotationState::Inconsistent : Persistence::RotationState::Failed + + @persistence.rotations.update_state(rotation_id, terminal_state, reason) + @bus.publish Events::RotationStepFailed.new(c.id, rotation_id, current_step, reason) + @bus.publish Events::RotationFailed.new(c.id, rotation_id, reason) + + if terminal_state == Persistence::RotationState::Inconsistent + @bus.publish Events::AlertRaised.new( + severity: Events::Severity::Critical, + message: "rotation #{rotation_id} for credential #{c.id} left in inconsistent state at commit step: #{reason}", + ) + end + + terminal_state + end + + private def persist_credential_version(c : Domain::Credential, secret : Domain::NewSecret) : UUID? + env = @envelope + return nil if env.nil? + + aad = "cred=#{c.id}|kind=#{c.kind}".to_slice + sealed = env.seal(secret.ciphertext, aad) + version = Domain::CredentialVersion.new( + id: UUID.random, + credential_id: c.id, + ciphertext: sealed.ciphertext, + dek_wrapped: sealed.dek_wrapped, + kek_version: sealed.kek_version, + algorithm_id: sealed.algorithm_id, + metadata: secret.metadata, + generated_at: secret.generated_at, + ) + @persistence.versions.insert(version) + version.id + end + + private def bump_credential(c : Domain::Credential, new_version_id : UUID?) : Nil + now = Time.utc + updated = Domain::Credential.new( + id: c.id, + external_id: c.external_id, + kind: c.kind, + name: c.name, + tags: c.tags, + current_version_id: new_version_id || c.current_version_id, + pending_version_id: nil, + previous_version_id: new_version_id ? c.current_version_id : c.previous_version_id, + created_at: c.created_at, + updated_at: now, + last_rotated_at: now, + ) + @persistence.credentials.update(updated) + end + private def kind_to_enum(kind : Symbol) : Persistence::RotatorKind case kind when :aws_secretsmgr then Persistence::RotatorKind::AwsSecretsmgr diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_worker.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_worker.cr index 1c5dde97..5c202fb5 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_worker.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/rotation_worker.cr @@ -38,6 +38,10 @@ module CRE::Engine @rotators.keys end + def rotator_for_kind(kind : Domain::CredentialKind) : Rotators::Rotator? + @rotators[symbol_for_kind(kind)]? + end + def start : Nil @running = true ch = @bus.subscribe(buffer: 32, overflow: EventBus::Overflow::Block) @@ -79,6 +83,11 @@ module CRE::Engine return end + if @persistence.rotations.in_flight.any? { |r| r.credential_id == cred.id } + Log.info { "rotation already in flight for credential #{cred.id}; skipping duplicate schedule" } + return + end + @orchestrator.run(cred, rotator) rescue ex Log.error(exception: ex) { "rotation_worker.handle failed for event #{ev.class.name}" } @@ -90,8 +99,6 @@ module CRE::Engine in .vault_dynamic? then :vault_dynamic in .github_pat? then :github_pat in .env_file? then :env_file - in .aws_iam_key? then :aws_iam_key - in .database? then :database end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/subscribers/audit_subscriber.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/subscribers/audit_subscriber.cr index fab3eb8c..77d14af0 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/subscribers/audit_subscriber.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/engine/subscribers/audit_subscriber.cr @@ -12,9 +12,11 @@ module CRE::Engine::Subscribers class AuditSubscriber @ch : Channel(Events::Event)? @running : Bool + @drain_ready : ::Channel(Nil) def initialize(@bus : EventBus, @log : Audit::AuditLog, @actor : String = "system") @running = false + @drain_ready = ::Channel(Nil).new(1) end def start : Nil @@ -38,8 +40,24 @@ module CRE::Engine::Subscribers @ch.try(&.close) end + # Block until either the subscriber receives ShutdownRequested (i.e. + # has drained everything published before the shutdown signal) or the + # timeout elapses. Lets engine.stop deterministically wait instead of + # sleeping for a magic 50ms. + def await_drain(timeout_span : Time::Span = 2.seconds) : Bool + select + when @drain_ready.receive + true + when timeout(timeout_span) + false + end + end + private def handle(ev : Events::Event) : Nil case ev + when Events::ShutdownRequested + @drain_ready.send(nil) rescue nil + return when Events::RotationCompleted @log.append("rotation.completed", @actor, ev.credential_id, { "rotation_id" => ev.rotation_id.to_s, @@ -77,9 +95,25 @@ module CRE::Engine::Subscribers "severity" => ev.severity.to_s, "message" => ev.message, }) + when Events::AuditBatchSealed + @log.append("audit.batch.sealed", @actor, nil, { + "start_seq" => ev.start_seq.to_s, + "end_seq" => ev.end_seq.to_s, + "signing_key_version" => ev.signing_key_version.to_s, + }) end rescue ex EventBus::Log.error(exception: ex) { "audit subscriber failed to write" } + @bus.publish(Events::AlertRaised.new( + severity: Events::Severity::Critical, + message: "audit log write failed: #{ex.message} (event=#{ev.class.name})", + )) rescue nil + + mode = ENV["CRE_AUDIT_FAILURE_MODE"]? || "panic" + if mode == "panic" + STDERR.puts "FATAL: audit log write failed in panic mode: #{ex.message} (event=#{ev.class.name})" + Process.exit(2) + end end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/events/system_events.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/events/system_events.cr index 9b75e305..61d10dbd 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/events/system_events.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/events/system_events.cr @@ -26,4 +26,14 @@ module CRE::Events class ShutdownRequested < Event end + + class AuditBatchSealed < Event + getter start_seq : Int64 + getter end_seq : Int64 + getter signing_key_version : Int32 + + def initialize(@start_seq : Int64, @end_seq : Int64, @signing_key_version : Int32) + super() + end + end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/github/client.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/github/client.cr index f4f1baa8..534c2549 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/github/client.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/github/client.cr @@ -5,6 +5,7 @@ require "http/client" require "json" +require "../http/retry" module CRE::Github class GithubError < Exception @@ -15,10 +16,11 @@ module CRE::Github end end - # Thin GitHub REST client. We target fine-grained PATs for rotation since the - # /user/personal-access-tokens endpoint accepts programmatic creation / - # deletion when the bearer token has the appropriate Apps-managed permission. - # For the test/portfolio path we mock these endpoints directly. + # Thin GitHub REST client. We target fine-grained PATs for rotation since + # the /user/personal-access-tokens endpoint accepts programmatic creation + # and deletion when the bearer has the appropriate Apps-managed + # permission. For the test/portfolio path we mock these endpoints + # directly. class Client record Token, id : Int64, token_value : String, expires_at : String? @@ -50,19 +52,19 @@ module CRE::Github end private def get(path : String) : JSON::Any - response = HTTP::Client.get(url(path), headers: headers) + response = CRE::Http.request("GET", url(path), headers, label: "github.GET#{path}") raise GithubError.new("GET #{path}: #{response.body[0, 200]?}", response.status_code) unless response.status_code < 300 JSON.parse(response.body) end private def post(path : String, body : String) : JSON::Any - response = HTTP::Client.post(url(path), headers: headers, body: body) + response = CRE::Http.request("POST", url(path), headers, body, label: "github.POST#{path}") raise GithubError.new("POST #{path}: #{response.body[0, 200]?}", response.status_code) unless response.status_code < 300 JSON.parse(response.body) end private def delete(path : String) : Nil - response = HTTP::Client.delete(url(path), headers: headers) + response = CRE::Http.request("DELETE", url(path), headers, label: "github.DELETE#{path}") raise GithubError.new("DELETE #{path}: #{response.body[0, 200]?}", response.status_code) unless response.status_code < 300 end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/http/retry.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/http/retry.cr new file mode 100644 index 00000000..baf79dbe --- /dev/null +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/http/retry.cr @@ -0,0 +1,90 @@ +# =================== +# ©AngelaMos | 2026 +# retry.cr +# =================== + +require "http/client" +require "log" + +module CRE::Http + # Transient HTTP statuses where retrying makes sense. We do NOT retry + # on 401/403/404 — those mean the request is bad, not flaky. + RETRY_STATUSES = {408, 425, 429, 500, 502, 503, 504}.to_set + + # Retry wraps a Proc with bounded retry-with-jitter. Default policy: + # 3 attempts (initial + 2 retries), 200ms..2s exponential backoff with + # full jitter, retries network errors and the canonical transient + # statuses listed above. + module Retry + Log = ::Log.for("cre.http.retry") + + def self.with( + *, + attempts : Int32 = 3, + base_delay : Time::Span = 200.milliseconds, + max_delay : Time::Span = 2.seconds, + retry_statuses : Set(Int32) = RETRY_STATUSES, + label : String = "http", + &block : -> HTTP::Client::Response + ) : HTTP::Client::Response + attempt = 0 + loop do + attempt += 1 + begin + response = block.call + if retry_statuses.includes?(response.status_code) && attempt < attempts + Log.warn { "#{label}: HTTP #{response.status_code} on attempt #{attempt}; retrying" } + sleep_with_jitter(base_delay, max_delay, attempt) + next + end + return response + rescue ex : IO::TimeoutError | Socket::ConnectError | IO::Error + if attempt < attempts + Log.warn(exception: ex) { "#{label}: transient error on attempt #{attempt}; retrying" } + sleep_with_jitter(base_delay, max_delay, attempt) + next + end + raise ex + end + end + end + + private def self.sleep_with_jitter(base : Time::Span, ceiling : Time::Span, attempt : Int32) : Nil + backoff_ms = base.total_milliseconds * (1 << (attempt - 1)) + capped_ms = Math.min(backoff_ms, ceiling.total_milliseconds) + jittered = (capped_ms * Random.rand).to_i + sleep jittered.milliseconds + end + end + + # Connect/read timeouts come from env so operators can tune per + # environment without recompiling. Defaults are conservative for cloud + # APIs where 5xx blips are normal. + CONNECT_TIMEOUT = ((ENV["CRE_HTTP_CONNECT_TIMEOUT_S"]? || "5").to_f).seconds + READ_TIMEOUT = ((ENV["CRE_HTTP_READ_TIMEOUT_S"]? || "30").to_f).seconds + + # Single-shot HTTP request with timeouts + retry. Returns the + # HTTP::Client::Response. The HTTP::Client instance is created per + # call and closed in an ensure block so a hung peer can't pin + # resources past the timeout. Webmock intercepts via instance method + # override so test stubs continue to apply unchanged. + def self.request(method : String, url : String, headers : HTTP::Headers, body : String = "", label : String = "http") : HTTP::Client::Response + uri = URI.parse(url) + request_target = String.build do |s| + s << (uri.path.empty? ? "/" : uri.path) + if (q = uri.query) && !q.empty? + s << "?" << q + end + end + Retry.with(label: label) do + client = HTTP::Client.new(uri) + client.connect_timeout = CONNECT_TIMEOUT + client.read_timeout = READ_TIMEOUT + begin + client.exec(method, request_target, headers: headers, body: body) + ensure + client.close + end + end + end +end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram.cr index b8628a15..66d7804f 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram.cr @@ -6,11 +6,18 @@ require "http/client" require "json" require "log" +require "../http/retry" module CRE::Notifiers # Thin Telegram Bot API client. We hit api.telegram.org/bot/ - # directly with HTTP::Client; no tourmaline dependency for the notification - # path keeps the footprint small. + # directly with HTTP::Client; no tourmaline dependency for the + # notification path keeps the footprint small. + # + # Telegram requires the bot token to live in the URL path. To prevent + # leaks, error messages substitute the token with '****' before + # surfacing any URL fragment. The token is still in the underlying + # request's URI; operators wiring a logging proxy in front of cre + # should redact at the proxy layer too. class Telegram Log = ::Log.for("cre.telegram") DEFAULT_API = "https://api.telegram.org" @@ -55,11 +62,18 @@ module CRE::Notifiers end private def call(method : String, body : String) : JSON::Any - uri = "#{@api_base}/bot#{@token}/#{method}" + url = "#{@api_base}/bot#{@token}/#{method}" headers = HTTP::Headers{"Content-Type" => "application/json"} - response = HTTP::Client.post(uri, headers: headers, body: body) - raise TelegramError.new("telegram #{method} #{response.status_code}: #{response.body[0, 200]?}", response.status_code) unless response.status_code < 300 + response = CRE::Http.request("POST", url, headers, body, label: "telegram.#{method}") + unless response.status_code < 300 + raise TelegramError.new("telegram #{method} #{response.status_code}: #{redact(response.body[0, 200]?)}", response.status_code) + end JSON.parse(response.body) end + + private def redact(text : String?) : String? + return nil if text.nil? + text.gsub(@token, "****") + end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram_bot.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram_bot.cr index f59767e6..18d424f5 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram_bot.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/notifiers/telegram_bot.cr @@ -68,7 +68,6 @@ module CRE::Notifiers when "alerts" then alerts_message when "help", "" then help_message when "rotate" then handle_rotate(chat_id, rest) - when "snooze" then handle_snooze(chat_id, rest) when "history" then history_message(rest) else "unknown command: /#{cmd} (try /help)" @@ -112,7 +111,6 @@ module CRE::Notifiers /history - last events for a credential /alerts - critical alerts pointer /rotate - force rotation (operator) - /snooze 24h - defer scheduled rotation (operator) MD end @@ -126,11 +124,6 @@ module CRE::Notifiers "rotation scheduled for #{uuid}" end - private def handle_snooze(chat_id : Int64, rest : String) : String - return "operator-only command" unless authorized_operator?(chat_id) - "snooze is not yet implemented; track via /queue" - end - private def history_message(rest : String) : String id_str = rest.strip return "usage: /history " if id_str.empty? diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/audit_repo.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/audit_repo.cr index a991749a..72c24711 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/audit_repo.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/audit_repo.cr @@ -11,12 +11,15 @@ module CRE::Persistence::Postgres class AuditRepo < CRE::Persistence::AuditRepo GENESIS_HASH = Bytes.new(32, 0_u8) + SELECT_ENTRY_COLS = "seq, event_id::text, occurred_at, event_type, actor, target_id::text, payload::text, prev_hash, content_hash, hmac, hmac_key_version" + SELECT_BATCH_COLS = "id::text, start_seq, end_seq, merkle_root, signature, signing_key_version, sealed_at" + def initialize(@db : DB::Database) end def append(entry : AuditEntry) : Nil @db.exec( - "INSERT INTO audit_events (event_id, occurred_at, event_type, actor, target_id, payload, prev_hash, content_hash, hmac, hmac_key_version) VALUES ($1::uuid, $2, $3, $4, $5::uuid, $6::jsonb, $7, $8, $9, $10) ON CONFLICT (event_id) DO NOTHING", + "INSERT INTO audit_events (event_id, occurred_at, event_type, actor, target_id, payload, prev_hash, content_hash, hmac, hmac_key_version) VALUES ($1::uuid, $2, $3, $4, $5::uuid, $6::jsonb, $7, $8, $9, $10)", entry.event_id.to_s, entry.occurred_at, entry.event_type, entry.actor, entry.target_id.try(&.to_s), @@ -44,12 +47,36 @@ module CRE::Persistence::Postgres def range(start_seq : Int64, end_seq : Int64) : Array(AuditEntry) @db.query_all( - "SELECT seq, event_id::text, occurred_at, event_type, actor, target_id::text, payload::text, prev_hash, content_hash, hmac, hmac_key_version FROM audit_events WHERE seq >= $1 AND seq <= $2 ORDER BY seq ASC", + "SELECT #{SELECT_ENTRY_COLS} FROM audit_events WHERE seq >= $1 AND seq <= $2 ORDER BY seq ASC", start_seq, end_seq, as: {Int64, String, Time, String, String, String?, String, Bytes, Bytes, Bytes, Int32}, ).map { |row| row_to_entry(row) } end + def each_in_range(start_seq : Int64, end_seq : Int64, &block : AuditEntry ->) : Nil + @db.query( + "SELECT #{SELECT_ENTRY_COLS} FROM audit_events WHERE seq >= $1 AND seq <= $2 ORDER BY seq ASC", + start_seq, end_seq, + ) do |rs| + rs.each do + entry = AuditEntry.new( + seq: rs.read(Int64), + event_id: UUID.new(rs.read(String)), + occurred_at: rs.read(Time), + event_type: rs.read(String), + actor: rs.read(String), + target_id: rs.read(String?).try { |s| UUID.new(s) }, + payload: rs.read(String), + prev_hash: rs.read(Bytes), + content_hash: rs.read(Bytes), + hmac: rs.read(Bytes), + hmac_key_version: rs.read(Int32), + ) + block.call(entry) + end + end + end + def insert_batch(batch : AuditBatch) : Nil @db.exec( "INSERT INTO audit_batches (id, start_seq, end_seq, merkle_root, signature, signing_key_version, sealed_at) VALUES ($1::uuid, $2, $3, $4, $5, $6, $7)", @@ -67,6 +94,13 @@ module CRE::Persistence::Postgres result || 0_i64 end + def all_batches : Array(AuditBatch) + @db.query_all( + "SELECT #{SELECT_BATCH_COLS} FROM audit_batches ORDER BY start_seq ASC", + as: {String, Int64, Int64, Bytes, Bytes, Int32, Time}, + ).map { |row| row_to_batch(row) } + end + private def row_to_entry(row) : AuditEntry AuditEntry.new( seq: row[0], @@ -82,5 +116,17 @@ module CRE::Persistence::Postgres hmac_key_version: row[10], ) end + + private def row_to_batch(row) : AuditBatch + AuditBatch.new( + id: UUID.new(row[0]), + start_seq: row[1], + end_seq: row[2], + merkle_root: row[3], + signature: row[4], + signing_key_version: row[5], + sealed_at: row[6], + ) + end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/credentials_repo.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/credentials_repo.cr index 315a1297..2e4186ab 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/credentials_repo.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/credentials_repo.cr @@ -11,25 +11,31 @@ require "../../domain/credential" module CRE::Persistence::Postgres class CredentialsRepo < CRE::Persistence::CredentialsRepo - SELECT_COLS = "id::text, external_id, kind, name, tags::text, current_version_id::text, pending_version_id::text, previous_version_id::text, created_at, updated_at" + SELECT_COLS = "id::text, external_id, kind, name, tags::text, current_version_id::text, pending_version_id::text, previous_version_id::text, created_at, updated_at, last_rotated_at" def initialize(@db : DB::Database) end def insert(c : Domain::Credential) : Nil @db.exec( - "INSERT INTO credentials (id, external_id, kind, name, tags, created_at, updated_at) VALUES ($1::uuid, $2, $3, $4, $5::jsonb, $6, $7)", + "INSERT INTO credentials (id, external_id, kind, name, tags, current_version_id, pending_version_id, previous_version_id, created_at, updated_at, last_rotated_at) VALUES ($1::uuid, $2, $3, $4, $5::jsonb, $6::uuid, $7::uuid, $8::uuid, $9, $10, $11)", c.id.to_s, c.external_id, c.kind.to_s, c.name, - c.tags.to_json, c.created_at, c.updated_at, + c.tags.to_json, + c.current_version_id.try(&.to_s), + c.pending_version_id.try(&.to_s), + c.previous_version_id.try(&.to_s), + c.created_at, c.updated_at, c.last_rotated_at, ) end def update(c : Domain::Credential) : Nil @db.exec( - "UPDATE credentials SET name = $1, tags = $2::jsonb, current_version_id = $3::uuid, pending_version_id = $4::uuid, previous_version_id = $5::uuid, updated_at = $6 WHERE id = $7::uuid", + "UPDATE credentials SET name = $1, tags = $2::jsonb, current_version_id = $3::uuid, pending_version_id = $4::uuid, previous_version_id = $5::uuid, updated_at = $6, last_rotated_at = $7 WHERE id = $8::uuid", c.name, c.tags.to_json, - c.current_version_id.try(&.to_s), c.pending_version_id.try(&.to_s), c.previous_version_id.try(&.to_s), - Time.utc, c.id.to_s, + c.current_version_id.try(&.to_s), + c.pending_version_id.try(&.to_s), + c.previous_version_id.try(&.to_s), + Time.utc, c.last_rotated_at, c.id.to_s, ) end @@ -37,7 +43,7 @@ module CRE::Persistence::Postgres @db.query_one?( "SELECT #{SELECT_COLS} FROM credentials WHERE id = $1::uuid", id.to_s, - as: {String, String, String, String, String, String?, String?, String?, Time, Time}, + as: {String, String, String, String, String, String?, String?, String?, Time, Time, Time?}, ).try { |row| row_to_credential(row) } end @@ -45,23 +51,23 @@ module CRE::Persistence::Postgres @db.query_one?( "SELECT #{SELECT_COLS} FROM credentials WHERE kind = $1 AND external_id = $2", kind.to_s, external_id, - as: {String, String, String, String, String, String?, String?, String?, Time, Time}, + as: {String, String, String, String, String, String?, String?, String?, Time, Time, Time?}, ).try { |row| row_to_credential(row) } end def all : Array(Domain::Credential) @db.query_all( "SELECT #{SELECT_COLS} FROM credentials", - as: {String, String, String, String, String, String?, String?, String?, Time, Time}, + as: {String, String, String, String, String, String?, String?, String?, Time, Time, Time?}, ).map { |row| row_to_credential(row) } end def overdue(now : Time, max_age : Time::Span) : Array(Domain::Credential) cutoff = now - max_age @db.query_all( - "SELECT #{SELECT_COLS} FROM credentials WHERE updated_at < $1", + "SELECT #{SELECT_COLS} FROM credentials WHERE COALESCE(last_rotated_at, created_at) < $1", cutoff, - as: {String, String, String, String, String, String?, String?, String?, Time, Time}, + as: {String, String, String, String, String, String?, String?, String?, Time, Time, Time?}, ).map { |row| row_to_credential(row) } end @@ -78,6 +84,7 @@ module CRE::Persistence::Postgres previous_version_id: row[7].try { |s| UUID.new(s) }, created_at: row[8], updated_at: row[9], + last_rotated_at: row[10], ) end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/migrations.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/migrations.cr index 678e8b92..12fcd683 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/migrations.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/migrations.cr @@ -7,105 +7,134 @@ require "db" module CRE::Persistence::Postgres module Migrations - SCHEMA = [ - <<-SQL, - CREATE TABLE IF NOT EXISTS credentials ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - external_id TEXT NOT NULL, - kind TEXT NOT NULL, - name TEXT NOT NULL, - tags JSONB NOT NULL DEFAULT '{}'::jsonb, - current_version_id UUID, - pending_version_id UUID, - previous_version_id UUID, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - UNIQUE (kind, external_id) - ) - SQL - "CREATE INDEX IF NOT EXISTS credentials_tags_gin ON credentials USING gin (tags jsonb_path_ops)", - <<-SQL, - CREATE TABLE IF NOT EXISTS credential_versions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - credential_id UUID NOT NULL REFERENCES credentials(id), - ciphertext BYTEA NOT NULL, - dek_wrapped BYTEA NOT NULL, - kek_version INT NOT NULL, - algorithm_id SMALLINT NOT NULL, - metadata JSONB NOT NULL DEFAULT '{}'::jsonb, - generated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - revoked_at TIMESTAMPTZ - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS rotations ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - credential_id UUID NOT NULL REFERENCES credentials(id), - rotator_kind TEXT NOT NULL, - state TEXT NOT NULL, - started_at TIMESTAMPTZ NOT NULL DEFAULT now(), - completed_at TIMESTAMPTZ, - step_outcomes JSONB NOT NULL DEFAULT '{}'::jsonb, - failure_reason TEXT - ) - SQL - <<-SQL, - CREATE INDEX IF NOT EXISTS rotations_in_flight - ON rotations(state) - WHERE state NOT IN ('completed','failed','aborted') - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS audit_events ( - seq BIGSERIAL PRIMARY KEY, - event_id UUID UNIQUE NOT NULL DEFAULT gen_random_uuid(), - occurred_at TIMESTAMPTZ NOT NULL DEFAULT now(), - event_type TEXT NOT NULL, - actor TEXT NOT NULL, - target_id UUID, - payload JSONB NOT NULL, - prev_hash BYTEA NOT NULL, - content_hash BYTEA NOT NULL, - hmac BYTEA NOT NULL, - hmac_key_version INT NOT NULL - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS audit_batches ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - start_seq BIGINT NOT NULL, - end_seq BIGINT NOT NULL, - merkle_root BYTEA NOT NULL, - signature BYTEA NOT NULL, - signing_key_version INT NOT NULL, - sealed_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS kek_versions ( - version INT PRIMARY KEY, - source TEXT NOT NULL, - source_ref TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - retired_at TIMESTAMPTZ - ) - SQL - <<-SQL, - CREATE OR REPLACE FUNCTION audit_no_modify() RETURNS trigger LANGUAGE plpgsql AS $$ - BEGIN RAISE EXCEPTION 'audit_events is append-only'; END $$ - SQL - <<-SQL, - DROP TRIGGER IF EXISTS audit_events_no_update ON audit_events - SQL - <<-SQL, - CREATE TRIGGER audit_events_no_update - BEFORE UPDATE OR DELETE OR TRUNCATE - ON audit_events - EXECUTE FUNCTION audit_no_modify() - SQL + record Step, version : Int32, statements : Array(String) + + MIGRATIONS = [ + Step.new(1, [ + <<-SQL, + CREATE TABLE IF NOT EXISTS credentials ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_id TEXT NOT NULL, + kind TEXT NOT NULL, + name TEXT NOT NULL, + tags JSONB NOT NULL DEFAULT '{}'::jsonb, + current_version_id UUID, + pending_version_id UUID, + previous_version_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (kind, external_id) + ) + SQL + "CREATE INDEX IF NOT EXISTS credentials_tags_gin ON credentials USING gin (tags jsonb_path_ops)", + <<-SQL, + CREATE TABLE IF NOT EXISTS credential_versions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + credential_id UUID NOT NULL REFERENCES credentials(id), + ciphertext BYTEA NOT NULL, + dek_wrapped BYTEA NOT NULL, + kek_version INT NOT NULL, + algorithm_id SMALLINT NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + generated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + revoked_at TIMESTAMPTZ + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS rotations ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + credential_id UUID NOT NULL REFERENCES credentials(id), + rotator_kind TEXT NOT NULL, + state TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + step_outcomes JSONB NOT NULL DEFAULT '{}'::jsonb, + failure_reason TEXT + ) + SQL + <<-SQL, + CREATE INDEX IF NOT EXISTS rotations_in_flight + ON rotations(state) + WHERE state NOT IN ('completed','failed','aborted','inconsistent') + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS audit_events ( + seq BIGSERIAL PRIMARY KEY, + event_id UUID UNIQUE NOT NULL DEFAULT gen_random_uuid(), + occurred_at TIMESTAMPTZ NOT NULL DEFAULT now(), + event_type TEXT NOT NULL, + actor TEXT NOT NULL, + target_id UUID, + payload JSONB NOT NULL, + prev_hash BYTEA NOT NULL, + content_hash BYTEA NOT NULL, + hmac BYTEA NOT NULL, + hmac_key_version INT NOT NULL + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS audit_batches ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + start_seq BIGINT NOT NULL, + end_seq BIGINT NOT NULL, + merkle_root BYTEA NOT NULL, + signature BYTEA NOT NULL, + signing_key_version INT NOT NULL, + sealed_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS kek_versions ( + version INT PRIMARY KEY, + source TEXT NOT NULL, + source_ref TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + retired_at TIMESTAMPTZ + ) + SQL + <<-SQL, + CREATE OR REPLACE FUNCTION audit_no_modify() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'audit_events is append-only'; END $$ + SQL + "DROP TRIGGER IF EXISTS audit_events_no_update ON audit_events", + <<-SQL, + CREATE TRIGGER audit_events_no_update + BEFORE UPDATE OR DELETE OR TRUNCATE + ON audit_events + EXECUTE FUNCTION audit_no_modify() + SQL + ]), + Step.new(2, [ + "ALTER TABLE credentials ADD COLUMN IF NOT EXISTS last_rotated_at TIMESTAMPTZ", + "UPDATE credentials SET last_rotated_at = updated_at WHERE last_rotated_at IS NULL", + ]), + Step.new(3, [ + "CREATE INDEX IF NOT EXISTS credentials_last_rotated_at ON credentials(last_rotated_at)", + "CREATE INDEX IF NOT EXISTS credential_versions_credential_id ON credential_versions(credential_id)", + <<-SQL, + CREATE UNIQUE INDEX IF NOT EXISTS rotations_one_in_flight_per_cred + ON rotations(credential_id) + WHERE state NOT IN ('completed','failed','aborted','inconsistent') + SQL + ]), ] def self.run(db : DB::Database) : Nil - SCHEMA.each { |stmt| db.exec(stmt) } + db.exec("CREATE TABLE IF NOT EXISTS schema_migrations (version INT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())") + applied = applied_versions(db) + MIGRATIONS.each do |step| + next if applied.includes?(step.version) + step.statements.each { |stmt| db.exec(stmt) } + db.exec("INSERT INTO schema_migrations (version) VALUES ($1)", step.version) + end + end + + private def self.applied_versions(db : DB::Database) : Set(Int32) + versions = Set(Int32).new + db.query("SELECT version FROM schema_migrations") do |rs| + rs.each { versions << rs.read(Int32) } + end + versions end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/rotations_repo.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/rotations_repo.cr index a7488889..8380e503 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/rotations_repo.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/postgres/rotations_repo.cr @@ -33,7 +33,7 @@ module CRE::Persistence::Postgres def in_flight : Array(RotationRecord) @db.query_all( - "SELECT #{SELECT_COLS} FROM rotations WHERE state NOT IN ('completed','failed','aborted')", + "SELECT #{SELECT_COLS} FROM rotations WHERE state NOT IN ('completed','failed','aborted','inconsistent')", as: {String, String, String, String, Time, Time?, String?}, ).map { |row| row_to_record(row) } end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/repos.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/repos.cr index 016753b2..ea0bfaa9 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/repos.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/repos.cr @@ -27,7 +27,7 @@ module CRE::Persistence Inconsistent end - TERMINAL_STATES = [RotationState::Completed, RotationState::Failed, RotationState::Aborted] + TERMINAL_STATES = [RotationState::Completed, RotationState::Failed, RotationState::Aborted, RotationState::Inconsistent] record RotationRecord, id : UUID, @@ -87,7 +87,9 @@ module CRE::Persistence abstract def latest_hash : Bytes abstract def latest_seq : Int64 abstract def range(start_seq : Int64, end_seq : Int64) : Array(AuditEntry) + abstract def each_in_range(start_seq : Int64, end_seq : Int64, &block : AuditEntry ->) : Nil abstract def insert_batch(batch : AuditBatch) : Nil abstract def last_sealed_seq : Int64 + abstract def all_batches : Array(AuditBatch) end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/audit_repo.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/audit_repo.cr index c2d38041..a470e9f7 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/audit_repo.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/audit_repo.cr @@ -11,12 +11,15 @@ module CRE::Persistence::Sqlite class AuditRepo < CRE::Persistence::AuditRepo GENESIS_HASH = Bytes.new(32, 0_u8) + SELECT_ENTRY_COLS = "seq, event_id, occurred_at, event_type, actor, target_id, payload, prev_hash, content_hash, hmac, hmac_key_version" + SELECT_BATCH_COLS = "id, start_seq, end_seq, merkle_root, signature, signing_key_version, sealed_at" + def initialize(@db : DB::Database) end def append(entry : AuditEntry) : Nil @db.exec( - "INSERT OR IGNORE INTO audit_events (event_id, occurred_at, event_type, actor, target_id, payload, prev_hash, content_hash, hmac, hmac_key_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO audit_events (event_id, occurred_at, event_type, actor, target_id, payload, prev_hash, content_hash, hmac, hmac_key_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", entry.event_id.to_s, entry.occurred_at.to_rfc3339, entry.event_type, entry.actor, entry.target_id.try(&.to_s), @@ -44,12 +47,36 @@ module CRE::Persistence::Sqlite def range(start_seq : Int64, end_seq : Int64) : Array(AuditEntry) @db.query_all( - "SELECT seq, event_id, occurred_at, event_type, actor, target_id, payload, prev_hash, content_hash, hmac, hmac_key_version FROM audit_events WHERE seq >= ? AND seq <= ? ORDER BY seq ASC", + "SELECT #{SELECT_ENTRY_COLS} FROM audit_events WHERE seq >= ? AND seq <= ? ORDER BY seq ASC", start_seq, end_seq, as: {Int64, String, String, String, String, String?, String, Bytes, Bytes, Bytes, Int32}, ).map { |row| row_to_entry(row) } end + def each_in_range(start_seq : Int64, end_seq : Int64, &block : AuditEntry ->) : Nil + @db.query( + "SELECT #{SELECT_ENTRY_COLS} FROM audit_events WHERE seq >= ? AND seq <= ? ORDER BY seq ASC", + start_seq, end_seq, + ) do |rs| + rs.each do + entry = AuditEntry.new( + seq: rs.read(Int64), + event_id: UUID.new(rs.read(String)), + occurred_at: Time.parse_rfc3339(rs.read(String)), + event_type: rs.read(String), + actor: rs.read(String), + target_id: rs.read(String?).try { |s| UUID.new(s) }, + payload: rs.read(String), + prev_hash: rs.read(Bytes), + content_hash: rs.read(Bytes), + hmac: rs.read(Bytes), + hmac_key_version: rs.read(Int32), + ) + block.call(entry) + end + end + end + def insert_batch(batch : AuditBatch) : Nil @db.exec( "INSERT INTO audit_batches (id, start_seq, end_seq, merkle_root, signature, signing_key_version, sealed_at) VALUES (?, ?, ?, ?, ?, ?, ?)", @@ -67,6 +94,13 @@ module CRE::Persistence::Sqlite result || 0_i64 end + def all_batches : Array(AuditBatch) + @db.query_all( + "SELECT #{SELECT_BATCH_COLS} FROM audit_batches ORDER BY start_seq ASC", + as: {String, Int64, Int64, Bytes, Bytes, Int32, String}, + ).map { |row| row_to_batch(row) } + end + private def row_to_entry(row) : AuditEntry AuditEntry.new( seq: row[0], @@ -82,5 +116,17 @@ module CRE::Persistence::Sqlite hmac_key_version: row[10], ) end + + private def row_to_batch(row) : AuditBatch + AuditBatch.new( + id: UUID.new(row[0]), + start_seq: row[1], + end_seq: row[2], + merkle_root: row[3], + signature: row[4], + signing_key_version: row[5], + sealed_at: Time.parse_rfc3339(row[6]), + ) + end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/credentials_repo.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/credentials_repo.cr index ac7d4229..96f1aa4c 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/credentials_repo.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/credentials_repo.cr @@ -11,25 +11,34 @@ require "../../domain/credential" module CRE::Persistence::Sqlite class CredentialsRepo < CRE::Persistence::CredentialsRepo - SELECT_COLS = "id, external_id, kind, name, tags, current_version_id, pending_version_id, previous_version_id, created_at, updated_at" + SELECT_COLS = "id, external_id, kind, name, tags, current_version_id, pending_version_id, previous_version_id, created_at, updated_at, last_rotated_at" def initialize(@db : DB::Database) end def insert(c : Domain::Credential) : Nil @db.exec( - "INSERT INTO credentials (id, external_id, kind, name, tags, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO credentials (id, external_id, kind, name, tags, current_version_id, pending_version_id, previous_version_id, created_at, updated_at, last_rotated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", c.id.to_s, c.external_id, c.kind.to_s, c.name, - c.tags.to_json, c.created_at.to_rfc3339, c.updated_at.to_rfc3339, + c.tags.to_json, + c.current_version_id.try(&.to_s), + c.pending_version_id.try(&.to_s), + c.previous_version_id.try(&.to_s), + c.created_at.to_rfc3339, c.updated_at.to_rfc3339, + c.last_rotated_at.try(&.to_rfc3339), ) end def update(c : Domain::Credential) : Nil @db.exec( - "UPDATE credentials SET name = ?, tags = ?, current_version_id = ?, pending_version_id = ?, previous_version_id = ?, updated_at = ? WHERE id = ?", + "UPDATE credentials SET name = ?, tags = ?, current_version_id = ?, pending_version_id = ?, previous_version_id = ?, updated_at = ?, last_rotated_at = ? WHERE id = ?", c.name, c.tags.to_json, - c.current_version_id.try(&.to_s), c.pending_version_id.try(&.to_s), c.previous_version_id.try(&.to_s), - Time.utc.to_rfc3339, c.id.to_s, + c.current_version_id.try(&.to_s), + c.pending_version_id.try(&.to_s), + c.previous_version_id.try(&.to_s), + Time.utc.to_rfc3339, + c.last_rotated_at.try(&.to_rfc3339), + c.id.to_s, ) end @@ -37,7 +46,7 @@ module CRE::Persistence::Sqlite @db.query_one?( "SELECT #{SELECT_COLS} FROM credentials WHERE id = ?", id.to_s, - as: {String, String, String, String, String, String?, String?, String?, String, String}, + as: {String, String, String, String, String, String?, String?, String?, String, String, String?}, ).try { |row| row_to_credential(row) } end @@ -45,23 +54,23 @@ module CRE::Persistence::Sqlite @db.query_one?( "SELECT #{SELECT_COLS} FROM credentials WHERE kind = ? AND external_id = ?", kind.to_s, external_id, - as: {String, String, String, String, String, String?, String?, String?, String, String}, + as: {String, String, String, String, String, String?, String?, String?, String, String, String?}, ).try { |row| row_to_credential(row) } end def all : Array(Domain::Credential) @db.query_all( "SELECT #{SELECT_COLS} FROM credentials", - as: {String, String, String, String, String, String?, String?, String?, String, String}, + as: {String, String, String, String, String, String?, String?, String?, String, String, String?}, ).map { |row| row_to_credential(row) } end def overdue(now : Time, max_age : Time::Span) : Array(Domain::Credential) cutoff = (now - max_age).to_rfc3339 @db.query_all( - "SELECT #{SELECT_COLS} FROM credentials WHERE updated_at < ?", + "SELECT #{SELECT_COLS} FROM credentials WHERE COALESCE(last_rotated_at, created_at) < ?", cutoff, - as: {String, String, String, String, String, String?, String?, String?, String, String}, + as: {String, String, String, String, String, String?, String?, String?, String, String, String?}, ).map { |row| row_to_credential(row) } end @@ -78,6 +87,7 @@ module CRE::Persistence::Sqlite previous_version_id: row[7].try { |s| UUID.new(s) }, created_at: Time.parse_rfc3339(row[8]), updated_at: Time.parse_rfc3339(row[9]), + last_rotated_at: row[10].try { |s| Time.parse_rfc3339(s) }, ) end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/migrations.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/migrations.cr index 78e681fd..84d25aac 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/migrations.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/persistence/sqlite/migrations.cr @@ -7,86 +7,124 @@ require "db" module CRE::Persistence::Sqlite module Migrations - SCHEMA = [ - <<-SQL, - CREATE TABLE IF NOT EXISTS credentials ( - id TEXT PRIMARY KEY, - external_id TEXT NOT NULL, - kind TEXT NOT NULL, - name TEXT NOT NULL, - tags TEXT NOT NULL DEFAULT '{}', - current_version_id TEXT, - pending_version_id TEXT, - previous_version_id TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - UNIQUE (kind, external_id) - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS credential_versions ( - id TEXT PRIMARY KEY, - credential_id TEXT NOT NULL REFERENCES credentials(id), - ciphertext BLOB NOT NULL, - dek_wrapped BLOB NOT NULL, - kek_version INTEGER NOT NULL, - algorithm_id INTEGER NOT NULL, - metadata TEXT NOT NULL DEFAULT '{}', - generated_at TEXT NOT NULL, - revoked_at TEXT - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS rotations ( - id TEXT PRIMARY KEY, - credential_id TEXT NOT NULL, - rotator_kind TEXT NOT NULL, - state TEXT NOT NULL, - started_at TEXT NOT NULL, - completed_at TEXT, - step_outcomes TEXT NOT NULL DEFAULT '{}', - failure_reason TEXT - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS audit_events ( - seq INTEGER PRIMARY KEY AUTOINCREMENT, - event_id TEXT UNIQUE NOT NULL, - occurred_at TEXT NOT NULL, - event_type TEXT NOT NULL, - actor TEXT NOT NULL, - target_id TEXT, - payload TEXT NOT NULL, - prev_hash BLOB NOT NULL, - content_hash BLOB NOT NULL, - hmac BLOB NOT NULL, - hmac_key_version INTEGER NOT NULL - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS audit_batches ( - id TEXT PRIMARY KEY, - start_seq INTEGER NOT NULL, - end_seq INTEGER NOT NULL, - merkle_root BLOB NOT NULL, - signature BLOB NOT NULL, - signing_key_version INTEGER NOT NULL, - sealed_at TEXT NOT NULL - ) - SQL - <<-SQL, - CREATE TABLE IF NOT EXISTS kek_versions ( - version INTEGER PRIMARY KEY, - source TEXT NOT NULL, - source_ref TEXT, - created_at TEXT NOT NULL, - retired_at TEXT - ) - SQL + record Step, version : Int32, statements : Array(String) + + MIGRATIONS = [ + Step.new(1, [ + <<-SQL, + CREATE TABLE IF NOT EXISTS credentials ( + id TEXT PRIMARY KEY, + external_id TEXT NOT NULL, + kind TEXT NOT NULL, + name TEXT NOT NULL, + tags TEXT NOT NULL DEFAULT '{}', + current_version_id TEXT, + pending_version_id TEXT, + previous_version_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (kind, external_id) + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS credential_versions ( + id TEXT PRIMARY KEY, + credential_id TEXT NOT NULL REFERENCES credentials(id), + ciphertext BLOB NOT NULL, + dek_wrapped BLOB NOT NULL, + kek_version INTEGER NOT NULL, + algorithm_id INTEGER NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}', + generated_at TEXT NOT NULL, + revoked_at TEXT + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS rotations ( + id TEXT PRIMARY KEY, + credential_id TEXT NOT NULL, + rotator_kind TEXT NOT NULL, + state TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + step_outcomes TEXT NOT NULL DEFAULT '{}', + failure_reason TEXT + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS audit_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT UNIQUE NOT NULL, + occurred_at TEXT NOT NULL, + event_type TEXT NOT NULL, + actor TEXT NOT NULL, + target_id TEXT, + payload TEXT NOT NULL, + prev_hash BLOB NOT NULL, + content_hash BLOB NOT NULL, + hmac BLOB NOT NULL, + hmac_key_version INTEGER NOT NULL + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS audit_batches ( + id TEXT PRIMARY KEY, + start_seq INTEGER NOT NULL, + end_seq INTEGER NOT NULL, + merkle_root BLOB NOT NULL, + signature BLOB NOT NULL, + signing_key_version INTEGER NOT NULL, + sealed_at TEXT NOT NULL + ) + SQL + <<-SQL, + CREATE TABLE IF NOT EXISTS kek_versions ( + version INTEGER PRIMARY KEY, + source TEXT NOT NULL, + source_ref TEXT, + created_at TEXT NOT NULL, + retired_at TEXT + ) + SQL + ]), + Step.new(2, [ + <<-SQL, + CREATE TRIGGER IF NOT EXISTS audit_events_no_update + BEFORE UPDATE ON audit_events + BEGIN SELECT RAISE(ABORT, 'audit_events is append-only'); END + SQL + <<-SQL, + CREATE TRIGGER IF NOT EXISTS audit_events_no_delete + BEFORE DELETE ON audit_events + BEGIN SELECT RAISE(ABORT, 'audit_events is append-only'); END + SQL + ]), + Step.new(3, [ + "ALTER TABLE credentials ADD COLUMN last_rotated_at TEXT", + "UPDATE credentials SET last_rotated_at = updated_at WHERE last_rotated_at IS NULL", + ]), + Step.new(4, [ + "CREATE INDEX IF NOT EXISTS credentials_last_rotated_at ON credentials(last_rotated_at)", + "CREATE INDEX IF NOT EXISTS credential_versions_credential_id ON credential_versions(credential_id)", + ]), ] def self.run(db : DB::Database) : Nil - SCHEMA.each { |stmt| db.exec(stmt) } + db.exec("CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)") + applied = applied_versions(db) + MIGRATIONS.each do |step| + next if applied.includes?(step.version) + step.statements.each { |stmt| db.exec(stmt) } + db.exec("INSERT INTO schema_migrations (version) VALUES (?)", step.version) + end + end + + private def self.applied_versions(db : DB::Database) : Set(Int32) + versions = Set(Int32).new + db.query("SELECT version FROM schema_migrations") do |rs| + rs.each { versions << rs.read(Int32) } + end + versions end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/dsl.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/dsl.cr index 3cd3e9fd..ff7c4b71 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/dsl.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/dsl.cr @@ -6,24 +6,28 @@ require "./builder" require "./policy" -# Top-level `policy` method makes the DSL feel native: -# -# require "cre/policy/dsl" -# -# policy "production-databases" do -# description "All prod DB credentials rotate every 30 days" -# match { |c| c.kind.database? && c.tag(:env) == "prod" } -# max_age 30.days -# enforce :rotate_immediately -# notify_via :telegram, :structured_log -# end -# -# The `with builder yield` makes every Builder method (description, match, -# max_age, enforce, notify_via, on_rotation_failure, on_drift_detected) callable -# without a receiver inside the block. Symbol literals autocast to enum values -# so typos like `enforce :rotate_immediatly` fail at compile time. -def policy(name : String, &block) - builder = CRE::Policy::Builder.new(name) - with builder yield - CRE::Policy::REGISTRY << builder.build +module CRE::Policy::Dsl + # Top-level-feeling DSL for declaring policies. Users opt in: + # + # require "cre/policy/dsl" + # include CRE::Policy::Dsl + # + # policy "production-aws-secrets" do + # description "All prod AWS secrets rotate every 30 days" + # match { |c| c.kind.aws_secretsmgr? && c.tag(:env) == "prod" } + # max_age 30.days + # enforce :rotate_immediately + # notify_via :telegram, :structured_log + # end + # + # Single-symbol args (`enforce :rotate_immediately`) autocast to enum + # values via Crystal's parameter typing — typos like + # `enforce :rotate_immediatly` fail at compile time. Splat-symbol args + # (`notify_via :telegram, :structured_log`) are validated at policy + # registration time and raise `BuilderError` on typos. + def policy(name : String, &block) + builder = CRE::Policy::Builder.new(name) + with builder yield + CRE::Policy::REGISTRY << builder.build + end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/evaluator.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/evaluator.cr index 0ef4522e..11cc25cc 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/evaluator.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/evaluator.cr @@ -11,6 +11,8 @@ require "../events/system_events" require "../persistence/persistence" module CRE::Policy + class PolicyConflictError < Exception; end + class Evaluator Log = ::Log.for("cre.policy.evaluator") @@ -46,22 +48,45 @@ module CRE::Policy @ch.try(&.close) end + # evaluate_all groups policies by max_age and uses the persistence layer's + # 'overdue' query so we don't pull every credential into memory each tick. def evaluate_all(now : Time = Time.utc) : Nil - @persistence.credentials.all.each { |c| evaluate(c, now) } + return if @policies.empty? + + seen = Set(UUID).new + @policies.group_by(&.max_age).each do |max_age, policies_with_age| + @persistence.credentials.overdue(now, max_age).each do |c| + next if seen.includes?(c.id) + seen << c.id + evaluate(c, now) + end + end end + # Evaluates a single credential. When more than one policy matches + # we raise PolicyConflictError instead of silently picking by + # registration order; conflicting policies are an operator bug, not + # a runtime decision. def evaluate(c : Domain::Credential, now : Time = Time.utc) : Nil matching = @policies.select(&.matches?(c)) return if matching.empty? - # Most specific match wins on conflicts (last-match wins as a simple tiebreaker) - policy = matching.last + if matching.size > 1 + names = matching.map(&.name).join(", ") + @bus.publish Events::AlertRaised.new( + severity: Events::Severity::Critical, + message: "credential #{c.id} matches #{matching.size} policies (#{names}); refusing to act until ambiguity is resolved", + ) + raise PolicyConflictError.new("credential #{c.id} matches #{matching.size} policies: #{names}") + end + + policy = matching.first return unless policy.overdue?(c, now) @bus.publish Events::PolicyViolation.new( c.id, policy.name, - "credential exceeded max_age=#{policy.max_age} (last update #{c.updated_at.to_rfc3339})", + "credential exceeded max_age=#{policy.max_age} (rotation_anchor #{c.rotation_anchor.to_rfc3339})", ) case policy.enforce_action @@ -78,6 +103,8 @@ module CRE::Policy message: "policy '#{policy.name}' triggered quarantine on credential '#{c.id}'", ) end + rescue ex : PolicyConflictError + Log.error(exception: ex) { "policy conflict for #{c.id}" } rescue ex Log.error(exception: ex) { "policy evaluation failed for #{c.id}" } end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/policy.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/policy.cr index b6f78450..a8e75e6d 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/policy.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/policy/policy.cr @@ -54,12 +54,12 @@ module CRE::Policy end def overdue?(c : Domain::Credential, now : Time = Time.utc) : Bool - (now - c.updated_at) > @max_age + (now - c.rotation_anchor) > @max_age end def in_warning_window?(c : Domain::Credential, now : Time = Time.utc) : Bool return false unless w = @warn_at - age = now - c.updated_at + age = now - c.rotation_anchor age > w && age <= @max_age end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/rotators/env_file.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/rotators/env_file.cr index e349ea17..e33a015a 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/rotators/env_file.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/rotators/env_file.cr @@ -11,6 +11,12 @@ module CRE::Rotators # The rotation produces fresh random bytes (base64-encoded) and atomically # swaps the live file on commit using temp+rename. # + # Cross-process safety: each instance writes to a per-PID pending file + # so two daemons targeting the same .env never collide on the staging + # write. The commit-time rename is serialized through an exclusive + # flock(2) on a sibling .lock file so the live file is updated by at + # most one process at a time. + # # Credential.tags must include: # "path" - absolute path to the .env file # "key" - the key whose value rotates @@ -41,23 +47,23 @@ module CRE::Rotators def apply(c : Domain::Credential, s : Domain::NewSecret) : Nil path = c.tag("path").not_nil! key = c.tag("key").not_nil! - pending_path = "#{path}.pending" + pending = pending_path(path) existing = File.exists?(path) ? File.read(path) : "" lines = existing.lines(chomp: true).reject { |line| line.strip.starts_with?("#{key}=") } new_value = String.new(s.ciphertext) lines << "#{key}=#{new_value}" - File.write(pending_path, lines.join('\n') + "\n", perm: 0o600) + File.write(pending, lines.join('\n') + "\n", perm: 0o600) end def verify(c : Domain::Credential, s : Domain::NewSecret) : Bool path = c.tag("path").not_nil! key = c.tag("key").not_nil! - pending_path = "#{path}.pending" - return false unless File.exists?(pending_path) + pending = pending_path(path) + return false unless File.exists?(pending) - content = File.read(pending_path) + content = File.read(pending) expected_line = "#{key}=#{String.new(s.ciphertext)}" content.includes?(expected_line) && content.bytesize > 0 end @@ -65,16 +71,30 @@ module CRE::Rotators def commit(c : Domain::Credential, s : Domain::NewSecret) : Nil _ = s path = c.tag("path").not_nil! - pending_path = "#{path}.pending" - raise RotatorError.new("pending file missing at commit time: #{pending_path}") unless File.exists?(pending_path) - File.rename(pending_path, path) + pending = pending_path(path) + raise RotatorError.new("pending file missing at commit time: #{pending}") unless File.exists?(pending) + + with_lock(path) do + File.rename(pending, path) + end end def rollback_apply(c : Domain::Credential, s : Domain::NewSecret) : Nil _ = s path = c.tag("path").not_nil! - pending_path = "#{path}.pending" - File.delete(pending_path) if File.exists?(pending_path) + pending = pending_path(path) + File.delete(pending) if File.exists?(pending) + end + + private def pending_path(path : String) : String + "#{path}.pending.#{Process.pid}" + end + + private def with_lock(path : String, & : -> _) : Nil + lock_path = "#{path}.lock" + File.open(lock_path, "w+") do |lock| + lock.flock_exclusive { yield } + end end end end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/renderer.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/renderer.cr index cc6bc628..68afb463 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/renderer.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/renderer.cr @@ -45,12 +45,16 @@ module CRE::Tui private def render_status : Nil header = " STATUS CREDS DUE-NOW OVERDUE ROTATED-24h KEK" - values = " #{Ansi.green("● live")} ? ? ? #{@state.completed_24h.to_s.ljust(13)} v#{@state.kek_version}" + values = " #{Ansi.green("● live")} #{cell(@state.creds_total, 8)}#{cell(@state.due_now, 11)}#{cell(@state.overdue, 11)}#{cell(@state.completed_24h, 15)}v#{@state.kek_version}" @io << PANEL_VL << pad(header, WIDTH - 2) << PANEL_VL << '\n' @io << PANEL_VL << pad(values, WIDTH - 2) << PANEL_VL << '\n' @io << PANEL_LMID << " Active Rotations " << PANEL_HR * (WIDTH - 21) << PANEL_RMID << '\n' end + private def cell(n : Int, width : Int) : String + n.to_s.ljust(width) + end + private def render_active : Nil if @state.active.empty? @io << PANEL_VL << pad(" (no active rotations)", WIDTH - 2) << PANEL_VL << '\n' @@ -78,7 +82,7 @@ module CRE::Tui private def render_footer : Nil @io << PANEL_BL << PANEL_HR * (WIDTH - 2) << PANEL_BR << '\n' - @io << Ansi.dim(" q=quit r=refresh ?=help") << '\n' + @io << Ansi.dim(" Press Ctrl+C to exit") << '\n' end private def colorize(text : String, color : String) : String diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/snapshotter.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/snapshotter.cr new file mode 100644 index 00000000..790ed522 --- /dev/null +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/snapshotter.cr @@ -0,0 +1,69 @@ +# =================== +# ©AngelaMos | 2026 +# snapshotter.cr +# =================== + +require "log" +require "./state" +require "../persistence/persistence" +require "../policy/policy" + +module CRE::Tui + # Snapshotter polls the persistence layer at a low frequency and updates + # State#creds_total / due_now / overdue. The TUI render path reads the + # cached snapshot, never the database — so a slow query can't stall + # frame paint. + class Snapshotter + Log = ::Log.for("cre.tui.snapshotter") + + @running : Bool + + def initialize( + @state : State, + @persistence : Persistence::Persistence, + @policies : Array(Policy::Policy) = Policy.registry, + @interval : Time::Span = 5.seconds, + ) + @running = false + end + + def start : Nil + @running = true + spawn(name: "tui-snapshotter") do + refresh + while @running + sleep @interval + break unless @running + refresh + end + end + end + + def stop : Nil + @running = false + end + + def refresh : Nil + now = Time.utc + creds = @persistence.credentials.all + total = creds.size + + due_now = 0 + overdue = 0 + creds.each do |c| + matching = @policies.select(&.matches?(c)) + next if matching.empty? + policy = matching.last + if policy.overdue?(c, now) + overdue += 1 + elsif policy.in_warning_window?(c, now) + due_now += 1 + end + end + + @state.update_counts(total, due_now, overdue) + rescue ex + Log.warn(exception: ex) { "tui snapshot refresh failed" } + end + end +end diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/state.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/state.cr index ff459e3c..479770bc 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/state.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/state.cr @@ -29,14 +29,26 @@ module CRE::Tui @completed_24h : Int32 @started_at : Time @kek_version : Int32 + @creds_total : Int32 + @due_now : Int32 + @overdue : Int32 def initialize(@kek_version : Int32 = 0) @recent = [] of EventRow @active = {} of UUID => RotationRow @completed_24h = 0 + @creds_total = 0 + @due_now = 0 + @overdue = 0 @started_at = Time.utc end + def update_counts(creds_total : Int32, due_now : Int32, overdue : Int32) : Nil + @creds_total = creds_total + @due_now = due_now + @overdue = overdue + end + def apply(ev : Events::Event) : Nil case ev when Events::RotationStarted @@ -90,6 +102,9 @@ module CRE::Tui getter completed_24h : Int32 getter started_at : Time getter kek_version : Int32 + getter creds_total : Int32 + getter due_now : Int32 + getter overdue : Int32 def uptime : Time::Span Time.utc - @started_at diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/tui.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/tui.cr index 253d218a..f96348f2 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/tui.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/tui/tui.cr @@ -6,6 +6,7 @@ require "./ansi" require "./state" require "./renderer" +require "./snapshotter" require "../engine/event_bus" module CRE::Tui diff --git a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/vault/client.cr b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/vault/client.cr index 329628c6..c722efc3 100644 --- a/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/vault/client.cr +++ b/PROJECTS/intermediate/credential-rotation-enforcer/src/cre/vault/client.cr @@ -5,6 +5,7 @@ require "http/client" require "json" +require "../http/retry" module CRE::Vault class VaultError < Exception @@ -54,20 +55,18 @@ module CRE::Vault end private def http_get(path : String) : JSON::Any - uri = URI.parse(@addr + path) headers = HTTP::Headers{"X-Vault-Token" => @token} - response = HTTP::Client.get(uri.to_s, headers: headers) + response = CRE::Http.request("GET", @addr + path, headers, label: "vault.GET#{path}") raise VaultError.new("vault GET #{path}: #{response.body[0, 200]?}", response.status_code) unless response.status_code < 300 JSON.parse(response.body) end private def http_put(path : String, body : String) : JSON::Any - uri = URI.parse(@addr + path) headers = HTTP::Headers{ "X-Vault-Token" => @token, "Content-Type" => "application/json", } - response = HTTP::Client.put(uri.to_s, headers: headers, body: body) + response = CRE::Http.request("PUT", @addr + path, headers, body, label: "vault.PUT#{path}") raise VaultError.new("vault PUT #{path}: #{response.body[0, 200]?}", response.status_code) unless response.status_code < 300 response.body.empty? ? JSON::Any.new(Hash(String, JSON::Any).new) : JSON.parse(response.body) end