From b59652547512144a64fc772e7da69100aec2131e Mon Sep 17 00:00:00 2001 From: Netquirk Primary Developer Date: Fri, 11 Sep 2026 23:04:08 +0000 Subject: [PATCH] feat(server): NET-6820 reintroduce GET /api/host-ops/lock-status with board auth + TTL validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reintroduces the host-ops lock-status endpoint dropped from PR #13276 (NET-4074 / NET-6815) with all four Greptile findings addressed: 1. P1 — gate the route on requireBoard(req). Default-deny: non-board actors receive 403 {error: 'board_only'} before any disk I/O. The host-ops.ts module imports requireBoard from server/src/middleware/auth.ts. 2. P2 — trim the response shape to {host, canonicalHost, status, age_seconds, ttl_seconds}. Internal lock metadata (agent, issue, intent, pid, started, heartbeat) is no longer exposed over the network; the bash helper and JSON-on-disk lock files retain full provenance for netquirk_acquire / netquirk_heartbeat callers. 3. P2 — parseNetquirkLockTtlSec now rejects empty, non-numeric, fractional, negative, zero, and out-of-range values with InvalidLockTtlError. The accepted range is 30 <= n <= 3600 seconds. Validated TTL is captured once at module load via resolvedLockTtl() and surfaced through /healthz's 'host-ops.lock_ttl_sec' block (status: ok | invalid | default) so an operator sees a broken env without reading server logs. 4. P2 — this PR's title names the lock-status endpoint explicitly; the prior bundled commit was buried in a 'narrow null-environment retry' PR scope. Test coverage in server/src/__tests__/host-ops-route.test.ts: - parseNetquirkLockTtlSec boundary (30, 3600), invalid inputs (NaN, fractional, empty, undefined, negative, above max) - resolvedLockTtlFromEnv default vs env source paths - GET /api/host-ops/lock-status returns 403 board_only for missing / non-board actors, 200 trimmed shape for board actors, stale status when the admin-supplied TTL crosses the heartbeat, absent when no lock is present, and 400 missing_host when the host query param is absent. - Alias canonicalisation (apps-arm1.* fold to apps-arm1). /healthz surfaces 'host-ops.lock_ttl_sec' with ttl_seconds, source, env_key, env_value, min, max, default, and status. The field is included in both the redacted (no auth) and full responses; the config carries no host / agent / intent metadata so it is safe to expose without a board session. No host mutation. No data export. No secrets. No auth bypass. Co-Authored-By: Claude --- server/src/__tests__/host-ops-route.test.ts | 306 +++++++++++ server/src/app.ts | 46 ++ server/src/routes/health.ts | 27 + server/src/routes/host-ops.ts | 535 ++++++++++++++++++++ 4 files changed, 914 insertions(+) create mode 100644 server/src/__tests__/host-ops-route.test.ts create mode 100644 server/src/routes/host-ops.ts diff --git a/server/src/__tests__/host-ops-route.test.ts b/server/src/__tests__/host-ops-route.test.ts new file mode 100644 index 0000000000..76089be087 --- /dev/null +++ b/server/src/__tests__/host-ops-route.test.ts @@ -0,0 +1,306 @@ +// NET-6820 host-ops route tests. Cover: +// - parseNetquirkLockTtlSec valid boundary + invalid input +// (NaN, fractional, empty, below min, above max, negative) +// - GET /api/host-ops/lock-status with no actor → 403 board_only +// - GET /api/host-ops/lock-status with non-board actor → 403 board_only +// - GET /api/host-ops/lock-status with board actor → 200 trimmed shape +// - stale heartbeat with admin-specified TTL → status: "stale" +// - absent lock → status: "absent" +// +// The prior `host-ops.test.ts` (NET-3946) covers pure `canonicaliseHost` +// and `readLockStatus`; this file is the dedicated suite for the +// auth-gated route + the new `parseNetquirkLockTtlSec` validator, so +// Greptile review of the route can no longer be confused with the +// process-lost retry review. + +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import express from "express"; +import request from "supertest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + NETQUIRK_LOCK_TTL_SEC_DEFAULT, + NETQUIRK_LOCK_TTL_SEC_ENV_KEY, + NETQUIRK_LOCK_TTL_SEC_MAX, + NETQUIRK_LOCK_TTL_SEC_MIN, + hostOpsRoutes, + parseNetquirkLockTtlSec, + resolvedLockTtlFromEnv, +} from "../routes/host-ops.js"; + +import "./setup-supertest.js"; + +// Anchored "now" so supertest route tests can compare `age_seconds` +// deterministically. +const NOW_MS = Date.parse("2026-09-11T22:00:00Z"); + +type ActorType = "board" | "agent" | "none" | { type: string }; + +function createApp(opts: { + opsDir: string; + actor?: ActorType; + ttlSeconds?: number; + now?: () => number; +}) { + const app = express(); + const actorType = + opts.actor === undefined + ? "board" + : typeof opts.actor === "string" + ? opts.actor + : opts.actor.type; + app.use((req, _res, next) => { + (req as express.Request & { actor: { type: string } }).actor = { + type: actorType, + }; + next(); + }); + app.use( + "/api/host-ops", + hostOpsRoutes({ + opsDir: opts.opsDir, + ...(opts.ttlSeconds !== undefined ? { ttlSeconds: opts.ttlSeconds } : {}), + now: opts.now ?? (() => NOW_MS), + }), + ); + return app; +} + +describe("parseNetquirkLockTtlSec", () => { + it("accepts the default 300s", () => { + expect(parseNetquirkLockTtlSec("300")).toBe(300); + expect(parseNetquirkLockTtlSec("300")).toBe(NETQUIRK_LOCK_TTL_SEC_DEFAULT); + }); + + it("accepts the documented boundaries", () => { + expect(parseNetquirkLockTtlSec(String(NETQUIRK_LOCK_TTL_SEC_MIN))).toBe( + NETQUIRK_LOCK_TTL_SEC_MIN, + ); + expect(parseNetquirkLockTtlSec(String(NETQUIRK_LOCK_TTL_SEC_MAX))).toBe( + NETQUIRK_LOCK_TTL_SEC_MAX, + ); + }); + + it("accepts trimmed whitespace around a valid integer", () => { + expect(parseNetquirkLockTtlSec(" 300 ")).toBe(300); + }); + + it("throws InvalidLockTtlError on non-numeric input", () => { + expect(() => parseNetquirkLockTtlSec("abc")).toThrow(/positive integer|base-10/); + expect(() => parseNetquirkLockTtlSec("Infinity")).toThrow(); + expect(() => parseNetquirkLockTtlSec("-Infinity")).toThrow(); + }); + + it("throws on fractional / non-integer numeric input", () => { + expect(() => parseNetquirkLockTtlSec("1.5")).toThrow(/base-10 integer/); + expect(() => parseNetquirkLockTtlSec("300.0001")).toThrow(/base-10 integer/); + expect(() => parseNetquirkLockTtlSec("0x10")).toThrow(/base-10 integer/); + }); + + it("throws on empty / whitespace-only input", () => { + expect(() => parseNetquirkLockTtlSec("")).toThrow(/empty string/); + expect(() => parseNetquirkLockTtlSec(" ")).toThrow(/empty string/); + }); + + it("throws on undefined / null input", () => { + expect(() => parseNetquirkLockTtlSec(undefined)).toThrow(/positive integer/); + }); + + it("throws on values below the minimum (≤ 0, < 30)", () => { + expect(() => parseNetquirkLockTtlSec("0")).toThrow(/minimum/); + expect(() => parseNetquirkLockTtlSec("29")).toThrow(/minimum/); + expect(() => parseNetquirkLockTtlSec("-5")).toThrow(/minimum/); + }); + + it("throws on values above the maximum (> 3600)", () => { + expect(() => parseNetquirkLockTtlSec("3601")).toThrow(/maximum/); + expect(() => parseNetquirkLockTtlSec("999999")).toThrow(/maximum/); + }); +}); + +describe("resolvedLockTtlFromEnv", () => { + it("returns the default when env is undefined", () => { + expect(resolvedLockTtlFromEnv(undefined)).toEqual({ + ttlSeconds: NETQUIRK_LOCK_TTL_SEC_DEFAULT, + source: "default", + }); + }); + + it("returns the validated env value with source: env", () => { + expect(resolvedLockTtlFromEnv("600")).toEqual({ + ttlSeconds: 600, + source: "env", + }); + }); + + it("propagates validation errors for invalid env values", () => { + expect(() => resolvedLockTtlFromEnv("garbage")).toThrow(); + expect(() => resolvedLockTtlFromEnv("0")).toThrow(); + expect(() => resolvedLockTtlFromEnv("3601")).toThrow(); + }); +}); + +describe("GET /api/host-ops/lock-status", () => { + let scratch = ""; + let opsDir = ""; + + beforeEach(() => { + scratch = mkdtempSync(join(tmpdir(), "host-ops-route-test-")); + opsDir = join(scratch, "ops"); + mkdirSync(opsDir, { recursive: true }); + }); + + afterEach(() => { + rmSync(scratch, { recursive: true, force: true }); + delete process.env[NETQUIRK_LOCK_TTL_SEC_ENV_KEY]; + }); + + function writeLock(canonical: string, line: string): void { + writeFileSync(join(opsDir, `${canonical}.lock`), `${line}\n`, "utf8"); + } + + function liveLockLine(pid: number, hbAgoSec: number): string { + const hbMs = NOW_MS - hbAgoSec * 1000; + const hb = new Date(hbMs).toISOString().replace(/\.\d{3}Z$/, "Z"); + const started = new Date(NOW_MS - 600_000) + .toISOString() + .replace(/\.\d{3}Z$/, "Z"); + return `agent=alpha-uuid issue=NET-TEST intent=binary install pid=${pid} started=${started} heartbeat=${hb}`; + } + + function staleLockLine(pid: number, hbAgoSec: number): string { + const hbMs = NOW_MS - hbAgoSec * 1000; + const hb = new Date(hbMs).toISOString().replace(/\.\d{3}Z$/, "Z"); + const started = new Date(NOW_MS - 1_800_000) + .toISOString() + .replace(/\.\d{3}Z$/, "Z"); + return `agent=alpha-uuid issue=NET-TEST intent=binary install pid=${pid} started=${started} heartbeat=${hb}`; + } + + it("returns 403 board_only when the actor is missing entirely", async () => { + const res = await request(createApp({ opsDir, actor: "none" })).get( + "/api/host-ops/lock-status?host=apps-arm1", + ); + expect(res.status).toBe(403); + expect(res.body).toEqual({ error: "board_only" }); + }); + + it("returns 403 board_only when the actor is not a board user", async () => { + const res = await request(createApp({ opsDir, actor: "agent" })).get( + "/api/host-ops/lock-status?host=apps-arm1", + ); + expect(res.status).toBe(403); + expect(res.body).toEqual({ error: "board_only" }); + }); + + it("returns the trimmed response shape for a board actor with no lock present", async () => { + const res = await request(createApp({ opsDir })).get( + "/api/host-ops/lock-status?host=apps-arm1", + ); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + host: "apps-arm1", + canonicalHost: "apps-arm1", + status: "absent", + age_seconds: 0, + ttl_seconds: NETQUIRK_LOCK_TTL_SEC_DEFAULT, + }); + // Trimmed shape — these fields MUST NOT appear. + expect(res.body).not.toHaveProperty("agent"); + expect(res.body).not.toHaveProperty("issue"); + expect(res.body).not.toHaveProperty("intent"); + expect(res.body).not.toHaveProperty("pid"); + expect(res.body).not.toHaveProperty("started"); + expect(res.body).not.toHaveProperty("heartbeat"); + // And the response carries no key outside the documented set. + expect(Object.keys(res.body).sort()).toEqual( + ["age_seconds", "canonicalHost", "host", "status", "ttl_seconds"].sort(), + ); + }); + + it("returns the trimmed live response for a board actor when the lock is fresh", async () => { + writeLock("apps-arm1", liveLockLine(4242, 30)); + const res = await request(createApp({ opsDir })).get( + "/api/host-ops/lock-status?host=apps-arm1", + ); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ + host: "apps-arm1", + canonicalHost: "apps-arm1", + status: "live", + ttl_seconds: NETQUIRK_LOCK_TTL_SEC_DEFAULT, + }); + expect(res.body.age_seconds).toBeGreaterThanOrEqual(29); + expect(res.body.age_seconds).toBeLessThan(31); + expect(res.body).not.toHaveProperty("agent"); + expect(res.body).not.toHaveProperty("issue"); + expect(res.body).not.toHaveProperty("intent"); + expect(res.body).not.toHaveProperty("pid"); + expect(res.body).not.toHaveProperty("started"); + expect(res.body).not.toHaveProperty("heartbeat"); + }); + + it("returns status:stale when the heartbeat is older than the admin-specified TTL", async () => { + // Lock file written with a 10-minute-old heartbeat; caller injects + // ttlSeconds=120 to ensure it crosses the freshness threshold. + writeLock("apps-arm1", staleLockLine(4242, 600)); + const res = await request( + createApp({ opsDir, ttlSeconds: 120 }), + ).get("/api/host-ops/lock-status?host=apps-arm1"); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ + host: "apps-arm1", + canonicalHost: "apps-arm1", + status: "stale", + ttl_seconds: 120, + }); + expect(res.body.age_seconds).toBeGreaterThanOrEqual(600); + }); + + it("rejects an invalid ttlSeconds dependency injection with 500 + the read failure", async () => { + writeLock("apps-arm1", liveLockLine(4242, 30)); + // Force a throw inside readLockStatus by handing the route a TTL + // value the validator would normally reject at module load. + // The route catches InvalidLockTtlError and falls back to the + // default 300s, so this case actually still passes — instead we + // confirm that the documented fall-back TTL surfaces in the + // response when the injected value is rejected. + const res = await request( + createApp({ opsDir, ttlSeconds: -1 }), + ).get("/api/host-ops/lock-status?host=apps-arm1"); + expect(res.status).toBe(200); + expect(res.body.ttl_seconds).toBe(NETQUIRK_LOCK_TTL_SEC_DEFAULT); + }); + + it("returns 400 when the host query param is missing", async () => { + const res = await request(createApp({ opsDir })).get( + "/api/host-ops/lock-status", + ); + expect(res.status).toBe(400); + expect(res.body.error).toBe("missing_host"); + }); + + it("canonicalises an alias to the same lock as the canonical name", async () => { + writeLock("apps-arm1", liveLockLine(4242, 30)); + for (const alias of [ + "apps-arm1", + "apps-arm1.nq.vmgen.ie", + "apps-arm1.bigeye-nominal.ts.net", + "apps.netquirk.com", + "79.72.69.146", + "100.80.86.23", + ]) { + const res = await request(createApp({ opsDir })).get( + `/api/host-ops/lock-status?host=${encodeURIComponent(alias)}`, + ); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ + host: alias, + canonicalHost: "apps-arm1", + status: "live", + }); + } + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index 568761f1ff..3c0e5c1b13 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -35,6 +35,15 @@ import { } from "./services/company-import-transfers.js"; import { companyTransferRunService } from "./services/company-transfer-runs.js"; import { healthRoutes } from "./routes/health.js"; +import { + hostOpsRoutes, + parseNetquirkLockTtlSec, + resolvedLockTtl, + NETQUIRK_LOCK_TTL_SEC_ENV_KEY, + NETQUIRK_LOCK_TTL_SEC_MAX, + NETQUIRK_LOCK_TTL_SEC_MIN, + NETQUIRK_LOCK_TTL_SEC_DEFAULT, +} from "./routes/host-ops.js"; import { cloudRuntimeIdentityMiddleware } from "./middleware/cloud-runtime-identity.js"; import { cloudControlMiddleware } from "./middleware/cloud-control.js"; import { cloudRoutes } from "./routes/cloud.js"; @@ -619,6 +628,28 @@ export async function createApp( // Mount API routes const api = Router(); api.use(boardMutationGuard()); + // NET-6820: compute the host-ops lock TTL snapshot once at app + // boot. /healthz surfaces this so an operator can see, at a + // glance, whether `NETQUIRK_LOCK_TTL_SEC` is set, valid, and within + // the 30..3600 range — without having to read server logs. + const hostOpsLockTtlRaw = process.env[NETQUIRK_LOCK_TTL_SEC_ENV_KEY]; + const resolvedHostOpsTtl = resolvedLockTtl(); + const hostOpsLockTtlStatus: "ok" | "invalid" | "default" = + hostOpsLockTtlRaw === undefined + ? "default" + : (() => { + try { + // Re-run the validator so /healthz reports `invalid` + // when the env value would have been rejected by the + // module loader. resolvedLockTtl() swallows this so we + // can keep the server up; the healthz surface is the + // operator-visible signal. + parseNetquirkLockTtlSec(hostOpsLockTtlRaw); + return "ok"; + } catch { + return "invalid"; + } + })(); api.use( "/health", healthRoutes(db, { @@ -627,9 +658,24 @@ export async function createApp( authReady: opts.authReady, companyDeletionEnabled: opts.companyDeletionEnabled, databaseBackupHealth: opts.databaseBackupHealth, + hostOpsLockTtl: { + ttl_seconds: resolvedHostOpsTtl.ttlSeconds, + source: resolvedHostOpsTtl.source, + env_key: NETQUIRK_LOCK_TTL_SEC_ENV_KEY, + env_value: hostOpsLockTtlRaw ?? null, + min: NETQUIRK_LOCK_TTL_SEC_MIN, + max: NETQUIRK_LOCK_TTL_SEC_MAX, + default: NETQUIRK_LOCK_TTL_SEC_DEFAULT, + status: hostOpsLockTtlStatus, + }, }), ); api.use(openApiRoutes()); + // NET-6820: host-ops lock-status probe. The route itself enforces + // `requireBoard(req)` so the mount is safe; default-deny at the + // handler means non-board actors get 403 regardless of how the + // express router is layered. + api.use("/host-ops", hostOpsRoutes()); api.use("/cloud", cloudRoutes()); api.use("/companies", companyRoutes(db, opts.storageService)); api.use(llmRoutes(db)); diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts index 0e4d954722..62f761dc9d 100644 --- a/server/src/routes/health.ts +++ b/server/src/routes/health.ts @@ -116,6 +116,17 @@ function getCloudHealthStatus(env: CloudInstanceEnv) { }; } +export interface HostOpsLockTtlHealth { + ttl_seconds: number; + source: "env" | "default"; + env_key: string; + env_value: string | null; + min: number; + max: number; + default: number; + status: "ok" | "invalid" | "default"; +} + export function healthRoutes( db?: Db, opts: { @@ -125,6 +136,14 @@ export function healthRoutes( companyDeletionEnabled: boolean; serverInfo?: ServerInfoSnapshot; databaseBackupHealth?: InspectDatabaseBackupHealthOptions; + /** + * NET-6820: a pre-computed snapshot of the validated + * `NETQUIRK_LOCK_TTL_SEC` value plus its source / status. The + * `app.ts` mount point builds this once at boot from + * `resolvedLockTtl()` so /healthz can surface a broken env + * without crashing the server. + */ + hostOpsLockTtl?: HostOpsLockTtlHealth; runtimeEnv?: CloudInstanceEnv; } = { deploymentMode: "local_trusted", @@ -403,6 +422,12 @@ export function healthRoutes( ...(workspaceReadiness ? { workspace: workspaceReadiness } : {}), ...(cloud ? { cloud } : {}), ...(hiddenSettings.length ? { hiddenSettings } : {}), + // NET-6820: surface the validated host-ops lock TTL even on + // the redacted path — it carries no host / agent / intent + // metadata, just the TTL configuration. Operators watching + // /healthz need to see `host-ops.lock_ttl_sec.status: invalid` + // without authenticating first. + ...(opts.hostOpsLockTtl ? { "host-ops": { lock_ttl_sec: opts.hostOpsLockTtl } } : {}), }); return; } @@ -429,6 +454,8 @@ export function healthRoutes( ...(workspaceReadiness ? { workspace: workspaceReadiness } : {}), ...(cloud ? { cloud } : {}), ...(hiddenSettings.length ? { hiddenSettings } : {}), + // NET-6820: full /healthz response mirrors the redacted shape. + ...(opts.hostOpsLockTtl ? { "host-ops": { lock_ttl_sec: opts.hostOpsLockTtl } } : {}), }); }); diff --git a/server/src/routes/host-ops.ts b/server/src/routes/host-ops.ts new file mode 100644 index 0000000000..3fd78569a3 --- /dev/null +++ b/server/src/routes/host-ops.ts @@ -0,0 +1,535 @@ +//! NET-3946 / NET-4074 / NET-6820 host-ops routes. +//! +//! Surfaces the orchestrator's NET-2626 host-mutual-exclusion locks to +//! in-cluster consumers that need to know whether a host is currently +//! under an authorized mutation window. The primary caller is +//! monitoring-api's NET-2663 incoming webhook receiver, which now +//! suppresses monitor pages while a lock is live and notes stale +//! locks in the page body (NET-3946). +//! +//! ## Alias canonicalisation (the "single shared function" of NET-3946 AC #4) +//! +//! One physical host answers to several spellings — short name, FQDN, +//! public IP, Tailscale MagicDNS name, Tailscale IP, user@host form. +//! Locking on the literal caller-supplied token would let the same +//! physical host take independent, non-excluding locks under each of +//! its spellings (NET-3936). The bash helper `netquirk-host-ops.sh` +//! has a `_nqho_canon_host` table for this; this module mirrors that +//! table byte-for-byte so every caller resolves onto the SAME canonical +//! key as the bash lock files. If you add an alias here, also add it +//! to `_NQHO_HOST_ALIASES_BUILTIN` in +//! `/home/paperclip/bin/netquirk-host-ops.sh` — and vice versa. The +//! two sources of truth are deliberately kept in sync by convention +//! (a CI guard or a build-time diff would be a worthwhile follow-up; +//! for now the file headers document the requirement). +//! +//! ## Lock-file format +//! +//! `/.lock` contains a single +//! whitespace-tolerant line of `key=value` fields (see +//! `_nqho_parse_lock_line` in netquirk-host-ops.sh): +//! +//! ```text +//! agent= issue= intent= pid= +//! started= heartbeat= [taken-from=] +//! ``` +//! +//! `heartbeat` is the freshness signal. TTL defaults to 300s; the +//! orchestrator's auto-heartbeat refresher (NET-3946) keeps it warm +//! while the lock is held. Anything older than TTL is STALE — the lock +//! has been abandoned and the next acquire will take it over +//! (NET-3128 stale-takeover branch). +//! +//! ## NET-6820 — auth and TTL-validation tightening +//! +//! Greptile flagged four findings on the original `host-ops.ts` +//! shipped via NET-4074 / PR #13276: +//! +//! 1. P1 — `GET /api/host-ops/lock-status` was routable by any +//! caller, including non-board actors. Re-gated on +//! `requireBoard(req)` (see `server/src/middleware/auth.ts`). +//! 2. P2 — the response exposed internal lock metadata +//! (`agent`, `issue`, `intent`, `pid`, `started`, `heartbeat`). +//! The route now only returns the four fields the operator UI +//! needs to decide whether to suppress a page: `host`, +//! `canonicalHost`, `status`, `age_seconds`, `ttl_seconds`. +//! The bash helper and the JSON-on-disk lock files still carry +//! full provenance for `netquirk_acquire` / `netquirk_heartbeat` +//! callers. +//! 3. P2 — `NETQUIRK_LOCK_TTL_SEC` was parsed with bare `Number(...)`; +//! non-numeric, zero, or negative input made every lock appear +//! stale. Validation now goes through `parseNetquirkLockTtlSec`, +//! which rejects the empty string, NaN, fractional values, and +//! anything outside `30 ≤ n ≤ 3600`. The validated TTL is the +//! single source of truth for both the route and `/healthz`'s +//! `host-ops.lock_ttl_sec` field. +//! 4. P2 — the route mount was bundled into a PR titled +//! "narrow null-environment retry" without scope documentation. +//! NET-6820 is a dedicated PR for the lock-status endpoint; its +//! title names the endpoint. + +import { existsSync, readFileSync, statSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { Request } from "express"; +import { Router } from "express"; +import { requireBoard } from "../middleware/auth.js"; + +// ---- TTL validation (NET-6820 finding #3) ---- +// +// `parseNetquirkLockTtlSec` accepts the raw `NETQUIRK_LOCK_TTL_SEC` +// env value and returns a validated integer seconds count. The +// accepted range (30 ≤ n ≤ 3600) is the union of: +// - the NET-3946 default of 300s (we lower-bound to half that to +// reject obvious operator typos that would flip every lock to +// "stale" within seconds); and +// - an upper bound that keeps a stale lock from living forever +// and silently blocking stale-takeover (NET-3128) on a dead +// orchestrator. +// +// `parseNetquirkLockTtlSec` throws on bad input so the failure mode +// is loud at startup rather than silently mis-classifying every lock +// at runtime. The throw is caught once, at module load, and surfaced +// via `/healthz`'s `host-ops.lock_ttl_sec` field (`status: "invalid"`) +// so an operator can see the broken env without reading a stack +// trace from the logs. +export const NETQUIRK_LOCK_TTL_SEC_ENV_KEY = "NETQUIRK_LOCK_TTL_SEC"; +export const NETQUIRK_LOCK_TTL_SEC_MIN = 30; +export const NETQUIRK_LOCK_TTL_SEC_MAX = 3600; +export const NETQUIRK_LOCK_TTL_SEC_DEFAULT = 300; + +export type ParsedLockTtl = { + readonly ttlSeconds: number; + readonly source: "env" | "default"; +}; + +export class InvalidLockTtlError extends Error { + readonly code = "invalid_lock_ttl"; + constructor( + public readonly raw: string | undefined, + message: string, + ) { + super(message); + this.name = "InvalidLockTtlError"; + } +} + +export function parseNetquirkLockTtlSec(raw: string | undefined): number { + if (raw === undefined || raw === null) { + throw new InvalidLockTtlError( + raw ?? undefined, + `${NETQUIRK_LOCK_TTL_SEC_ENV_KEY} must be a positive integer (received ${JSON.stringify(raw)})`, + ); + } + const trimmed = raw.trim(); + if (trimmed.length === 0) { + throw new InvalidLockTtlError( + raw, + `${NETQUIRK_LOCK_TTL_SEC_ENV_KEY} must be a positive integer (received empty string)`, + ); + } + // Reject explicit "Infinity" / "NaN" / hex / whitespace-padded garbage + // before coercing. `Number.isInteger(Number("Infinity"))` is false, so + // the same check rejects "Infinity" — but spell it out for clarity. + if (!/^-?\d+$/.test(trimmed)) { + throw new InvalidLockTtlError( + raw, + `${NETQUIRK_LOCK_TTL_SEC_ENV_KEY} must be a base-10 integer (received ${JSON.stringify(raw)})`, + ); + } + const parsed = Number(trimmed); + if (!Number.isInteger(parsed)) { + throw new InvalidLockTtlError( + raw, + `${NETQUIRK_LOCK_TTL_SEC_ENV_KEY} must be an integer (received ${JSON.stringify(raw)})`, + ); + } + if (parsed < NETQUIRK_LOCK_TTL_SEC_MIN) { + throw new InvalidLockTtlError( + raw, + `${NETQUIRK_LOCK_TTL_SEC_ENV_KEY}=${parsed} is below the ${NETQUIRK_LOCK_TTL_SEC_MIN}s minimum`, + ); + } + if (parsed > NETQUIRK_LOCK_TTL_SEC_MAX) { + throw new InvalidLockTtlError( + raw, + `${NETQUIRK_LOCK_TTL_SEC_ENV_KEY}=${parsed} is above the ${NETQUIRK_LOCK_TTL_SEC_MAX}s maximum`, + ); + } + return parsed; +} + +/** + * Resolve the validated TTL once at module load. If the env value is + * unparseable we keep the process running (the orchestrator can still + * take locks via the bash helper, which uses its own 300s default), + * but `/healthz` flips to `host-ops.lock_ttl_sec.status: "invalid"` + * so the broken env is visible at the surface operators actually + * watch. The route itself never reads the env directly — it consumes + * `resolvedLockTtl()`. + */ +function resolveLockTtl(): ParsedLockTtl { + const raw = process.env[NETQUIRK_LOCK_TTL_SEC_ENV_KEY]; + if (raw === undefined) { + return { ttlSeconds: NETQUIRK_LOCK_TTL_SEC_DEFAULT, source: "default" }; + } + try { + return { + ttlSeconds: parseNetquirkLockTtlSec(raw), + source: "env", + }; + } catch { + return { + ttlSeconds: NETQUIRK_LOCK_TTL_SEC_DEFAULT, + source: "default", + }; + } +} + +const RESOLVED_LOCK_TTL: ParsedLockTtl = resolveLockTtl(); + +export function resolvedLockTtl(): ParsedLockTtl { + return RESOLVED_LOCK_TTL; +} + +/** + * Re-parse the env at request time. Used by tests that mutate + * `process.env.NETQUIRK_LOCK_TTL_SEC` between cases. Production + * callers should read `resolvedLockTtl()` once and trust the + * module-level capture. + */ +export function resolvedLockTtlFromEnv(envValue: string | undefined): ParsedLockTtl { + if (envValue === undefined) { + return { ttlSeconds: NETQUIRK_LOCK_TTL_SEC_DEFAULT, source: "default" }; + } + return { + ttlSeconds: parseNetquirkLockTtlSec(envValue), + source: "env", + }; +} + +// ---- alias canonicalisation (mirror of netquirk-host-ops.sh) ---- +// +// KEEP IN SYNC with `_NQHO_HOST_ALIASES_BUILTIN` in +// `/home/paperclip/bin/netquirk-host-ops.sh`. Each row is `canonical +// alias alias …`. The first field is the canonical name NET-2626 +// expects under its lock key; aliases are alternate spellings that +// fold onto it. Verified against `getent hosts` / `tailscale status` +// / `~/.ssh/config` on 2026-09-03. +// +// (doc) entries are documented aliases that may not resolve in DNS +// today; folding them is safe because the lock key is a lock +// identity, not a connection target. The ssh mirror always dials the +// literal string the caller passed. +const HOST_ALIAS_TABLE: readonly string[][] = [ + ["144.21.58.216", "cp.netquirk.com", "cloud.netquirk.com", "panel"], + ["apps-arm1", "apps-arm1.nq.vmgen.ie", "apps-arm1.bigeye-nominal.ts.net", "apps.netquirk.com", "79.72.69.146", "100.80.86.23"], + ["mon-dash", "mon-dashboard", "mon.netquirk.com"], + ["nas", "nas.srvb.yt", "100.104.244.88", "nas.bigeye-nominal.ts.net"], + ["new", "100.108.28.26", "57.129.148.177", "new.bigeye-nominal.ts.net"], + ["paperclip", "100.87.114.97", "paperclip.bigeye-nominal.ts.net", "localhost", "127.0.0.1"], +]; + +// Path-safety transformation applied to every token before either +// alias-table lookup or filesystem write. Mirrors +// `_nqho_canon_host`'s normalisation in netquirk-host-ops.sh: strip +// `user@`, lowercase, strip trailing dots, map non-[a-z0-9._-] to +// `_`, collapse bare `.` / `..` / empty token to `_`. +function normaliseToken(raw: string): string { + if (raw.length === 0) return "_"; + let tok = raw; + const at = tok.lastIndexOf("@"); + if (at >= 0) tok = tok.slice(at + 1); + tok = tok.toLowerCase(); + while (tok.endsWith(".")) tok = tok.slice(0, -1); + tok = tok.replace(/[^a-z0-9._-]/g, "_"); + if (tok === "" || tok === "." || tok === "..") return "_"; + return tok; +} + +export function canonicaliseHost(raw: string | null | undefined): string { + const tok = normaliseToken(raw ?? ""); + for (const row of HOST_ALIAS_TABLE) { + const canonical = row[0]; + for (const alias of row) { + if (normaliseToken(alias) === tok) return canonical; + } + } + return tok; +} + +// ---- lock-file reading ---- +// +// Default lock directory mirrors `NETQUIRK_OPS_DIR` from +// netquirk-host-ops.sh (which defaults to `~/.netquirk/ops`). +// Override via `PAPERCLIP_NETQUIRK_OPS_DIR` env var for tests +// and for operators who keep the legacy /var/lib/netquirk/ops +// path. +// +// Note: do NOT derive the default from `resolvePaperclipHomeDir()` +// — that returns `~/.paperclip`, not `~`, and would silently miss +// every real lock. Use `os.homedir()` to match bash's `~` expansion. +const DEFAULT_OPS_DIR = path.join(os.homedir(), ".netquirk", "ops"); + +interface ParsedLockLine { + agent: string; + issue: string; + intent: string; + pid: string; + started: string; + heartbeat: string; + takenFrom: string; +} + +function parseLockLine(line: string): ParsedLockLine { + const tokens = line.split(/\s+/).filter(Boolean); + const out: ParsedLockLine = { + agent: "", + issue: "", + intent: "", + pid: "", + started: "", + heartbeat: "", + takenFrom: "", + }; + for (let i = 0; i < tokens.length; i++) { + const kv = tokens[i]; + if (kv.startsWith("agent=")) out.agent = kv.slice("agent=".length); + else if (kv.startsWith("issue=")) out.issue = kv.slice("issue=".length); + else if (kv.startsWith("intent=")) { + // intent may contain spaces — collect everything up to the next + // known key. Mirrors `_nqho_parse_lock_line`. + let v = kv.slice("intent=".length); + let j = i + 1; + while ( + j < tokens.length && + !tokens[j].startsWith("agent=") && + !tokens[j].startsWith("issue=") && + !tokens[j].startsWith("pid=") && + !tokens[j].startsWith("started=") && + !tokens[j].startsWith("heartbeat=") && + !tokens[j].startsWith("taken-from=") + ) { + v = `${v} ${tokens[j]}`; + j++; + } + out.intent = v; + i = j - 1; + } else if (kv.startsWith("pid=")) out.pid = kv.slice("pid=".length); + else if (kv.startsWith("started=")) out.started = kv.slice("started=".length); + else if (kv.startsWith("heartbeat=")) out.heartbeat = kv.slice("heartbeat=".length); + else if (kv.startsWith("taken-from=")) out.takenFrom = kv.slice("taken-from=".length); + } + return out; +} + +function parseRfc3339Utc(s: string): number { + // Accept "YYYY-MM-DDTHH:MM:SSZ" with optional fractional seconds. + // Returns ms since epoch, or NaN on unparseable input. + if (!s) return NaN; + const m = Date.parse(s); + return Number.isFinite(m) ? m : NaN; +} + +export type LockStatus = + | { status: "absent" } + | { + status: "live"; + canonicalHost: string; + agent: string; + issue: string; + intent: string; + pid: string; + started: string; + heartbeat: string; + ageSeconds: number; + ttlSeconds: number; + } + | { + status: "stale"; + canonicalHost: string; + agent: string; + issue: string; + intent: string; + pid: string; + started: string; + heartbeat: string; + ageSeconds: number; + ttlSeconds: number; + }; + +export interface ReadLockStatusOpts { + opsDir?: string; + ttlSeconds?: number; + now?: number; +} + +export function readLockStatus( + rawHost: string, + opts?: ReadLockStatusOpts, +): LockStatus { + const opsDir = + opts?.opsDir ?? + process.env.PAPERCLIP_NETQUIRK_OPS_DIR ?? + DEFAULT_OPS_DIR; + // TTL precedence: caller-supplied opts > validated module-level + // capture > default 300s. Anything invalid is silently dropped to + // the default — the validator surfaces bad env via `/healthz`, + // not by throwing out of `readLockStatus`. + let ttlSeconds: number; + if (typeof opts?.ttlSeconds === "number") { + try { + ttlSeconds = parseNetquirkLockTtlSec(String(opts.ttlSeconds)); + } catch { + ttlSeconds = NETQUIRK_LOCK_TTL_SEC_DEFAULT; + } + } else { + ttlSeconds = RESOLVED_LOCK_TTL.ttlSeconds; + } + const now = opts?.now ?? Date.now(); + const canonical = canonicaliseHost(rawHost); + const lockPath = path.join(opsDir, `${canonical}.lock`); + if (!existsSync(lockPath)) { + return { status: "absent" }; + } + // Touching statSync validates readability and lets us return a clean + // error rather than a noisy stack trace if the path is a directory, + // a broken symlink, etc. + try { + statSync(lockPath); + } catch { + return { status: "absent" }; + } + let raw = ""; + try { + raw = readFileSync(lockPath, "utf8"); + } catch { + return { status: "absent" }; + } + const line = raw.trim(); + if (line.length === 0) return { status: "absent" }; + const parsed = parseLockLine(line); + const hbMs = parseRfc3339Utc(parsed.heartbeat); + // Fallback: heartbeat is unparseable — treat as stale (safer than + // live because stale pages trigger an operator-visible "mutation + // may be unattended" notice per NET-3946 AC #1). + const ageSeconds = Number.isFinite(hbMs) + ? Math.max(0, Math.floor((now - hbMs) / 1000)) + : Number.POSITIVE_INFINITY; + const status = ageSeconds < ttlSeconds ? "live" : "stale"; + return { + status, + canonicalHost: canonical, + agent: parsed.agent, + issue: parsed.issue, + intent: parsed.intent, + pid: parsed.pid, + started: parsed.started, + heartbeat: parsed.heartbeat, + ageSeconds: Number.isFinite(ageSeconds) ? ageSeconds : ttlSeconds, + ttlSeconds, + }; +} + +// ---- HTTP route ---- +// +// Auth: require a board actor (any board user can poll lock status; +// agents cannot, by NET-2626 convention — the lock holder is the only +// party that should ever need this view, and they hold the lock in +// their own bash session, not via this endpoint). +// +// Response shape is intentionally narrow: only the four fields the +// operator UI needs to decide whether to suppress a page. Internal +// lock metadata (agent / issue / intent / pid / started / heartbeat) +// stays on disk and in the bash helper — this endpoint is a probe, +// not an admin surface. +function readHostQuery(req: Pick): string | null { + const raw = req.query.host; + if (typeof raw === "string" && raw.trim().length > 0) return raw.trim(); + if ( + Array.isArray(raw) && + raw.length > 0 && + typeof raw[0] === "string" + ) { + return raw[0].trim(); + } + return null; +} + +export interface HostOpsRouteDeps { + opsDir?: string; + ttlSeconds?: number; + /** + * Inject a fixed `now` (epoch ms) so the route can be exercised + * deterministically from a supertest harness. Defaults to + * `Date.now()`. Production callers pass nothing. + */ + now?: () => number; +} + +export function hostOpsRoutes(deps: HostOpsRouteDeps = {}): import("express-serve-static-core").Router { + const router = Router(); + // GET /api/host-ops/lock-status?host= + // + // Note: the `api` router is mounted at `/api/host-ops` (see app.ts), + // so this route's full URL is `/api/host-ops/lock-status`. The route + // path below MUST therefore be `/lock-status`, NOT + // `/host-ops/lock-status` — using the latter would yield the broken + // double-prefix `/api/host-ops/host-ops/lock-status`. + // + // Returns the live / stale / absent lock state for the supplied + // host after alias canonicalisation. Used by monitoring-api's + // NET-2663 receiver to decide whether a monitor page should be + // suppressed or annotated. + router.get("/lock-status", (req, res) => { + // NET-6820 finding #1: gate on requireBoard BEFORE any disk I/O + // so a non-board caller cannot probe hosts at all. Default-deny. + if (!requireBoard(req)) { + res.status(403).json({ error: "board_only" }); + return; + } + const host = readHostQuery(req); + if (!host) { + res.status(400).json({ + error: "missing_host", + message: "query param `host` is required", + }); + return; + } + let status: LockStatus; + try { + status = readLockStatus(host, { + ...(deps.opsDir !== undefined ? { opsDir: deps.opsDir } : {}), + ...(deps.ttlSeconds !== undefined ? { ttlSeconds: deps.ttlSeconds } : {}), + ...(deps.now ? { now: deps.now() } : {}), + }); + } catch (err) { + res.status(500).json({ + error: "lock_read_failed", + message: err instanceof Error ? err.message : String(err), + }); + return; + } + if (status.status === "absent") { + res.json({ + host, + canonicalHost: canonicaliseHost(host), + status: "absent", + age_seconds: 0, + ttl_seconds: RESOLVED_LOCK_TTL.ttlSeconds, + }); + return; + } + // Trimmed response shape — no agent / issue / intent / pid / + // started / heartbeat fields. The bash helper retains full + // provenance for `netquirk_acquire` callers. + res.json({ + host, + canonicalHost: status.canonicalHost, + status: status.status, + age_seconds: status.ageSeconds, + ttl_seconds: status.ttlSeconds, + }); + }); + return router; +}