/** * Smoke Lab recurring-routine wrapper (PAP-13351 / plan §D5). * * This is what the daily Smoke Lab routine's woken QA agent runs. It wraps the * S5 agent-driven browser runner (`smoke-lab-browser-runner.mts`) with the three * behaviours the plan asks for: * * 1. budget-bounded — the browser run is killed after SMOKE_BUDGET_MS. * 2. file-on-failure — a real product failure files a control-plane issue * (failing step + screenshot) assigned to the owning coder * and links it back to the routine issue. * 3. amber/skipped — if the local_trusted instance or the flag is unavailable * (or the runner itself crashes on an environment problem), * record an amber/skipped outcome instead of failing * silently or filing a bogus product bug. * * It never forks the P1-P7 step list: the actual lifecycle lives in the S5 * runner, which imports the S4 catalog. * * Run (as the woken routine agent, after booting a throwaway local_trusted * instance per doc/connections/SMOKE-LAB-BROWSER-RUNNER.md §0): * node --experimental-strip-types tests/e2e/smoke-lab-routine.mts * * Target under test (the throwaway instance): * SMOKE_BASE=http://127.0.0.1:3251 local_trusted, non-prod instance * SMOKE_ONLY=P1,P3 budget-bounding path subset (optional) * SMOKE_SHOT_DIR= screenshot output (default under TMPDIR) * SMOKE_BUDGET_MS=600000 hard timeout for the browser run * * Control plane where failure issues + amber notes are recorded (the REAL * Paperclip company — provided in every routine run's env): * PAPERCLIP_API_URL, PAPERCLIP_API_KEY, PAPERCLIP_COMPANY_ID, PAPERCLIP_RUN_ID * ROUTINE_ISSUE_ID= issue to record against (default PAPERCLIP_TASK_ID) * SMOKE_OWNER_DEFAULT / SMOKE_OWNER_UI / SMOKE_OWNER_CTO owner overrides * SMOKE_DRY_RUN=1 log the control-plane writes, don't perform them */ import { spawn } from "node:child_process"; import { promises as fs } from "node:fs"; import path from "node:path"; import os from "node:os"; import { pathToFileURL } from "node:url"; const SMOKE_BASE = (process.env.SMOKE_BASE ?? "http://127.0.0.1:3251").replace(/\/$/, ""); const SHOT_DIR = process.env.SMOKE_SHOT_DIR ?? path.join(os.tmpdir(), `pap13351-routine-${Date.now()}`); const BUDGET_MS = Number(process.env.SMOKE_BUDGET_MS ?? 600_000); const ONLY = process.env.SMOKE_ONLY ?? ""; const DRY = process.env.SMOKE_DRY_RUN === "1"; const CP_BASE = (() => { const b = (process.env.PAPERCLIP_API_URL ?? "").replace(/\/$/, ""); return b.replace(/\/api$/, ""); })(); const CP_KEY = process.env.PAPERCLIP_API_KEY ?? ""; const CP_COMPANY = process.env.PAPERCLIP_COMPANY_ID ?? ""; const CP_RUN = process.env.PAPERCLIP_RUN_ID ?? ""; const ROUTINE_ISSUE_ID = process.env.ROUTINE_ISSUE_ID ?? process.env.PAPERCLIP_TASK_ID ?? ""; // Owning coder per plan (§5): S1/S4 governance+catalog = CodexCoder; S2 UI = ClaudeCoder; // escalation fallback = CTO. Steps recorded by the runner are governance/API behaviours, // so the default owner is the S1/S4 coder; the UI owner is available for matrix-render defects. const OWNER_DEFAULT = process.env.SMOKE_OWNER_DEFAULT ?? "eab6f1c7-5950-410e-8eed-7d074426165d"; // CodexCoder const OWNER_UI = process.env.SMOKE_OWNER_UI ?? "3108ef8e-5ed0-41d9-b561-6b41c41b8545"; // ClaudeCoder const OWNER_CTO = process.env.SMOKE_OWNER_CTO ?? "66b3c071-6cb8-4424-b833-9d9b6318de0b"; // CTO const RUNNER = path.join(import.meta.dirname, "smoke-lab-browser-runner.mts"); type Json = Record; export function isForcedFailureSelfTest(failed: string[], forcedFailDetail = process.env.SMOKE_FORCE_FAIL): boolean { if (!forcedFailDetail || failed.length !== 1) return false; return /^[A-Z0-9]+\/forced-failure:\s*SMOKE_FORCE_FAIL:/.test(failed[0] ?? ""); } export function findDuplicateFailureIssue(issues: Json[], title: string, routineIssueId: string): Json | null { return ( issues.find( (issue) => issue?.title === title && issue?.parentId === routineIssueId && issue?.status !== "cancelled", ) ?? null ); } function log(...a: any[]) { console.log("[routine]", ...a); } async function cp(method: string, apiPath: string, body?: Json): Promise { if (DRY) { log(`DRY ${method} ${apiPath}`, body ? JSON.stringify(body).slice(0, 200) : ""); return {}; } const res = await fetch(`${CP_BASE}${apiPath}`, { method, headers: { "content-type": "application/json", authorization: `Bearer ${CP_KEY}`, "x-paperclip-run-id": CP_RUN, }, body: body === undefined ? undefined : JSON.stringify(body), }); const text = await res.text(); if (!res.ok) throw new Error(`CP ${method} ${apiPath} -> ${res.status}: ${text.slice(0, 300)}`); return text ? JSON.parse(text) : {}; } async function cpComment(issueId: string, bodyText: string) { if (!issueId) return; try { await cp("POST", `/api/issues/${issueId}/comments`, { body: bodyText }); } catch (e: any) { log("WARN comment failed:", e?.message ?? e); } } function issueLink(issue: Json): string { const identifier = issue.identifier ?? issue.id ?? "issue"; const prefix = String(identifier).split("-")[0] || "PAP"; return `[${identifier}](/${prefix}/issues/${identifier})`; } async function findExistingFailureIssue(title: string): Promise { if (!CP_COMPANY || !ROUTINE_ISSUE_ID) return null; try { const found = await cp("GET", `/api/companies/${CP_COMPANY}/issues?q=${encodeURIComponent(title)}`); const issues = Array.isArray(found) ? found : found.issues ?? []; return findDuplicateFailureIssue(issues, title, ROUTINE_ISSUE_ID); } catch (e: any) { log("WARN duplicate failure lookup failed:", e?.message ?? e); return null; } } // Upload a screenshot as a company asset and return its served URL. Assets are // company-scoped, so the run token can upload one even when the target issue is // assigned to another agent (attachments to an assigned-away issue 403 on the // run-token authorization boundary). Embed the URL in the issue body instead. async function cpUploadAsset(file: string): Promise { if (DRY) { log(`DRY upload asset ${file}`); return `${CP_BASE}/api/assets/dry-run/content`; } try { const buf = await fs.readFile(file); const form = new FormData(); form.append("file", new Blob([buf], { type: "image/png" }), path.basename(file)); form.append("namespace", "smoke-lab"); const res = await fetch(`${CP_BASE}/api/companies/${CP_COMPANY}/assets/images`, { method: "POST", headers: { authorization: `Bearer ${CP_KEY}`, "x-paperclip-run-id": CP_RUN }, body: form as any, }); if (!res.ok) { log("WARN asset upload failed:", res.status, (await res.text()).slice(0, 200)); return null; } const j = await res.json(); return j.contentPath ? `${CP_BASE}${j.contentPath}` : null; } catch (e: any) { log("WARN asset upload error:", e?.message ?? e); return null; } } async function cpAttach(issueId: string, file: string): Promise { if (DRY) { log(`DRY attach ${file} -> ${issueId}`); return true; } try { const buf = await fs.readFile(file); const form = new FormData(); form.append("file", new Blob([buf], { type: "image/png" }), path.basename(file)); const res = await fetch(`${CP_BASE}/api/companies/${CP_COMPANY}/issues/${issueId}/attachments`, { method: "POST", headers: { authorization: `Bearer ${CP_KEY}`, "x-paperclip-run-id": CP_RUN }, body: form as any, }); if (!res.ok) { log("WARN attach failed:", res.status, (await res.text()).slice(0, 200)); return false; } return true; } catch (e: any) { log("WARN attach error:", e?.message ?? e); return false; } } // ---- precondition: is a private (non-public) instance reachable + can we enable the flag? ---- async function precondition(): Promise<{ ok: true } | { ok: false; reason: string }> { let health: Json; try { const res = await fetch(`${SMOKE_BASE}/api/health`, { signal: AbortSignal.timeout(8000) }); if (!res.ok) return { ok: false, reason: `health ${res.status}` }; health = await res.json(); } catch (e: any) { return { ok: false, reason: `instance unreachable at ${SMOKE_BASE} (${e?.message ?? e})` }; } // The Smoke Lab gate only blocks public exposure — any private instance works, // including an `authenticated` dev server behind Tailscale + login. Match that // here so the routine runs on the everyday dev box, not just a throwaway // `local_trusted` instance. if (health.deploymentExposure === "public") { return { ok: false, reason: `exposure=public (need private/loopback)` }; } // Enable the flag (idempotent). On a `local_trusted` box board access is implicit // with the Origin header; on an `authenticated` instance this PATCH needs a board // session, so if it 403s the run degrades to amber/skipped rather than failing. try { const res = await fetch(`${SMOKE_BASE}/api/instance/settings/experimental`, { method: "PATCH", headers: { "content-type": "application/json", origin: SMOKE_BASE }, body: JSON.stringify({ enableSmokeLab: true }), signal: AbortSignal.timeout(8000), }); if (!res.ok) return { ok: false, reason: `cannot enable smoke-lab flag (${res.status})` }; } catch (e: any) { return { ok: false, reason: `flag enable failed (${e?.message ?? e})` }; } return { ok: true }; } function runBrowserSmoke(): Promise<{ code: number | null; timedOut: boolean }> { return new Promise((resolve) => { const child = spawn( process.execPath, ["--experimental-strip-types", RUNNER], { stdio: ["ignore", "inherit", "inherit"], env: { ...process.env, SMOKE_BASE, SMOKE_SHOT_DIR: SHOT_DIR, SMOKE_ONLY: ONLY, SMOKE_TRIGGER: "routine", }, }, ); let timedOut = false; const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, BUDGET_MS); child.on("exit", (code) => { clearTimeout(timer); resolve({ code, timedOut }); }); }); } async function readResult(): Promise { try { return JSON.parse(await fs.readFile(path.join(SHOT_DIR, "result.json"), "utf8")); } catch { return null; } } function ownerForFailure(failed: string[]): { id: string; name: string } { // The runner's failing steps are governance/API behaviours (connect, catalog, // policy, gateway, audit) → S1/S4 coder. UI matrix-render defects surface in the // tab screenshot, not step failures, so the default is the API owner. const anyUi = failed.some((f) => /matrix|dashboard|render|tab/i.test(f)); return anyUi ? { id: OWNER_UI, name: "ClaudeCoder (S2 UI)" } : { id: OWNER_DEFAULT, name: "CodexCoder (S1/S4)" }; } async function recordAmber(reason: string) { log(`AMBER/SKIPPED: ${reason}`); await fs.mkdir(SHOT_DIR, { recursive: true }).catch(() => {}); await fs .writeFile(path.join(SHOT_DIR, "result.json"), JSON.stringify({ outcome: "skipped", reason, at: new Date().toISOString() }, null, 2)) .catch(() => {}); const body = [ `### Smoke Lab routine — amber / skipped`, ``, `The daily Smoke Lab smoke did **not** run this cycle, and is recorded as **skipped** (not a failure).`, ``, `- **Target:** \`${SMOKE_BASE}\``, `- **Reason:** ${reason}`, ``, `This is expected when no private instance is reachable or the flag can't be enabled for the smoke (see \`doc/connections/SMOKE-LAB-BROWSER-RUNNER.md\` §0). No product issue was filed — an amber/skipped run is not a defect.`, ].join("\n"); await cpComment(ROUTINE_ISSUE_ID, body); } async function recordPass(result: Json) { const body = [ `### Smoke Lab routine — ✅ passed`, ``, `- **Target:** \`${SMOKE_BASE}\``, `- **Company (throwaway):** \`${result.companyId}\` · **run:** \`${result.runId}\``, `- **Steps:** ${result.steps} across ${Object.keys(result.byPath ?? {}).length} path(s) — ${JSON.stringify(result.byPath ?? {})}`, `- **Failures:** 0`, ``, `Recorded to the Smoke Lab results API on the throwaway instance (matrix + dashboard card). Tab + dashboard screenshots attached below.`, ].join("\n"); await cpComment(ROUTINE_ISSUE_ID, body); for (const f of ["zz-smoke-lab-tab.png", "zz-dashboard-card.png"]) { await cpAttach(ROUTINE_ISSUE_ID, path.join(SHOT_DIR, f)); } } async function recordForcedFailureDuplicate(result: Json, existing: Json, failed: string[]) { const body = [ `### Smoke Lab routine — forced-failure drill already recorded`, ``, `- **Target:** \`${SMOKE_BASE}\``, `- **Throwaway company:** \`${result.companyId}\` · **run:** \`${result.runId}\``, `- **Synthetic step:** ${failed.map((f) => "`" + f + "`").join(", ")}`, `- **Existing evidence issue:** ${issueLink(existing)}`, ``, `No new product-failure issue was filed. The existing induced-failure issue already proves the D5 file-on-failure path; repeated \`SMOKE_FORCE_FAIL\` runs should not keep waking the owning coder.`, ].join("\n"); await cpComment(ROUTINE_ISSUE_ID, body); } async function recordFailure(result: Json): Promise<"filed" | "suppressed_duplicate"> { const failed: string[] = result.failed ?? []; const owner = ownerForFailure(failed); const first = failed[0] ?? "unknown step"; const [pathStep] = first.split(":"); const [pathId, step] = (pathStep ?? "").split("/"); const title = `Smoke Lab failure: ${pathId ?? "path"} / ${step ?? "step"}`; if (isForcedFailureSelfTest(failed)) { const existing = await findExistingFailureIssue(title); if (existing) { await recordForcedFailureDuplicate(result, existing, failed); return "suppressed_duplicate"; } } const shotFile = path.join(SHOT_DIR, `${(pathId ?? "").toLowerCase()}-${step ?? ""}-failed.png`); const shotUrl = await cpUploadAsset(shotFile); const bodyLines = [ `## Smoke Lab smoke failed — ${pathId ?? "path"} / ${step ?? "step"}`, ``, `Auto-filed by the daily Smoke Lab routine (PAP-13351 / plan §D5). A real product failure was observed while driving the P1–P7 lifecycle in a real browser against a \`local_trusted\` instance.`, ``, `### Failing step(s)`, "```", ...failed.map((f) => `- ${f}`), "```", ``, ...(shotUrl ? [`### Screenshot of the failing step`, `![failing step](${shotUrl})`, ``] : []), `### Repro`, `1. Boot a throwaway \`local_trusted\` instance (see \`doc/connections/SMOKE-LAB-BROWSER-RUNNER.md\` §0).`, `2. \`SMOKE_BASE= SMOKE_ONLY=${pathId ?? ""} node --experimental-strip-types tests/e2e/smoke-lab-browser-runner.mts\``, `3. Observe the failure above; the failing step's screenshot is attached.`, ``, `### Context`, `- **Throwaway company:** \`${result.companyId}\` · **run:** \`${result.runId}\` · **status:** \`${result.status}\``, `- **Suggested owner:** ${owner.name}. If this is the wrong surface, reassign — a governance/API step failure is S1/S4; a matrix/dashboard render defect is S2.`, `- **Parent / linked:** routine issue \`${ROUTINE_ISSUE_ID}\`.`, ]; let issueId = ""; try { const created = await cp("POST", `/api/companies/${CP_COMPANY}/issues`, { title, description: bodyLines.join("\n"), priority: "high", assigneeAgentId: owner.id, parentId: ROUTINE_ISSUE_ID || undefined, }); issueId = created.id ?? created.issue?.id ?? ""; log(`filed failure issue ${issueId} (assignee ${owner.name}; screenshot ${shotUrl ? "embedded" : "unavailable"})`); } catch (e: any) { log("ERROR filing failure issue:", e?.message ?? e); } // Link back on the routine issue. await cpComment( ROUTINE_ISSUE_ID, [ `### Smoke Lab routine — ❌ failed`, ``, `- **Target:** \`${SMOKE_BASE}\``, `- **Failing step(s):** ${failed.map((f) => "`" + f + "`").join(", ")}`, `- **Filed:** ${issueId ? `issue \`${issueId}\` assigned to ${owner.name}` : "(issue filing failed — see logs)"}`, ].join("\n"), ); return "filed"; } async function main() { await fs.mkdir(SHOT_DIR, { recursive: true }); log(`base=${SMOKE_BASE} only=${ONLY || "(all)"} budgetMs=${BUDGET_MS} shots=${SHOT_DIR} dry=${DRY}`); const pre = await precondition(); if (!pre.ok) { await recordAmber(pre.reason); process.exit(0); } const { code, timedOut } = await runBrowserSmoke(); const result = await readResult(); if (timedOut) { await recordAmber(`browser smoke exceeded budget of ${BUDGET_MS}ms and was killed`); process.exit(0); } // exit 0 = clean pass; exit 2 = recorded step failure(s); anything else = runner crashed. if (code === 0 && result && !(result.failed?.length)) { await recordPass(result); process.exit(0); } if (code === 2 && result?.failed?.length) { const outcome = await recordFailure(result); process.exit(outcome === "suppressed_duplicate" ? 0 : 2); } // Runner crashed before recording a clean pass/fail → environment/runner problem, not a // product defect. Amber, don't file a bogus bug (matches the S5 runbook triage rule). await recordAmber(`runner exited ${code} without a recorded pass/fail (environment/runner issue, not a product defect)`); process.exit(0); } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { main().catch(async (e) => { await recordAmber(`routine wrapper crashed: ${e?.message ?? e}`); process.exit(0); }); }