/** * Smoke Lab agent-driven browser runner (PAP-13350 / plan §D4 item 1). * * This is the reference implementation the `smoke-lab-browser-runner` runbook * points at: a QA agent drives a REAL browser through the P1-P7 Smoke Lab * lifecycle against a live local_trusted instance, typing demo credentials into * the fake OAuth consent page, and posts every step + a viewable screenshot to * the S1 results API so the run lands in the Smoke Lab UI + dashboard card. * * It deliberately imports the S4 catalog as the single source of truth for the * scenario/step list (do NOT fork the step list). * * Run: * node --experimental-strip-types tests/e2e/smoke-lab-browser-runner.mts * Env (all optional except a reachable BASE): * SMOKE_BASE=http://127.0.0.1:3211 target local_trusted, non-prod instance * SMOKE_COMPANY_ID= reuse a company (else creates one) * SMOKE_SHOT_DIR=/tmp/pap13350-shots screenshot output dir * SMOKE_ONLY=P1,P3 restrict to a subset of catalog paths * (budget-bounding for the daily D5 routine; * selects scenarios, does NOT fork the steps) * SMOKE_TRIGGER=manual|ci|routine smoke_run trigger (default "manual") * SMOKE_CHROMIUM_PATH=/path/to/chromium optional browser executable override * SMOKE_FORCE_FAIL= inject one synthetic failing step on the * first scenario — the canonical self-test for * the §D5 routine's file-on-failure wiring. */ import { promises as fs } from "node:fs"; import { chromium } from "playwright"; import { ciSmokeLabScenarios } from "./smoke-lab.catalog.ts"; const BASE = (process.env.SMOKE_BASE ?? "http://127.0.0.1:3211").replace(/\/$/, ""); const SHOT_DIR = process.env.SMOKE_SHOT_DIR ?? "/tmp/pap13350-shots"; const CHROMIUM_PATH = process.env.SMOKE_CHROMIUM_PATH?.trim() || null; const DEMO_EMAIL = "smoke@paperclip.test"; const DEMO_PASSWORD = "smoke-password"; const ONLY = (process.env.SMOKE_ONLY ?? "") .split(",") .map((s) => s.trim().toUpperCase()) .filter(Boolean); const TRIGGER = process.env.SMOKE_TRIGGER ?? "manual"; type Json = Record; function assert(cond: any, msg: string): asserts cond { if (!cond) throw new Error(`ASSERT FAILED: ${msg}`); } async function api(method: string, path: string, body?: Json): Promise { const res = await fetch(`${BASE}${path}`, { method, headers: { "content-type": "application/json", origin: BASE }, body: body === undefined ? undefined : JSON.stringify(body), }); const text = await res.text(); if (!res.ok) throw new Error(`${method} ${path} -> ${res.status}: ${text.slice(0, 300)}`); return text ? JSON.parse(text) : {}; } async function uploadShot(companyId: string, file: string): Promise { const buf = await fs.readFile(file); const form = new FormData(); form.append("file", new Blob([buf], { type: "image/png" }), file.split("/").pop()); form.append("namespace", "smoke-lab"); const res = await fetch(`${BASE}/api/companies/${companyId}/assets/images`, { method: "POST", headers: { origin: BASE }, body: form as any, }); const text = await res.text(); if (!res.ok) throw new Error(`asset upload failed ${res.status}: ${text.slice(0, 200)}`); const j = text ? JSON.parse(text) : {}; return `${BASE}${j.contentPath}`; } async function main() { await fs.mkdir(SHOT_DIR, { recursive: true }); // Company + scout let companyId = process.env.SMOKE_COMPANY_ID ?? ""; let prefix = ""; if (!companyId) { const co = await api("POST", "/api/companies", { name: `Smoke Lab Browser Run ${Date.now()}` }); companyId = co.id; prefix = co.issuePrefix ?? co.urlKey ?? "SMO"; } else { const list = await api("GET", "/api/companies"); const co = (Array.isArray(list) ? list : list.companies ?? []).find((c: Json) => c.id === companyId); prefix = co?.issuePrefix ?? co?.urlKey ?? "SMO"; } await api("PATCH", "/api/instance/settings/experimental", { enableSmokeLab: true }); const scout = await api("POST", `/api/companies/${companyId}/agents`, { name: `Smoke Scout ${Date.now()}`, role: "qa", title: "Smoke Lab scout", capabilities: "Runs deterministic Smoke Lab fixture calls.", adapterType: "process", adapterConfig: { command: "node", args: ["-e", "setTimeout(()=>{},1000)"] }, }); console.log(`company=${companyId} prefix=${prefix} scout=${scout.id}`); const scenarios = ONLY.length ? ciSmokeLabScenarios.filter((s) => ONLY.includes(s.path)) : ciSmokeLabScenarios; assert(scenarios.length, `SMOKE_ONLY=${ONLY.join(",")} matched no catalog paths`); const run = (await api("POST", `/api/companies/${companyId}/smoke-lab/runs`, { trigger: TRIGGER, summary: { catalog: "tests/e2e/smoke-lab.catalog.ts", driver: "agent-browser (real chromium)", track: "D4 agent-driven browser", scenarioCount: scenarios.length, only: ONLY.length ? ONLY : undefined }, })).run; console.log(`smoke_run=${run.id}`); const browser = await chromium.launch({ ...(CHROMIUM_PATH ? { executablePath: CHROMIUM_PATH } : {}), headless: true, }); const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 }); const page = await ctx.newPage(); const failed: string[] = []; const shot = async (scenario: Json, step: string) => { const file = `${SHOT_DIR}/${scenario.path.toLowerCase()}-${step}.png`; await page.screenshot({ path: file, fullPage: true }); return await uploadShot(companyId, file); }; const record = async (scenario: Json, step: string, status: string, detail: string | null, url: string | null, ms: number) => { await api("POST", `/api/companies/${companyId}/smoke-lab/runs/${run.id}/steps`, { path: scenario.path, scenarioStep: step, status, detail, screenshotArtifactRef: url ? { kind: "agent_browser_screenshot", url } : null, durationMs: ms, }); }; const doStep = async (scenario: Json, step: string, action: () => Promise) => { const t0 = Date.now(); try { const hint = await action(); const url = await shot(scenario, step); await record(scenario, step, "pass", hint ?? null, url, Date.now() - t0); console.log(` ${scenario.path}/${step} PASS`); } catch (e: any) { let url: string | null = null; try { url = await shot(scenario, `${step}-failed`); } catch {} await record(scenario, step, "fail", e?.message ?? String(e), url, Date.now() - t0).catch(() => {}); failed.push(`${scenario.path}/${step}: ${e?.message ?? e}`); console.log(` ${scenario.path}/${step} FAIL: ${e?.message ?? e}`); } }; const gotoUI = async (scenario: Json, connId: string) => { const p = scenario.uiEntryPath; if (p === "advanced") await page.goto(`${BASE}/${prefix}/apps/advanced`, { waitUntil: "networkidle" }); else if (p === "review") await page.goto(`${BASE}/${prefix}/apps/${connId}/review`, { waitUntil: "networkidle" }); else if (p === "activity") await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" }); else if (p === "attention") await page.goto(`${BASE}/${prefix}/apps/attention`, { waitUntil: "networkidle" }); else await page.goto(`${BASE}/${prefix}/apps/${connId}`, { waitUntil: "networkidle" }); }; const auditHit = async (connId: string, search: string) => { const a = await api("GET", `/api/tool-gateway/audit?companyId=${companyId}&app=${connId}&agent=${scout.id}&search=${encodeURIComponent(search)}&limit=50`); assert((a.events?.length ?? 0) > 0, `audit row for ${search}`); }; for (const scenario of scenarios) { console.log(`--- ${scenario.path}: ${scenario.title} ---`); // Fixtures (board UI action; idempotent). Start services then install. const services = (await api("POST", `/api/companies/${companyId}/smoke-lab/services/start`)).services; const oauthUrl = services.find((s: Json) => s.id === "fake-oauth")?.url as string; const fx = await api("POST", `/api/companies/${companyId}/smoke-lab/install-fixtures`); const preferStdio = scenario.transport === "local_stdio" || scenario.transport === "plugin"; const wantTransport = preferStdio ? "local_stdio" : "mcp_remote"; const conn = fx.connections.find((c: Json) => c.transport === wantTransport); assert(conn, `${wantTransport} connection for ${scenario.path}`); // connect await doStep(scenario, "connect", async () => { if (scenario.authMode === "oauth") { // REAL BROWSER: fake OAuth login + consent page, type demo credentials. const authorize = `${oauthUrl}?client_id=smoke-client&redirect_uri=${encodeURIComponent("http://127.0.0.1/callback")}&scope=${encodeURIComponent("smoke:openid smoke:profile smoke:email")}&state=smoke-state&response_type=code`; await page.goto(authorize, { waitUntil: "domcontentloaded" }); await page.getByRole("heading", { name: /Smoke OAuth/i }).waitFor({ timeout: 15_000 }); await page.fill('input[name="email"]', DEMO_EMAIL); await page.fill('input[name="password"]', DEMO_PASSWORD); // Screenshot the filled consent page as evidence before submit. const consentUrl = await shot(scenario, "connect-consent"); await record(scenario, "connect-consent", "pass", `Typed demo creds ${DEMO_EMAIL} into fake OAuth consent page`, consentUrl, 0); // Submit consent; capture the 302 to prove the fake provider accepted the // demo creds (redirect target is a dead loopback callback that never loads). const [resp] = await Promise.all([ page.waitForResponse((r: any) => r.url().includes("/smoke-lab/oauth/authorize") && r.request().method() === "POST", { timeout: 12_000 }), page.click('button[type="submit"]'), ]); const loc = resp.headers()["location"] ?? ""; assert(resp.status() === 302 && /[?&]code=/.test(loc), `OAuth consent accepted creds → 302 with code (status ${resp.status()}, location ${loc.slice(0, 80)})`); // The 302 target is a dead loopback callback; let that failed navigation // commit (chrome-error page) before navigating into the Paperclip UI. await page.waitForLoadState("domcontentloaded").catch(() => {}); await page.waitForTimeout(800); await gotoUI(scenario, conn.id); return `${scenario.lifecycle.connect} (typed demo creds into consent page; provider issued code via 302)`; } await gotoUI(scenario, conn.id); return scenario.lifecycle.connect; }); // discover-catalog await doStep(scenario, "discover-catalog", async () => { const cat = await api("GET", `/api/tool-connections/${conn.id}/catalog`); const names = (cat.catalog ?? []).map((e: Json) => e.toolName); assert(names.includes(scenario.lifecycle.allowedRead.name), `catalog has ${scenario.lifecycle.allowedRead.name}`); await gotoUI(scenario, conn.id); return `${scenario.lifecycle.discoverCatalog}: ${names.length} entries`; }); // allowed-read await doStep(scenario, "allowed-read", async () => { const read = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, { agentId: scout.id, toolName: scenario.lifecycle.allowedRead.name, parameters: scenario.lifecycle.allowedRead.parameters, }); assert(read.decision === "allowed", `allowed-read decision=${read.decision}`); assert(!read.error, "allowed-read no error"); await auditHit(conn.id, scenario.lifecycle.allowedRead.name); await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" }); return `Allowed read ${scenario.lifecycle.allowedRead.name}`; }); // ask-first-write-approved await doStep(scenario, "ask-first-write-approved", async () => { await api("POST", `/api/companies/${companyId}/tools/policies`, { name: `${scenario.path} require approval ${Date.now()}`, policyType: "require_approval", priority: 10, selectors: { connectionId: conn.id, toolNames: [scenario.lifecycle.askFirstWrite.name] }, }); const pending = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, { agentId: scout.id, toolName: scenario.lifecycle.askFirstWrite.name, parameters: scenario.lifecycle.askFirstWrite.parameters, }); assert(pending.decision === "ask_first", `ask-first decision=${pending.decision}`); assert(pending.actionRequestId, "ask-first has actionRequestId"); await page.goto(`${BASE}/${prefix}/apps/${conn.id}/review`, { waitUntil: "networkidle" }); await api("POST", `/api/tool-gateway/action-requests/${pending.actionRequestId}/approve`, { companyId }); let approved = false; for (let i = 0; i < 40; i++) { const st = await api("GET", `/api/tool-connections/${conn.id}/test-calls/${pending.actionRequestId}`); if (st.phase === "done") { approved = true; break; } await new Promise((r) => setTimeout(r, 250)); } assert(approved, `Timed out waiting for test-call ${pending.actionRequestId} phase done`); return `Approved ask-first call ${scenario.lifecycle.askFirstWrite.name}`; }); // denied-blocked-call await doStep(scenario, "denied-blocked-call", async () => { await api("POST", `/api/companies/${companyId}/tools/policies`, { name: `${scenario.path} block ${Date.now()}`, policyType: "block", priority: 1, selectors: { connectionId: conn.id, toolNames: [scenario.lifecycle.deniedCall.name] }, }); const denied = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, { agentId: scout.id, toolName: scenario.lifecycle.deniedCall.name, parameters: scenario.lifecycle.deniedCall.parameters, }); assert(denied.decision === "off", `denied decision=${denied.decision}`); assert(denied.error?.reasonCode, "denied has reasonCode"); await page.goto(`${BASE}/${prefix}/apps/${conn.id}/review`, { waitUntil: "networkidle" }); return `Blocked call ${scenario.lifecycle.deniedCall.name}: ${denied.error?.reasonCode}`; }); // schema-change-quarantine await doStep(scenario, "schema-change-quarantine", async () => { if (conn.transport !== "mcp_remote") { await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" }); return "Non-HTTP path records governance/quarantine evidence through fixture metadata."; } await api("PATCH", `/api/tool-connections/${conn.id}`, { config: { ...(conn.config ?? {}), quarantineNewEntries: true } }); await api("POST", `/api/companies/${companyId}/tools/policies`, { name: `${scenario.path} allow schema flip ${Date.now()}`, policyType: "allow", priority: 5, selectors: { connectionId: conn.id, toolNames: [scenario.lifecycle.schemaChangeQuarantine.name] }, }); const flipped = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, { agentId: scout.id, toolName: scenario.lifecycle.schemaChangeQuarantine.name, parameters: scenario.lifecycle.schemaChangeQuarantine.parameters, }); assert(flipped.decision === "allowed", `schema-flip decision=${flipped.decision}`); const refresh = await api("POST", `/api/tool-connections/${conn.id}/catalog/refresh`); assert((refresh.quarantinedCount ?? 0) > 0, `quarantinedCount=${refresh.quarantinedCount}`); await page.goto(`${BASE}/${prefix}/apps/attention`, { waitUntil: "networkidle" }); return `Catalog refresh quarantined ${refresh.quarantinedCount} changed entries.`; }); // revoke await doStep(scenario, "revoke", async () => { if (scenario.transport === "gateway_session") { const invoked = await api("POST", `/api/agents/${scout.id}/heartbeat/invoke`); const session = await api("POST", "/api/tool-gateway/sessions", { companyId, agentId: scout.id, runId: invoked.id, ttlMs: 60_000 }); const listRes = await fetch(`${BASE}${new URL(session.toolsUrl, BASE).pathname}`, { headers: { "x-paperclip-tool-gateway-token": session.token } }); assert(listRes.ok, "gateway tools list ok pre-revoke"); await api("POST", `/api/tool-gateway/sessions/${session.sessionId}/revoke`, { companyId }); const after = await fetch(`${BASE}${new URL(session.toolsUrl, BASE).pathname}`, { headers: { "x-paperclip-tool-gateway-token": session.token } }); assert(after.status === 401, `revoked token cut off (got ${after.status})`); await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" }); return scenario.lifecycle.revoke; } const disabled = await api("PATCH", `/api/tool-connections/${conn.id}`, { enabled: false }); assert(disabled.enabled === false, "connection disabled"); await page.goto(`${BASE}/${prefix}/apps/${conn.id}`, { waitUntil: "networkidle" }); await api("PATCH", `/api/tool-connections/${conn.id}`, { enabled: true }); return scenario.lifecycle.revoke; }); // audit-evidence await doStep(scenario, "audit-evidence", async () => { await auditHit(conn.id, scenario.lifecycle.allowedRead.name); await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" }); return scenario.lifecycle.auditEvidence; }); } // Optional injected failure — the canonical self-test for the §D5 routine's // file-on-failure wiring. Records a real fail step + screenshot so the wrapper // sees exit 2 + a `--failed.png` to attach. if (process.env.SMOKE_FORCE_FAIL) { const target = scenarios[0]; await doStep(target, "forced-failure", async () => { throw new Error(`SMOKE_FORCE_FAIL: ${process.env.SMOKE_FORCE_FAIL}`); }); } // Finalize run + capture the run in the live Smoke Lab UI. await api("PATCH", `/api/companies/${companyId}/smoke-lab/runs/${run.id}`, { status: failed.length ? "failed" : "passed", summary: { catalog: "tests/e2e/smoke-lab.catalog.ts", driver: "agent-browser (real chromium)", scenarioCount: scenarios.length, only: ONLY.length ? ONLY : undefined, failed }, }); await page.goto(`${BASE}/${prefix}/apps/advanced/smoke-lab`, { waitUntil: "networkidle" }); await page.waitForTimeout(1500); const tabFile = `${SHOT_DIR}/zz-smoke-lab-tab.png`; await page.screenshot({ path: tabFile, fullPage: true }); await page.goto(`${BASE}/${prefix}/dashboard`, { waitUntil: "networkidle" }); await page.waitForTimeout(1500); const dashFile = `${SHOT_DIR}/zz-dashboard-card.png`; await page.screenshot({ path: dashFile, fullPage: true }); await browser.close(); const finalRun = await api("GET", `/api/companies/${companyId}/smoke-lab/runs/${run.id}`); const steps = finalRun.steps ?? []; const byPath: Record = {}; for (const s of steps) byPath[s.path] = (byPath[s.path] ?? 0) + 1; console.log("\n=== RESULT ==="); console.log(`companyId=${companyId} prefix=${prefix} runId=${run.id} status=${finalRun.run?.status}`); console.log(`steps=${steps.length} perPath=${JSON.stringify(byPath)} failed=${failed.length}`); console.log(`withScreenshot=${steps.filter((s: Json) => s.screenshotArtifactRef?.url).length}/${steps.length}`); console.log(`tabShot=${tabFile} dashShot=${dashFile}`); await fs.writeFile(`${SHOT_DIR}/result.json`, JSON.stringify({ companyId, prefix, runId: run.id, status: finalRun.run?.status, steps: steps.length, byPath, failed }, null, 2)); if (failed.length) process.exitCode = 2; } main().catch((e) => { console.error("RUNNER ERROR", e); process.exit(1); });