353 lines
19 KiB
TypeScript
353 lines
19 KiB
TypeScript
/**
|
|
* Smoke Lab agent-driven browser runner (PAP-13350 / plan §D4 item 1).
|
|
*
|
|
* This is the reference implementation the `smoke-lab-browser-runner` runbook
|
|
* points at: a QA agent drives a REAL browser through the P1-P7 Smoke Lab
|
|
* lifecycle against a live local_trusted instance, typing demo credentials into
|
|
* the fake OAuth consent page, and posts every step + a viewable screenshot to
|
|
* the S1 results API so the run lands in the Smoke Lab UI + dashboard card.
|
|
*
|
|
* It deliberately imports the S4 catalog as the single source of truth for the
|
|
* scenario/step list (do NOT fork the step list).
|
|
*
|
|
* Run:
|
|
* node --experimental-strip-types tests/e2e/smoke-lab-browser-runner.mts
|
|
* Env (all optional except a reachable BASE):
|
|
* SMOKE_BASE=http://127.0.0.1:3211 target local_trusted, non-prod instance
|
|
* SMOKE_COMPANY_ID=<uuid> reuse a company (else creates one)
|
|
* SMOKE_SHOT_DIR=/tmp/pap13350-shots screenshot output dir
|
|
* SMOKE_ONLY=P1,P3 restrict to a subset of catalog paths
|
|
* (budget-bounding for the daily D5 routine;
|
|
* selects scenarios, does NOT fork the steps)
|
|
* SMOKE_TRIGGER=manual|ci|routine smoke_run trigger (default "manual")
|
|
* SMOKE_CHROMIUM_PATH=/path/to/chromium optional browser executable override
|
|
* SMOKE_FORCE_FAIL=<detail> inject one synthetic failing step on the
|
|
* first scenario — the canonical self-test for
|
|
* the §D5 routine's file-on-failure wiring.
|
|
*/
|
|
import { promises as fs } from "node:fs";
|
|
import { chromium } from "playwright";
|
|
import { ciSmokeLabScenarios } from "./smoke-lab.catalog.ts";
|
|
|
|
const BASE = (process.env.SMOKE_BASE ?? "http://127.0.0.1:3211").replace(/\/$/, "");
|
|
const SHOT_DIR = process.env.SMOKE_SHOT_DIR ?? "/tmp/pap13350-shots";
|
|
const CHROMIUM_PATH = process.env.SMOKE_CHROMIUM_PATH?.trim() || null;
|
|
const DEMO_EMAIL = "smoke@paperclip.test";
|
|
const DEMO_PASSWORD = "smoke-password";
|
|
const ONLY = (process.env.SMOKE_ONLY ?? "")
|
|
.split(",")
|
|
.map((s) => s.trim().toUpperCase())
|
|
.filter(Boolean);
|
|
const TRIGGER = process.env.SMOKE_TRIGGER ?? "manual";
|
|
|
|
type Json = Record<string, any>;
|
|
|
|
function assert(cond: any, msg: string): asserts cond {
|
|
if (!cond) throw new Error(`ASSERT FAILED: ${msg}`);
|
|
}
|
|
|
|
async function api(method: string, path: string, body?: Json): Promise<any> {
|
|
const res = await fetch(`${BASE}${path}`, {
|
|
method,
|
|
headers: { "content-type": "application/json", origin: BASE },
|
|
body: body === undefined ? undefined : JSON.stringify(body),
|
|
});
|
|
const text = await res.text();
|
|
if (!res.ok) throw new Error(`${method} ${path} -> ${res.status}: ${text.slice(0, 300)}`);
|
|
return text ? JSON.parse(text) : {};
|
|
}
|
|
|
|
async function uploadShot(companyId: string, file: string): Promise<string> {
|
|
const buf = await fs.readFile(file);
|
|
const form = new FormData();
|
|
form.append("file", new Blob([buf], { type: "image/png" }), file.split("/").pop());
|
|
form.append("namespace", "smoke-lab");
|
|
const res = await fetch(`${BASE}/api/companies/${companyId}/assets/images`, {
|
|
method: "POST",
|
|
headers: { origin: BASE },
|
|
body: form as any,
|
|
});
|
|
const text = await res.text();
|
|
if (!res.ok) throw new Error(`asset upload failed ${res.status}: ${text.slice(0, 200)}`);
|
|
const j = text ? JSON.parse(text) : {};
|
|
return `${BASE}${j.contentPath}`;
|
|
}
|
|
|
|
async function main() {
|
|
await fs.mkdir(SHOT_DIR, { recursive: true });
|
|
|
|
// Company + scout
|
|
let companyId = process.env.SMOKE_COMPANY_ID ?? "";
|
|
let prefix = "";
|
|
if (!companyId) {
|
|
const co = await api("POST", "/api/companies", { name: `Smoke Lab Browser Run ${Date.now()}` });
|
|
companyId = co.id;
|
|
prefix = co.issuePrefix ?? co.urlKey ?? "SMO";
|
|
} else {
|
|
const list = await api("GET", "/api/companies");
|
|
const co = (Array.isArray(list) ? list : list.companies ?? []).find((c: Json) => c.id === companyId);
|
|
prefix = co?.issuePrefix ?? co?.urlKey ?? "SMO";
|
|
}
|
|
await api("PATCH", "/api/instance/settings/experimental", { enableSmokeLab: true });
|
|
const scout = await api("POST", `/api/companies/${companyId}/agents`, {
|
|
name: `Smoke Scout ${Date.now()}`, role: "qa", title: "Smoke Lab scout",
|
|
capabilities: "Runs deterministic Smoke Lab fixture calls.",
|
|
adapterType: "process", adapterConfig: { command: "node", args: ["-e", "setTimeout(()=>{},1000)"] },
|
|
});
|
|
console.log(`company=${companyId} prefix=${prefix} scout=${scout.id}`);
|
|
|
|
const scenarios = ONLY.length
|
|
? ciSmokeLabScenarios.filter((s) => ONLY.includes(s.path))
|
|
: ciSmokeLabScenarios;
|
|
assert(scenarios.length, `SMOKE_ONLY=${ONLY.join(",")} matched no catalog paths`);
|
|
|
|
const run = (await api("POST", `/api/companies/${companyId}/smoke-lab/runs`, {
|
|
trigger: TRIGGER,
|
|
summary: { catalog: "tests/e2e/smoke-lab.catalog.ts", driver: "agent-browser (real chromium)", track: "D4 agent-driven browser", scenarioCount: scenarios.length, only: ONLY.length ? ONLY : undefined },
|
|
})).run;
|
|
console.log(`smoke_run=${run.id}`);
|
|
|
|
const browser = await chromium.launch({
|
|
...(CHROMIUM_PATH ? { executablePath: CHROMIUM_PATH } : {}),
|
|
headless: true,
|
|
});
|
|
const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 });
|
|
const page = await ctx.newPage();
|
|
const failed: string[] = [];
|
|
|
|
const shot = async (scenario: Json, step: string) => {
|
|
const file = `${SHOT_DIR}/${scenario.path.toLowerCase()}-${step}.png`;
|
|
await page.screenshot({ path: file, fullPage: true });
|
|
return await uploadShot(companyId, file);
|
|
};
|
|
const record = async (scenario: Json, step: string, status: string, detail: string | null, url: string | null, ms: number) => {
|
|
await api("POST", `/api/companies/${companyId}/smoke-lab/runs/${run.id}/steps`, {
|
|
path: scenario.path, scenarioStep: step, status, detail,
|
|
screenshotArtifactRef: url ? { kind: "agent_browser_screenshot", url } : null,
|
|
durationMs: ms,
|
|
});
|
|
};
|
|
const doStep = async (scenario: Json, step: string, action: () => Promise<string | null | undefined>) => {
|
|
const t0 = Date.now();
|
|
try {
|
|
const hint = await action();
|
|
const url = await shot(scenario, step);
|
|
await record(scenario, step, "pass", hint ?? null, url, Date.now() - t0);
|
|
console.log(` ${scenario.path}/${step} PASS`);
|
|
} catch (e: any) {
|
|
let url: string | null = null;
|
|
try { url = await shot(scenario, `${step}-failed`); } catch {}
|
|
await record(scenario, step, "fail", e?.message ?? String(e), url, Date.now() - t0).catch(() => {});
|
|
failed.push(`${scenario.path}/${step}: ${e?.message ?? e}`);
|
|
console.log(` ${scenario.path}/${step} FAIL: ${e?.message ?? e}`);
|
|
}
|
|
};
|
|
|
|
const gotoUI = async (scenario: Json, connId: string) => {
|
|
const p = scenario.uiEntryPath;
|
|
if (p === "advanced") await page.goto(`${BASE}/${prefix}/apps/advanced`, { waitUntil: "networkidle" });
|
|
else if (p === "review") await page.goto(`${BASE}/${prefix}/apps/${connId}/review`, { waitUntil: "networkidle" });
|
|
else if (p === "activity") await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" });
|
|
else if (p === "attention") await page.goto(`${BASE}/${prefix}/apps/attention`, { waitUntil: "networkidle" });
|
|
else await page.goto(`${BASE}/${prefix}/apps/${connId}`, { waitUntil: "networkidle" });
|
|
};
|
|
|
|
const auditHit = async (connId: string, search: string) => {
|
|
const a = await api("GET", `/api/tool-gateway/audit?companyId=${companyId}&app=${connId}&agent=${scout.id}&search=${encodeURIComponent(search)}&limit=50`);
|
|
assert((a.events?.length ?? 0) > 0, `audit row for ${search}`);
|
|
};
|
|
|
|
for (const scenario of scenarios) {
|
|
console.log(`--- ${scenario.path}: ${scenario.title} ---`);
|
|
// Fixtures (board UI action; idempotent). Start services then install.
|
|
const services = (await api("POST", `/api/companies/${companyId}/smoke-lab/services/start`)).services;
|
|
const oauthUrl = services.find((s: Json) => s.id === "fake-oauth")?.url as string;
|
|
const fx = await api("POST", `/api/companies/${companyId}/smoke-lab/install-fixtures`);
|
|
const preferStdio = scenario.transport === "local_stdio" || scenario.transport === "plugin";
|
|
const wantTransport = preferStdio ? "local_stdio" : "mcp_remote";
|
|
const conn = fx.connections.find((c: Json) => c.transport === wantTransport);
|
|
assert(conn, `${wantTransport} connection for ${scenario.path}`);
|
|
|
|
// connect
|
|
await doStep(scenario, "connect", async () => {
|
|
if (scenario.authMode === "oauth") {
|
|
// REAL BROWSER: fake OAuth login + consent page, type demo credentials.
|
|
const authorize = `${oauthUrl}?client_id=smoke-client&redirect_uri=${encodeURIComponent("http://127.0.0.1/callback")}&scope=${encodeURIComponent("smoke:openid smoke:profile smoke:email")}&state=smoke-state&response_type=code`;
|
|
await page.goto(authorize, { waitUntil: "domcontentloaded" });
|
|
await page.getByRole("heading", { name: /Smoke OAuth/i }).waitFor({ timeout: 15_000 });
|
|
await page.fill('input[name="email"]', DEMO_EMAIL);
|
|
await page.fill('input[name="password"]', DEMO_PASSWORD);
|
|
// Screenshot the filled consent page as evidence before submit.
|
|
const consentUrl = await shot(scenario, "connect-consent");
|
|
await record(scenario, "connect-consent", "pass", `Typed demo creds ${DEMO_EMAIL} into fake OAuth consent page`, consentUrl, 0);
|
|
// Submit consent; capture the 302 to prove the fake provider accepted the
|
|
// demo creds (redirect target is a dead loopback callback that never loads).
|
|
const [resp] = await Promise.all([
|
|
page.waitForResponse((r: any) => r.url().includes("/smoke-lab/oauth/authorize") && r.request().method() === "POST", { timeout: 12_000 }),
|
|
page.click('button[type="submit"]'),
|
|
]);
|
|
const loc = resp.headers()["location"] ?? "";
|
|
assert(resp.status() === 302 && /[?&]code=/.test(loc), `OAuth consent accepted creds → 302 with code (status ${resp.status()}, location ${loc.slice(0, 80)})`);
|
|
// The 302 target is a dead loopback callback; let that failed navigation
|
|
// commit (chrome-error page) before navigating into the Paperclip UI.
|
|
await page.waitForLoadState("domcontentloaded").catch(() => {});
|
|
await page.waitForTimeout(800);
|
|
await gotoUI(scenario, conn.id);
|
|
return `${scenario.lifecycle.connect} (typed demo creds into consent page; provider issued code via 302)`;
|
|
}
|
|
await gotoUI(scenario, conn.id);
|
|
return scenario.lifecycle.connect;
|
|
});
|
|
|
|
// discover-catalog
|
|
await doStep(scenario, "discover-catalog", async () => {
|
|
const cat = await api("GET", `/api/tool-connections/${conn.id}/catalog`);
|
|
const names = (cat.catalog ?? []).map((e: Json) => e.toolName);
|
|
assert(names.includes(scenario.lifecycle.allowedRead.name), `catalog has ${scenario.lifecycle.allowedRead.name}`);
|
|
await gotoUI(scenario, conn.id);
|
|
return `${scenario.lifecycle.discoverCatalog}: ${names.length} entries`;
|
|
});
|
|
|
|
// allowed-read
|
|
await doStep(scenario, "allowed-read", async () => {
|
|
const read = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, {
|
|
agentId: scout.id, toolName: scenario.lifecycle.allowedRead.name, parameters: scenario.lifecycle.allowedRead.parameters,
|
|
});
|
|
assert(read.decision === "allowed", `allowed-read decision=${read.decision}`);
|
|
assert(!read.error, "allowed-read no error");
|
|
await auditHit(conn.id, scenario.lifecycle.allowedRead.name);
|
|
await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" });
|
|
return `Allowed read ${scenario.lifecycle.allowedRead.name}`;
|
|
});
|
|
|
|
// ask-first-write-approved
|
|
await doStep(scenario, "ask-first-write-approved", async () => {
|
|
await api("POST", `/api/companies/${companyId}/tools/policies`, {
|
|
name: `${scenario.path} require approval ${Date.now()}`, policyType: "require_approval", priority: 10,
|
|
selectors: { connectionId: conn.id, toolNames: [scenario.lifecycle.askFirstWrite.name] },
|
|
});
|
|
const pending = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, {
|
|
agentId: scout.id, toolName: scenario.lifecycle.askFirstWrite.name, parameters: scenario.lifecycle.askFirstWrite.parameters,
|
|
});
|
|
assert(pending.decision === "ask_first", `ask-first decision=${pending.decision}`);
|
|
assert(pending.actionRequestId, "ask-first has actionRequestId");
|
|
await page.goto(`${BASE}/${prefix}/apps/${conn.id}/review`, { waitUntil: "networkidle" });
|
|
await api("POST", `/api/tool-gateway/action-requests/${pending.actionRequestId}/approve`, { companyId });
|
|
let approved = false;
|
|
for (let i = 0; i < 40; i++) {
|
|
const st = await api("GET", `/api/tool-connections/${conn.id}/test-calls/${pending.actionRequestId}`);
|
|
if (st.phase === "done") {
|
|
approved = true;
|
|
break;
|
|
}
|
|
await new Promise((r) => setTimeout(r, 250));
|
|
}
|
|
assert(approved, `Timed out waiting for test-call ${pending.actionRequestId} phase done`);
|
|
return `Approved ask-first call ${scenario.lifecycle.askFirstWrite.name}`;
|
|
});
|
|
|
|
// denied-blocked-call
|
|
await doStep(scenario, "denied-blocked-call", async () => {
|
|
await api("POST", `/api/companies/${companyId}/tools/policies`, {
|
|
name: `${scenario.path} block ${Date.now()}`, policyType: "block", priority: 1,
|
|
selectors: { connectionId: conn.id, toolNames: [scenario.lifecycle.deniedCall.name] },
|
|
});
|
|
const denied = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, {
|
|
agentId: scout.id, toolName: scenario.lifecycle.deniedCall.name, parameters: scenario.lifecycle.deniedCall.parameters,
|
|
});
|
|
assert(denied.decision === "off", `denied decision=${denied.decision}`);
|
|
assert(denied.error?.reasonCode, "denied has reasonCode");
|
|
await page.goto(`${BASE}/${prefix}/apps/${conn.id}/review`, { waitUntil: "networkidle" });
|
|
return `Blocked call ${scenario.lifecycle.deniedCall.name}: ${denied.error?.reasonCode}`;
|
|
});
|
|
|
|
// schema-change-quarantine
|
|
await doStep(scenario, "schema-change-quarantine", async () => {
|
|
if (conn.transport !== "mcp_remote") {
|
|
await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" });
|
|
return "Non-HTTP path records governance/quarantine evidence through fixture metadata.";
|
|
}
|
|
await api("PATCH", `/api/tool-connections/${conn.id}`, { config: { ...(conn.config ?? {}), quarantineNewEntries: true } });
|
|
await api("POST", `/api/companies/${companyId}/tools/policies`, {
|
|
name: `${scenario.path} allow schema flip ${Date.now()}`, policyType: "allow", priority: 5,
|
|
selectors: { connectionId: conn.id, toolNames: [scenario.lifecycle.schemaChangeQuarantine.name] },
|
|
});
|
|
const flipped = await api("POST", `/api/tool-connections/${conn.id}/test-calls`, {
|
|
agentId: scout.id, toolName: scenario.lifecycle.schemaChangeQuarantine.name, parameters: scenario.lifecycle.schemaChangeQuarantine.parameters,
|
|
});
|
|
assert(flipped.decision === "allowed", `schema-flip decision=${flipped.decision}`);
|
|
const refresh = await api("POST", `/api/tool-connections/${conn.id}/catalog/refresh`);
|
|
assert((refresh.quarantinedCount ?? 0) > 0, `quarantinedCount=${refresh.quarantinedCount}`);
|
|
await page.goto(`${BASE}/${prefix}/apps/attention`, { waitUntil: "networkidle" });
|
|
return `Catalog refresh quarantined ${refresh.quarantinedCount} changed entries.`;
|
|
});
|
|
|
|
// revoke
|
|
await doStep(scenario, "revoke", async () => {
|
|
if (scenario.transport === "gateway_session") {
|
|
const invoked = await api("POST", `/api/agents/${scout.id}/heartbeat/invoke`);
|
|
const session = await api("POST", "/api/tool-gateway/sessions", { companyId, agentId: scout.id, runId: invoked.id, ttlMs: 60_000 });
|
|
const listRes = await fetch(`${BASE}${new URL(session.toolsUrl, BASE).pathname}`, { headers: { "x-paperclip-tool-gateway-token": session.token } });
|
|
assert(listRes.ok, "gateway tools list ok pre-revoke");
|
|
await api("POST", `/api/tool-gateway/sessions/${session.sessionId}/revoke`, { companyId });
|
|
const after = await fetch(`${BASE}${new URL(session.toolsUrl, BASE).pathname}`, { headers: { "x-paperclip-tool-gateway-token": session.token } });
|
|
assert(after.status === 401, `revoked token cut off (got ${after.status})`);
|
|
await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" });
|
|
return scenario.lifecycle.revoke;
|
|
}
|
|
const disabled = await api("PATCH", `/api/tool-connections/${conn.id}`, { enabled: false });
|
|
assert(disabled.enabled === false, "connection disabled");
|
|
await page.goto(`${BASE}/${prefix}/apps/${conn.id}`, { waitUntil: "networkidle" });
|
|
await api("PATCH", `/api/tool-connections/${conn.id}`, { enabled: true });
|
|
return scenario.lifecycle.revoke;
|
|
});
|
|
|
|
// audit-evidence
|
|
await doStep(scenario, "audit-evidence", async () => {
|
|
await auditHit(conn.id, scenario.lifecycle.allowedRead.name);
|
|
await page.goto(`${BASE}/${prefix}/activity?action=tool_`, { waitUntil: "networkidle" });
|
|
return scenario.lifecycle.auditEvidence;
|
|
});
|
|
}
|
|
|
|
// Optional injected failure — the canonical self-test for the §D5 routine's
|
|
// file-on-failure wiring. Records a real fail step + screenshot so the wrapper
|
|
// sees exit 2 + a `<path>-<step>-failed.png` to attach.
|
|
if (process.env.SMOKE_FORCE_FAIL) {
|
|
const target = scenarios[0];
|
|
await doStep(target, "forced-failure", async () => {
|
|
throw new Error(`SMOKE_FORCE_FAIL: ${process.env.SMOKE_FORCE_FAIL}`);
|
|
});
|
|
}
|
|
|
|
// Finalize run + capture the run in the live Smoke Lab UI.
|
|
await api("PATCH", `/api/companies/${companyId}/smoke-lab/runs/${run.id}`, {
|
|
status: failed.length ? "failed" : "passed",
|
|
summary: { catalog: "tests/e2e/smoke-lab.catalog.ts", driver: "agent-browser (real chromium)", scenarioCount: scenarios.length, only: ONLY.length ? ONLY : undefined, failed },
|
|
});
|
|
await page.goto(`${BASE}/${prefix}/apps/advanced/smoke-lab`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1500);
|
|
const tabFile = `${SHOT_DIR}/zz-smoke-lab-tab.png`;
|
|
await page.screenshot({ path: tabFile, fullPage: true });
|
|
await page.goto(`${BASE}/${prefix}/dashboard`, { waitUntil: "networkidle" });
|
|
await page.waitForTimeout(1500);
|
|
const dashFile = `${SHOT_DIR}/zz-dashboard-card.png`;
|
|
await page.screenshot({ path: dashFile, fullPage: true });
|
|
|
|
await browser.close();
|
|
|
|
const finalRun = await api("GET", `/api/companies/${companyId}/smoke-lab/runs/${run.id}`);
|
|
const steps = finalRun.steps ?? [];
|
|
const byPath: Record<string, number> = {};
|
|
for (const s of steps) byPath[s.path] = (byPath[s.path] ?? 0) + 1;
|
|
console.log("\n=== RESULT ===");
|
|
console.log(`companyId=${companyId} prefix=${prefix} runId=${run.id} status=${finalRun.run?.status}`);
|
|
console.log(`steps=${steps.length} perPath=${JSON.stringify(byPath)} failed=${failed.length}`);
|
|
console.log(`withScreenshot=${steps.filter((s: Json) => s.screenshotArtifactRef?.url).length}/${steps.length}`);
|
|
console.log(`tabShot=${tabFile} dashShot=${dashFile}`);
|
|
await fs.writeFile(`${SHOT_DIR}/result.json`, JSON.stringify({ companyId, prefix, runId: run.id, status: finalRun.run?.status, steps: steps.length, byPath, failed }, null, 2));
|
|
if (failed.length) process.exitCode = 2;
|
|
}
|
|
|
|
main().catch((e) => { console.error("RUNNER ERROR", e); process.exit(1); });
|