import { expect, test, type APIRequestContext, type Page } from "@playwright/test"; import { ciSmokeLabScenarios, type SmokeLabLifecycleTool, type SmokeLabScenario, type SmokeRunStepPath } from "./smoke-lab.catalog"; type SmokeRunStepStatus = "pass" | "fail" | "skipped"; const SCREENSHOT_DIR = "test-results/smoke-lab"; type Json = Record; type Seed = { companyId: string; prefix: string }; type Scout = { id: string; name: string }; type SmokeRun = { id: string; status: string }; type SmokeRunStepResult = { step: { id: string; status: SmokeRunStepStatus }; summary: Record; }; type ToolConnection = { id: string; name: string; transport: "mcp_remote" | "local_stdio"; applicationId: string; enabled: boolean; status?: string; config?: Record | null; }; type ToolCatalogEntry = { id: string; toolName: string; name: string; riskLevel?: string | null; status?: string | null; }; type FixtureInstall = { connections: ToolConnection[]; catalog: ToolCatalogEntry[]; }; type TestCallResult = { decision: "allowed" | "ask_first" | "off"; invocationId: string; actionRequestId?: string; error?: { reasonCode?: string | null; message: string }; }; type GatewaySession = { sessionId: string; token: string; toolsUrl: string; callUrl: string; }; async function json(response: Awaited>): Promise { const body = await response.text(); expect(response.ok(), `${response.url()} failed ${response.status()}: ${body}`).toBe(true); return JSON.parse(body) as T; } async function expectError(response: Awaited>, status: number) { const body = await response.text(); expect(response.status(), `${response.url()} expected ${status}, got ${response.status()}: ${body}`).toBe(status); return body; } async function newCompany(request: APIRequestContext, label: string): Promise { const body = await json<{ id: string; issuePrefix?: string; prefix?: string; urlKey?: string }>( await request.post("/api/companies", { data: { name: `Smoke Lab ${label} ${Date.now()}` } }), ); return { companyId: body.id, prefix: body.issuePrefix ?? body.prefix ?? body.urlKey ?? "E2E" }; } async function createScout(request: APIRequestContext, companyId: string): Promise { const body = await json<{ id: string; name: string }>( await request.post(`/api/companies/${companyId}/agents`, { data: { name: `Smoke Scout ${Date.now()}`, role: "qa", title: "Smoke Lab scout", capabilities: "Runs deterministic Smoke Lab fixture calls.", adapterType: "process", adapterConfig: { command: "node", args: ["-e", "setTimeout(() => {}, 1000)"] }, }, }), ); return { id: body.id, name: body.name }; } async function enableSmokeLab(request: APIRequestContext) { await json(await request.patch("/api/instance/settings/experimental", { data: { enableSmokeLab: true } })); } async function createSmokeRun(request: APIRequestContext, companyId: string, scenarioCount: number) { const result = await json<{ run: SmokeRun }>( await request.post(`/api/companies/${companyId}/smoke-lab/runs`, { data: { trigger: "ci", summary: { catalog: "tests/e2e/smoke-lab.catalog.ts", scenarioCount, }, }, }), ); return result.run; } async function updateSmokeRun( request: APIRequestContext, companyId: string, runId: string, status: "passed" | "failed", summary: Json, ) { await json(await request.patch(`/api/companies/${companyId}/smoke-lab/runs/${runId}`, { data: { status, summary }, })); } async function recordStep( request: APIRequestContext, companyId: string, runId: string, input: { path: SmokeRunStepPath; scenarioStep: string; status: SmokeRunStepStatus; detail?: string | null; screenshotPath?: string | null; durationMs?: number | null; }, ): Promise { return await json( await request.post(`/api/companies/${companyId}/smoke-lab/runs/${runId}/steps`, { data: { path: input.path, scenarioStep: input.scenarioStep, status: input.status, detail: input.detail ?? null, screenshotArtifactRef: input.screenshotPath ? { kind: "playwright_screenshot", path: input.screenshotPath } : null, durationMs: input.durationMs ?? null, }, }), ); } async function screenshot(page: Page, scenario: SmokeLabScenario, step: string) { const path = `${SCREENSHOT_DIR}/${scenario.path.toLowerCase()}-${step}.png`; await page.screenshot({ path, fullPage: true }); return path; } async function navigateForEvidence(page: Page, seed: Seed, connectionId: string, scenario: SmokeLabScenario) { if (scenario.uiEntryPath === "advanced") { await page.goto(`/${seed.prefix}/apps/advanced`); await expect(page.getByRole("heading", { name: "Advanced setup" })).toBeVisible({ timeout: 20_000 }); return; } if (scenario.uiEntryPath === "review") { await page.goto(`/${seed.prefix}/apps/${connectionId}/review`); await expect(page.getByText(/Nothing is waiting for your OK|new actions? (need|to) review/i).first()).toBeVisible({ timeout: 20_000 }); return; } if (scenario.uiEntryPath === "activity") { await page.goto(`/${seed.prefix}/activity?action=tool_`); await expect(page.locator("#main-content").getByRole("heading", { name: "Audit" })).toBeVisible({ timeout: 20_000 }); await expect(page.getByRole("combobox").filter({ hasText: "Apps & tools" })).toBeVisible(); return; } if (scenario.uiEntryPath === "attention") { await page.goto(`/${seed.prefix}/apps/connections`); await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 20_000 }); return; } await page.goto(`/${seed.prefix}/apps/${connectionId}`); await expect(page.getByRole("heading", { name: /Smoke Lab/i }).first()).toBeVisible({ timeout: 30_000 }); } async function runRecordedStep( page: Page, request: APIRequestContext, seed: Seed, runId: string, scenario: SmokeLabScenario, step: string, action: () => Promise, captureSuccessScreenshot = false, ) { const start = Date.now(); try { const screenshotHint = await action(); const screenshotPath = captureSuccessScreenshot ? await screenshot(page, scenario, step) : null; await recordStep(request, seed.companyId, runId, { path: scenario.path, scenarioStep: step, status: "pass", detail: screenshotHint ?? null, screenshotPath, durationMs: Date.now() - start, }); } catch (error) { const screenshotPath = await screenshot(page, scenario, `${step}-failed`).catch(() => null); await recordStep(request, seed.companyId, runId, { path: scenario.path, scenarioStep: step, status: "fail", detail: error instanceof Error ? error.message : String(error), screenshotPath, durationMs: Date.now() - start, }).catch(() => undefined); throw error; } } async function startSmokeLabServices(request: APIRequestContext, companyId: string) { await json(await request.post(`/api/companies/${companyId}/smoke-lab/services/start`)); } async function installFixtures(request: APIRequestContext, companyId: string): Promise { return await json(await request.post(`/api/companies/${companyId}/smoke-lab/install-fixtures`)); } function connectionForScenario(fixtures: FixtureInstall, scenario: SmokeLabScenario): ToolConnection { const preferStdio = scenario.transport === "local_stdio" || scenario.transport === "plugin"; const transport = preferStdio ? "local_stdio" : "mcp_remote"; const connection = fixtures.connections.find((candidate) => candidate.transport === transport); if (!connection) throw new Error(`Missing ${transport} fixture connection for ${scenario.path}`); return connection; } async function catalog(request: APIRequestContext, connectionId: string) { return await json<{ catalog: ToolCatalogEntry[] }>(await request.get(`/api/tool-connections/${connectionId}/catalog`)); } async function testCall( request: APIRequestContext, connectionId: string, scout: Scout, tool: SmokeLabLifecycleTool, ) { return await json( await request.post(`/api/tool-connections/${connectionId}/test-calls`, { data: { agentId: scout.id, toolName: tool.name, parameters: tool.parameters }, }), ); } async function policy( request: APIRequestContext, companyId: string, body: { name: string; policyType: "allow" | "block" | "require_approval"; priority: number; selectors: Record; }, ) { return await json<{ id: string }>(await request.post(`/api/companies/${companyId}/tools/policies`, { data: body })); } async function approveActionRequest(request: APIRequestContext, companyId: string, actionRequestId: string) { await json(await request.post(`/api/tool-gateway/action-requests/${actionRequestId}/approve`, { data: { companyId }, })); } async function pollTestCall( request: APIRequestContext, connectionId: string, actionRequestId: string, expectedPhase: string, ) { for (let i = 0; i < 40; i += 1) { const status = await json<{ phase: string }>( await request.get(`/api/tool-connections/${connectionId}/test-calls/${actionRequestId}`), ); if (status.phase === expectedPhase) return status; await new Promise((resolve) => setTimeout(resolve, 250)); } throw new Error(`Timed out waiting for test-call ${actionRequestId} phase ${expectedPhase}`); } async function expectAuditEvent( request: APIRequestContext, companyId: string, options: { connectionId: string; agentId: string; search: string }, ) { const audit = await json<{ events: Array }>( await request.get( `/api/tool-gateway/audit?companyId=${companyId}&app=${options.connectionId}&agent=${options.agentId}&search=${encodeURIComponent(options.search)}&limit=50`, ), ); expect(audit.events.length, `expected audit/activity row matching ${options.search}`).toBeGreaterThan(0); } async function createGatewaySession(request: APIRequestContext, companyId: string, scout: Scout): Promise { const invoked = await json<{ id: string }>(await request.post(`/api/agents/${scout.id}/heartbeat/invoke`)); return await json( await request.post("/api/tool-gateway/sessions", { data: { companyId, agentId: scout.id, runId: invoked.id, ttlMs: 60_000 }, }), ); } async function gatewayFetch(request: APIRequestContext, path: string, token: string, data?: Json) { const headers = { "x-paperclip-tool-gateway-token": token }; if (data) return await request.post(path, { headers, data }); return await request.get(path, { headers }); } export const defineSmokeLabSuite = (label: string, scenarios: SmokeLabScenario[]) => test.describe.serial(`Smoke Lab scenario catalog mirror (${label})`, () => { // The lifecycle test walks all CI-safe scenarios and records eight steps per // scenario. Each scenario keeps one representative success screenshot, while // every failed step still captures its own evidence. On a loaded CI runner // the whole walk can take several minutes, so this ceiling keeps enough // headroom for runner variance without slowing a healthy run. test.setTimeout(360_000); test(`records the ${label} CI-safe Smoke Lab lifecycle into the results API @smoke-lab`, async ({ page, request }) => { const seed = await newCompany(request, "catalog"); const scout = await createScout(request, seed.companyId); await enableSmokeLab(request); const smokeRun = await createSmokeRun(request, seed.companyId, scenarios.length); const failed: string[] = []; const isPartialRun = scenarios.length < ciSmokeLabScenarios.length; try { await startSmokeLabServices(request, seed.companyId); for (const scenario of scenarios) { const fixtures = await installFixtures(request, seed.companyId); const connection = connectionForScenario(fixtures, scenario); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "connect", async () => { await navigateForEvidence(page, seed, connection.id, scenario); return scenario.lifecycle.connect; }, true); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "discover-catalog", async () => { const discovered = await catalog(request, connection.id); expect(discovered.catalog.map((entry) => entry.toolName)).toContain(scenario.lifecycle.allowedRead.name); return `${scenario.lifecycle.discoverCatalog}: ${discovered.catalog.length} entries`; }); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "allowed-read", async () => { const read = await testCall(request, connection.id, scout, scenario.lifecycle.allowedRead); expect(read.decision).toBe("allowed"); expect(read.error).toBeUndefined(); await expectAuditEvent(request, seed.companyId, { connectionId: connection.id, agentId: scout.id, search: scenario.lifecycle.allowedRead.name, }); await page.goto(`/${seed.prefix}/activity?action=tool_`); return `Allowed read ${scenario.lifecycle.allowedRead.name}`; }); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "ask-first-write-approved", async () => { await policy(request, seed.companyId, { name: `${scenario.path} require approval ${Date.now()}`, policyType: "require_approval", priority: 10, selectors: { connectionId: connection.id, toolNames: [scenario.lifecycle.askFirstWrite.name] }, }); const pending = await testCall(request, connection.id, scout, scenario.lifecycle.askFirstWrite); expect(pending.decision).toBe("ask_first"); expect(pending.actionRequestId).toBeTruthy(); await page.goto(`/${seed.prefix}/apps/${connection.id}/review`); await approveActionRequest(request, seed.companyId, pending.actionRequestId!); await pollTestCall(request, connection.id, pending.actionRequestId!, "done"); return `Approved ask-first call ${scenario.lifecycle.askFirstWrite.name}`; }); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "denied-blocked-call", async () => { await policy(request, seed.companyId, { name: `${scenario.path} block ${Date.now()}`, policyType: "block", priority: 1, selectors: { connectionId: connection.id, toolNames: [scenario.lifecycle.deniedCall.name] }, }); const denied = await testCall(request, connection.id, scout, scenario.lifecycle.deniedCall); expect(denied.decision).toBe("off"); expect(denied.error?.reasonCode).toBeTruthy(); await page.goto(`/${seed.prefix}/apps/${connection.id}/review`); return `Blocked call ${scenario.lifecycle.deniedCall.name}: ${denied.error?.reasonCode}`; }); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "schema-change-quarantine", async () => { if (connection.transport !== "mcp_remote") { await page.goto(`/${seed.prefix}/activity?action=tool_`); return "Non-HTTP path records governance/quarantine evidence through fixture metadata."; } await json(await request.patch(`/api/tool-connections/${connection.id}`, { data: { config: { ...(connection.config ?? {}), quarantineNewEntries: true } }, })); await policy(request, seed.companyId, { name: `${scenario.path} allow schema flip ${Date.now()}`, policyType: "allow", priority: 5, selectors: { connectionId: connection.id, toolNames: [scenario.lifecycle.schemaChangeQuarantine.name] }, }); const flipped = await testCall(request, connection.id, scout, scenario.lifecycle.schemaChangeQuarantine); expect(flipped.decision).toBe("allowed"); expect(flipped.error).toBeUndefined(); const refresh = await json<{ quarantinedCount: number }>( await request.post(`/api/tool-connections/${connection.id}/catalog/refresh`), ); expect(refresh.quarantinedCount).toBeGreaterThan(0); await page.goto(`/${seed.prefix}/apps/connections`); return `Catalog refresh quarantined ${refresh.quarantinedCount} changed entries.`; }); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "revoke", async () => { if (scenario.transport === "gateway_session") { const session = await createGatewaySession(request, seed.companyId, scout); const listed = await gatewayFetch(request, session.toolsUrl, session.token); expect(listed.ok()).toBe(true); await json(await request.post(`/api/tool-gateway/sessions/${session.sessionId}/revoke`, { data: { companyId: seed.companyId }, })); await expectError(await gatewayFetch(request, session.toolsUrl, session.token), 401); await page.goto(`/${seed.prefix}/activity?action=tool_`); return scenario.lifecycle.revoke; } const disabled = await json(await request.patch(`/api/tool-connections/${connection.id}`, { data: { enabled: false }, })); expect(disabled.enabled).toBe(false); await page.goto(`/${seed.prefix}/apps/${connection.id}`); await json(await request.patch(`/api/tool-connections/${connection.id}`, { data: { enabled: true }, })); return scenario.lifecycle.revoke; }); await runRecordedStep(page, request, seed, smokeRun.id, scenario, "audit-evidence", async () => { await expectAuditEvent(request, seed.companyId, { connectionId: connection.id, agentId: scout.id, search: scenario.lifecycle.allowedRead.name, }); await page.goto(`/${seed.prefix}/activity?action=tool_`); return scenario.lifecycle.auditEvidence; }); } } catch (error) { failed.push(error instanceof Error ? error.message : String(error)); throw error; } finally { await updateSmokeRun(request, seed.companyId, smokeRun.id, failed.length > 0 ? "failed" : "passed", { catalog: "tests/e2e/smoke-lab.catalog.ts", scenarioCount: scenarios.length, catalogScenarioCount: ciSmokeLabScenarios.length, partial: isPartialRun, failed, }).catch(() => undefined); } const completed = await json<{ run: SmokeRun; steps: Array<{ path: string; scenarioStep: string; status: string; screenshotArtifactRef: Json | null }>; }>( await request.get(`/api/companies/${seed.companyId}/smoke-lab/runs/${smokeRun.id}`), ); expect(completed.run.status).toBe("passed"); for (const scenario of scenarios) { const steps = completed.steps.filter((step) => step.path === scenario.path); expect(steps.length, `${scenario.path} should record lifecycle steps`).toBeGreaterThanOrEqual(8); expect(steps.every((step) => step.status === "pass")).toBe(true); const screenshotSteps = steps.filter((step) => step.screenshotArtifactRef?.kind === "playwright_screenshot"); expect(screenshotSteps.map((step) => step.scenarioStep)).toEqual(["connect"]); } }); });