import { expect, test, type APIRequestContext, type Page } from "@playwright/test"; import { createServer, type Server } from "node:http"; import { listenOnFetchAllowedPort } from "./fetch-allowed-port"; // prosumer MCP flow — QA harness for the prosumer Connect-an-app flow on top of the // tool-access foundation. Covers the M-series happy path (gallery + key paste // → choose access → install → success), the expired-key reconnect path, // the Needs-attention surface, and a regression check that /apps/advanced // still mounts. // // The spec boots the shared local_trusted Playwright webServer (see // playwright.config), spawns a small in-process mock HTTP MCP server that // responds to tools/list with read-only and write-side-effect tools, then // drives the wizard via the Apps UI for evidence + screenshots. const SCREENSHOT_DIR = "test-results"; type Seed = { companyId: string; prefix: string }; async function newCompany(request: APIRequestContext, label: string): Promise { const res = await request.post("/api/companies", { data: { name: `prosumer MCP flow ${label} ${Date.now()}` } }); expect(res.ok(), `create company failed ${res.status()}: ${await res.text()}`).toBe(true); const company = await res.json(); return { companyId: company.id, prefix: company.issuePrefix ?? company.prefix ?? company.urlKey ?? "E2E", }; } // ---- Mock MCP HTTP fixture -------------------------------------------------- // Minimal MCP JSON-RPC server. /catalog refresh hits this with method // `tools/list`; the gateway calls it with `tools/call`. We expose one // read-only and one write tool so setup can apply risk-based ask-first defaults. type MockMcpServer = { url: string; close: () => Promise; captures: Array<{ method: string; params: unknown }> }; async function startMockMcp(options: { expectedHeader?: string } = {}): Promise { const captures: Array<{ method: string; params: unknown }> = []; const server: Server = createServer(async (req, res) => { if (req.method !== "POST") { res.writeHead(405).end(); return; } if (options.expectedHeader && req.headers.authorization !== options.expectedHeader) { res.writeHead(401, { "Content-Type": "application/json", "WWW-Authenticate": "Bearer" }); res.end(JSON.stringify({ jsonrpc: "2.0", id: null, error: { code: -32000, message: "Unauthorized" } })); return; } const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(chunk as Buffer); let payload: { id?: string | number; method?: string; params?: unknown } = {}; try { payload = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}"); } catch { // fall through to method routing — will land in default } captures.push({ method: String(payload.method ?? ""), params: payload.params }); if (payload.method === "tools/list") { res.writeHead(200, { "Content-Type": "application/json" }); res.end( JSON.stringify({ jsonrpc: "2.0", id: payload.id ?? null, result: { tools: [ { name: "list_widgets", title: "List widgets", description: "Read-only listing of widgets.", inputSchema: { type: "object", properties: {}, additionalProperties: false }, }, { // Namespaced name (namespaced tool names): real MCP servers prefix tool names // ("github:create_issue"). The classifier must still see the "create" // verb and land this under "Can make changes" with the toggle OFF — // the old leading-anchor regex fell through to read-only. name: "qa10864:create_widget", title: "Create widget", description: "Creates a widget — has side effects.", inputSchema: { type: "object", properties: { name: { type: "string" } }, required: ["name"], additionalProperties: false, }, }, ], }, }) ); return; } if (payload.method === "tools/call") { res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify({ jsonrpc: "2.0", id: payload.id ?? null, result: { content: [{ type: "text", text: "ok" }] } })); return; } res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify({ jsonrpc: "2.0", id: payload.id ?? null, result: {} })); }); const port = await listenOnFetchAllowedPort(server); return { url: `http://127.0.0.1:${port}/`, captures, close: () => new Promise((resolve) => server.close(() => resolve())), }; } // ---- Helpers ---------------------------------------------------------------- async function gotoApps(page: Page, prefix: string) { await page.goto(`/${prefix}/apps/connections`); } async function gotoConnect(page: Page, prefix: string) { await page.goto(`/${prefix}/apps`); await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 30_000 }); const customConnector = page .getByRole("list", { name: "Connector list" }) .getByRole("listitem") .filter({ hasText: "Connect your own tool" }); await customConnector.getByRole("button", { name: "Connect", exact: true }).click(); await customConnector.getByRole("button", { name: "Connect your own MCP server" }).click(); } async function gotoAdvanced(page: Page, prefix: string) { await page.goto(`/${prefix}/apps/advanced`); } async function gotoNeedsAttention(page: Page, prefix: string) { await page.goto(`/${prefix}/apps/connections`); } // ---- Tests ------------------------------------------------------------------ test.describe.serial("prosumer MCP flow prosumer MCP flow", () => { test.setTimeout(180_000); // vite-dev cold bundling is slow on first hit let mock: MockMcpServer; test.beforeAll(async () => { mock = await startMockMcp(); }); test.afterAll(async () => { await mock?.close(); }); test("Connect wizard happy path: link mode → success", async ({ page, request }) => { const seed = await newCompany(request, "connect"); await gotoConnect(page, seed.prefix); // Browse launches the BYO link-mode connect wizard. await expect(page.getByText("Connect your own MCP server", { exact: true })).toBeVisible({ timeout: 30_000 }); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-01-gallery.png`, fullPage: true }); // Use the "Connect with a link" path against the mock MCP server. const linkInput = page.getByPlaceholder("https://example.com/actions"); await linkInput.fill(mock.url); await page.getByRole("button", { name: "Continue" }).click(); // Access is chosen before credentials so the user knows who and which // agents will receive the connection before Paperclip contacts it. await expect(page.getByText("Which humans can use this credential?")).toBeVisible(); await page.getByRole("button", { name: "Save and continue" }).click(); // LinkKey step keeps the BYO connection heading. Mock doesn't // require a key — leave the default "No" answer. await expect(page.getByRole("heading", { name: "Connect your own MCP server" })).toBeVisible({ timeout: 15_000 }); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-02-key-step.png`, fullPage: true }); // Submit (button label is "Check link"). await page.getByRole("button", { name: /Check link/i }).click(); // The Access choice was captured before credentials. A successful generic // probe now commits discovered actions and risk defaults transactionally, // so the key check lands directly on success. await expect(page.getByRole("heading", { name: /is ready\.$/i })).toBeVisible({ timeout: 30_000 }); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-05-success.png`, fullPage: true }); // Verify the mock saw a tools/list call from the catalog refresh. expect(mock.captures.some((c) => c.method === "tools/list")).toBe(true); // The new connection should show up on /apps/connections. await gotoApps(page, seed.prefix); await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 15_000 }); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-06-apps-list.png`, fullPage: true }); }); test("Expired key → health sweep → Needs attention → reconnect → green", async ({ page, request }) => { const seed = await newCompany(request, "attention"); // Spawn a dedicated mock we can break partway through. const ephemeral = await startMockMcp(); try { const connect = await request.post(`/api/companies/${seed.companyId}/tools/apps/connect`, { data: { link: ephemeral.url, credentialValues: { "credentials.authorization": "qa-token" } }, }); expect(connect.ok(), `connect failed ${connect.status()}: ${await connect.text()}`).toBe(true); const connectResult = await connect.json(); const connectionId = connectResult.connectionId as string; const enabledCatalogEntryIds = (connectResult.actions?.readOnly ?? []).map( (action: { catalogEntryId: string }) => action.catalogEntryId, ); const finish = await request.post(`/api/companies/${seed.companyId}/tools/apps/${connectionId}/finish`, { data: { enabledCatalogEntryIds, askFirstCatalogEntryIds: [], access: "all_agents" }, }); expect(finish.ok(), `finish failed ${finish.status()}: ${await finish.text()}`).toBe(true); // Break the mock so the next health-check fails (simulates expired key / // dead remote — the same observable shape the server uses to mark a // connection unhealthy). await ephemeral.close(); const health = await request.post(`/api/tool-connections/${connectionId}/health-check`); expect(health.status(), `health-check should fail with the mock down`).toBe(502); const before = await request.get(`/api/tool-connections/${connectionId}`); const beforeBody = await before.json(); expect(beforeBody.healthStatus).not.toBe("ok"); // Needs-attention page should surface this connection. await gotoNeedsAttention(page, seed.prefix); await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 30_000 }); await expect(page.getByText("Needs attention", { exact: true }).first()).toBeVisible({ timeout: 30_000 }); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-07-needs-attention.png`, fullPage: true }); // App detail should expose the reconnect call-to-action. await page.goto(`/${seed.prefix}/apps/${connectionId}`); await expect(page.getByRole("button", { name: /Reconnect|Replace key/i }).first()).toBeVisible({ timeout: 20_000 }); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-08-app-detail-reconnect.png`, fullPage: true }); // Bring the mock back so reconnect succeeds. const recovered = await startMockMcp(); try { // The reconnect endpoint replaces credentials but does not change the URL, // so we update the connection URL through PATCH first to point at the // recovered mock — this mirrors what users do when the remote moves. const repatch = await request.patch(`/api/tool-connections/${connectionId}`, { data: { config: { url: recovered.url } }, }); expect(repatch.ok(), `patch url failed ${repatch.status()}: ${await repatch.text()}`).toBe(true); const reconnect = await request.post(`/api/tool-connections/${connectionId}/reconnect`, { data: { credentialValues: { "credentials.authorization": "fresh-key" } }, }); expect(reconnect.ok(), `reconnect failed ${reconnect.status()}: ${await reconnect.text()}`).toBe(true); const after = await request.get(`/api/tool-connections/${connectionId}`); const afterBody = await after.json(); expect(afterBody.healthStatus).toBe("ok"); } finally { await recovered.close(); } } finally { await ephemeral.close().catch(() => {}); } }); test("/apps/advanced mounts the paste-config path", async ({ page, request }) => { const seed = await newCompany(request, "advanced"); await gotoAdvanced(page, seed.prefix); await expect(page.getByRole("heading", { name: "Advanced setup" })).toBeVisible({ timeout: 20_000 }); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-09-advanced-default.png`, fullPage: true }); await expect(page.getByRole("link", { name: "Paste a config" })).toBeVisible(); await expect(page.getByRole("link", { name: /Run your own|Self host|Stdio|Local/i })).toHaveCount(0); await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-10-advanced-paste-tab.png`, fullPage: true }); }); });