464 lines
19 KiB
TypeScript
464 lines
19 KiB
TypeScript
import { expect, test, type APIRequestContext, type Page } from "@playwright/test";
|
|
import { createServer, type Server } from "node:http";
|
|
import { listenOnFetchAllowedPort } from "./fetch-allowed-port";
|
|
import { storyById } from "./mcp-user-stories.catalog";
|
|
|
|
const SCREENSHOT_DIR = "test-results/mcp-user-stories";
|
|
|
|
type Seed = { companyId: string; prefix: string };
|
|
type Scout = { id: string; name: string };
|
|
type Json = Record<string, unknown>;
|
|
type MockMcpServer = {
|
|
url: string;
|
|
captures: Array<{ method: string; toolName: string | null; params: unknown }>;
|
|
close: () => Promise<void>;
|
|
};
|
|
type HeartbeatRun = { id: string; status: string; error?: string | null };
|
|
|
|
async function json<T = Json>(response: Awaited<ReturnType<APIRequestContext["get"]>>): Promise<T> {
|
|
expect(response.ok(), `${response.url()} failed ${response.status()}: ${await response.text()}`).toBe(true);
|
|
return await response.json() as T;
|
|
}
|
|
|
|
async function newCompany(request: APIRequestContext, label: string): Promise<Seed> {
|
|
const body = await json<{ id: string; issuePrefix?: string; prefix?: string; urlKey?: string }>(
|
|
await request.post("/api/companies", { data: { name: `MCP US ${label} ${Date.now()}` } }),
|
|
);
|
|
return { companyId: body.id, prefix: body.issuePrefix ?? body.prefix ?? body.urlKey ?? "E2E" };
|
|
}
|
|
|
|
async function createScout(request: APIRequestContext, companyId: string): Promise<Scout> {
|
|
const body = await json<{ id: string; name: string }>(
|
|
await request.post(`/api/companies/${companyId}/agents`, {
|
|
data: {
|
|
name: `Scout ${Date.now()}`,
|
|
role: "qa",
|
|
title: "MCP story scout",
|
|
capabilities: "Runs governed MCP user-story fixture calls.",
|
|
adapterType: "process",
|
|
adapterConfig: { command: "node", args: ["-e", "process.exit(0)"] },
|
|
},
|
|
}),
|
|
);
|
|
return { id: body.id, name: body.name };
|
|
}
|
|
|
|
function buildGatewayCallScript(connectionId: string, toolName: string, parameters: Json = {}) {
|
|
return `
|
|
const required = ["PAPERCLIP_API_URL", "PAPERCLIP_API_KEY", "PAPERCLIP_RUN_ID"];
|
|
for (const key of required) {
|
|
if (!process.env[key]) throw new Error(\`Missing \${key}\`);
|
|
}
|
|
const headers = {
|
|
"authorization": \`Bearer \${process.env.PAPERCLIP_API_KEY}\`,
|
|
"content-type": "application/json"
|
|
};
|
|
const sessionRes = await fetch(\`\${process.env.PAPERCLIP_API_URL}/api/tool-gateway/sessions\`, {
|
|
method: "POST",
|
|
headers,
|
|
body: JSON.stringify({ runId: process.env.PAPERCLIP_RUN_ID, ttlMs: 60000 })
|
|
});
|
|
if (!sessionRes.ok) throw new Error(\`session \${sessionRes.status}: \${await sessionRes.text()}\`);
|
|
const session = await sessionRes.json();
|
|
const toolsRes = await fetch(\`\${process.env.PAPERCLIP_API_URL}/api/tool-gateway/tools\`, {
|
|
headers: { "x-paperclip-tool-gateway-token": session.token }
|
|
});
|
|
if (!toolsRes.ok) throw new Error(\`tools \${toolsRes.status}: \${await toolsRes.text()}\`);
|
|
const tools = await toolsRes.json();
|
|
const tool = tools.find((entry) =>
|
|
entry.connectionId === ${JSON.stringify(connectionId)}
|
|
&& (entry.upstreamToolName === ${JSON.stringify(toolName)} || entry.name === ${JSON.stringify(toolName)})
|
|
);
|
|
if (!tool) throw new Error(\`Missing gateway tool for ${toolName}\`);
|
|
const callRes = await fetch(\`\${process.env.PAPERCLIP_API_URL}/api/tool-gateway/tools/call\`, {
|
|
method: "POST",
|
|
headers: {
|
|
"content-type": "application/json",
|
|
"x-paperclip-tool-gateway-token": session.token
|
|
},
|
|
body: JSON.stringify({
|
|
tool: tool.name,
|
|
parameters: ${JSON.stringify(parameters)}
|
|
})
|
|
});
|
|
const text = await callRes.text();
|
|
if (!callRes.ok) throw new Error(\`tool call \${callRes.status}: \${text}\`);
|
|
console.log(text);
|
|
`;
|
|
}
|
|
|
|
async function setScoutScript(
|
|
request: APIRequestContext,
|
|
scout: Scout,
|
|
script: string,
|
|
) {
|
|
await json(await request.patch(`/api/agents/${scout.id}`, {
|
|
data: {
|
|
adapterType: "process",
|
|
adapterConfig: {
|
|
command: process.execPath,
|
|
args: ["--input-type=module", "-e", script],
|
|
},
|
|
replaceAdapterConfig: true,
|
|
},
|
|
}));
|
|
}
|
|
|
|
async function invokeHeartbeat(request: APIRequestContext, agentId: string) {
|
|
return await json<HeartbeatRun>(await request.post(`/api/agents/${agentId}/heartbeat/invoke`));
|
|
}
|
|
|
|
async function waitForRun(request: APIRequestContext, runId: string) {
|
|
for (let i = 0; i < 60; i += 1) {
|
|
const run = await json<HeartbeatRun>(await request.get(`/api/heartbeat-runs/${runId}`));
|
|
if (!["queued", "running"].includes(run.status)) return run;
|
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
}
|
|
throw new Error(`Timed out waiting for heartbeat run ${runId}`);
|
|
}
|
|
|
|
async function startMockMcp(): Promise<MockMcpServer> {
|
|
const captures: MockMcpServer["captures"] = [];
|
|
const server: Server = createServer(async (req, res) => {
|
|
if (req.method !== "POST") {
|
|
res.writeHead(405).end();
|
|
return;
|
|
}
|
|
const chunks: Buffer[] = [];
|
|
for await (const chunk of req) chunks.push(chunk as Buffer);
|
|
const payload = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}") as {
|
|
id?: string | number;
|
|
method?: string;
|
|
params?: { name?: string; arguments?: unknown };
|
|
};
|
|
const toolName = payload.params?.name ?? null;
|
|
captures.push({ method: String(payload.method ?? "<unknown>"), toolName, params: payload.params ?? null });
|
|
|
|
if (payload.method === "tools/list") {
|
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({
|
|
jsonrpc: "2.0",
|
|
id: payload.id ?? null,
|
|
result: {
|
|
tools: [
|
|
{
|
|
name: "sheets:list_rows",
|
|
title: "List sheet rows",
|
|
description: "Read rows from the deterministic Sheets fixture.",
|
|
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
|
},
|
|
{
|
|
name: "sheets:update_cell",
|
|
title: "Update sheet cell",
|
|
description: "Updates one deterministic sheet cell.",
|
|
inputSchema: {
|
|
type: "object",
|
|
properties: { cell: { type: "string" }, value: { type: "string" } },
|
|
required: ["cell", "value"],
|
|
additionalProperties: false,
|
|
},
|
|
},
|
|
{
|
|
name: "sheets:delete_row",
|
|
title: "Delete sheet row",
|
|
description: "Deletes one deterministic sheet row.",
|
|
inputSchema: {
|
|
type: "object",
|
|
properties: { row: { type: "number" } },
|
|
required: ["row"],
|
|
additionalProperties: false,
|
|
},
|
|
},
|
|
],
|
|
},
|
|
}));
|
|
return;
|
|
}
|
|
if (payload.method === "tools/call") {
|
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({
|
|
jsonrpc: "2.0",
|
|
id: payload.id ?? null,
|
|
result: { content: [{ type: "text", text: `${toolName ?? "tool"} ok` }] },
|
|
}));
|
|
return;
|
|
}
|
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ jsonrpc: "2.0", id: payload.id ?? null, result: {} }));
|
|
});
|
|
const port = await listenOnFetchAllowedPort(server);
|
|
return {
|
|
url: `http://127.0.0.1:${port}/`,
|
|
captures,
|
|
close: () => new Promise<void>((resolve) => server.close(() => resolve())),
|
|
};
|
|
}
|
|
|
|
async function screenshot(page: Page, storyId: string, step: string) {
|
|
await page.screenshot({ path: `${SCREENSHOT_DIR}/${storyId.toLowerCase()}-${step}.png`, fullPage: true });
|
|
}
|
|
|
|
async function seedConnectedFixture(request: APIRequestContext, label: string) {
|
|
const seed = await newCompany(request, label);
|
|
const scout = await createScout(request, seed.companyId);
|
|
const mock = await startMockMcp();
|
|
const connect = await json<{
|
|
connectionId: string;
|
|
catalog: Array<{ id: string; toolName: string; riskLevel?: string | null }>;
|
|
}>(await request.post(`/api/companies/${seed.companyId}/tools/apps/connect`, {
|
|
data: { link: mock.url, name: `Sheets Fixture ${label}` },
|
|
}));
|
|
const enabled = connect.catalog.map((entry) => entry.id);
|
|
const askFirst = connect.catalog
|
|
.filter((entry) => /update|delete|create|send|write/i.test(entry.toolName) || entry.riskLevel === "write" || entry.riskLevel === "destructive")
|
|
.map((entry) => entry.id);
|
|
await json(await request.post(`/api/companies/${seed.companyId}/tools/apps/${connect.connectionId}/finish`, {
|
|
data: {
|
|
enabledCatalogEntryIds: enabled,
|
|
askFirstCatalogEntryIds: askFirst,
|
|
access: { agentIds: [scout.id] },
|
|
},
|
|
}));
|
|
return { seed, scout, mock, connectionId: connect.connectionId };
|
|
}
|
|
|
|
async function testCall(
|
|
request: APIRequestContext,
|
|
connectionId: string,
|
|
scout: Scout,
|
|
toolName: string,
|
|
parameters: Json = {},
|
|
) {
|
|
return await json<{
|
|
decision: "allowed" | "ask_first" | "off";
|
|
invocationId: string;
|
|
actionRequestId?: string;
|
|
result?: unknown;
|
|
error?: { reasonCode?: string | null; message: string };
|
|
}>(await request.post(`/api/tool-connections/${connectionId}/test-calls`, {
|
|
data: { agentId: scout.id, toolName, parameters },
|
|
}));
|
|
}
|
|
|
|
async function approveActionRequest(request: APIRequestContext, companyId: string, actionRequestId: string) {
|
|
return await json(await request.post(`/api/tool-gateway/action-requests/${actionRequestId}/approve`, {
|
|
data: { companyId },
|
|
}));
|
|
}
|
|
|
|
async function declineActionRequest(request: APIRequestContext, companyId: string, actionRequestId: string) {
|
|
return await json(await request.post(`/api/tool-gateway/action-requests/${actionRequestId}/decline`, {
|
|
data: { companyId },
|
|
}));
|
|
}
|
|
|
|
async function pollTestCall(
|
|
request: APIRequestContext,
|
|
connectionId: string,
|
|
actionRequestId: string,
|
|
expectedPhase: string,
|
|
) {
|
|
for (let i = 0; i < 20; i += 1) {
|
|
const status = await json<{ phase: string }>(
|
|
await request.get(`/api/tool-connections/${connectionId}/test-calls/${actionRequestId}`),
|
|
);
|
|
if (status.phase === expectedPhase) return status;
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
}
|
|
throw new Error(`Timed out waiting for test-call ${actionRequestId} phase ${expectedPhase}`);
|
|
}
|
|
|
|
async function expectAuditEvent(
|
|
request: APIRequestContext,
|
|
companyId: string,
|
|
options: { connectionId: string; agentId: string; search: string },
|
|
) {
|
|
const audit = await json<{ events: Array<Json> }>(
|
|
await request.get(
|
|
`/api/tool-gateway/audit?companyId=${companyId}&app=${options.connectionId}&agent=${options.agentId}&search=${encodeURIComponent(options.search)}&limit=50`,
|
|
),
|
|
);
|
|
expect(audit.events.length, `expected audit/activity row matching ${options.search}`).toBeGreaterThan(0);
|
|
}
|
|
|
|
test.describe.serial("MCP prod Phase 5a user-story harness", () => {
|
|
test.setTimeout(180_000);
|
|
|
|
test(`${storyById("US-1").id} ${storyById("US-1").title} @mcp-runnable @mcp-us1`, async ({ page, request }) => {
|
|
const { seed, scout, mock, connectionId } = await seedConnectedFixture(request, "us1");
|
|
try {
|
|
await setScoutScript(request, scout, buildGatewayCallScript(connectionId, "sheets:list_rows"));
|
|
const invoked = await invokeHeartbeat(request, scout.id);
|
|
const run = await waitForRun(request, invoked.id);
|
|
const failedLog = run.status === "succeeded"
|
|
? null
|
|
: await json<{ content?: string }>(await request.get(`/api/heartbeat-runs/${run.id}/log?offset=0&limitBytes=65536`));
|
|
expect(
|
|
run.status,
|
|
[run.error ?? `heartbeat run ${run.id} did not succeed`, failedLog?.content].filter(Boolean).join("\n"),
|
|
).toBe("succeeded");
|
|
expect(mock.captures.some((capture) => capture.method === "tools/call" && capture.toolName === "sheets:list_rows")).toBe(true);
|
|
await expectAuditEvent(request, seed.companyId, { connectionId, agentId: scout.id, search: "sheets:list_rows" });
|
|
|
|
await page.goto(`/${seed.prefix}/apps/${connectionId}`);
|
|
await expect(page.getByRole("heading", { name: /Sheets Fixture us1/i })).toBeVisible({ timeout: 30_000 });
|
|
await screenshot(page, "US-1", "01-connected-app");
|
|
await page.goto(`/${seed.prefix}/activity?action=tool_`);
|
|
await screenshot(page, "US-1", "02-activity");
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
test(`${storyById("US-2").id} ${storyById("US-2").title} @mcp-runnable @mcp-us2`, async ({ page, request }) => {
|
|
const { seed, scout, mock, connectionId } = await seedConnectedFixture(request, "us2");
|
|
try {
|
|
const pending = await testCall(request, connectionId, scout, "sheets:update_cell", { cell: "A1", value: "approved" });
|
|
expect(pending.decision).toBe("ask_first");
|
|
expect(pending.actionRequestId).toBeTruthy();
|
|
|
|
await page.goto(`/${seed.prefix}/apps/${connectionId}/review`);
|
|
await screenshot(page, "US-2", "01-review-pending");
|
|
|
|
await approveActionRequest(request, seed.companyId, pending.actionRequestId!);
|
|
await pollTestCall(request, connectionId, pending.actionRequestId!, "done");
|
|
expect(mock.captures.filter((capture) => capture.method === "tools/call" && capture.toolName === "sheets:update_cell")).toHaveLength(1);
|
|
await expectAuditEvent(request, seed.companyId, { connectionId, agentId: scout.id, search: "sheets:update_cell" });
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
test(`${storyById("US-3").id} ${storyById("US-3").title} @mcp-runnable @mcp-us3`, async ({ page, request }) => {
|
|
const { seed, scout, mock, connectionId } = await seedConnectedFixture(request, "us3");
|
|
try {
|
|
const pending = await testCall(request, connectionId, scout, "sheets:update_cell", { cell: "A2", value: "denied" });
|
|
expect(pending.actionRequestId).toBeTruthy();
|
|
await declineActionRequest(request, seed.companyId, pending.actionRequestId!);
|
|
await pollTestCall(request, connectionId, pending.actionRequestId!, "denied");
|
|
expect(mock.captures.some((capture) => capture.method === "tools/call" && capture.toolName === "sheets:update_cell")).toBe(false);
|
|
|
|
await page.goto(`/${seed.prefix}/apps/${connectionId}/review`);
|
|
await screenshot(page, "US-3", "01-review-denied");
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
test(`${storyById("US-4").id} ${storyById("US-4").title} @mcp-runnable @mcp-us4`, async ({ page, request }) => {
|
|
const { seed, scout, mock, connectionId } = await seedConnectedFixture(request, "us4");
|
|
try {
|
|
const block = await json<{ id: string }>(await request.post(`/api/companies/${seed.companyId}/tools/policies`, {
|
|
data: {
|
|
name: "US-4 block fixture connection",
|
|
policyType: "block",
|
|
priority: 1,
|
|
selectors: { connectionId },
|
|
},
|
|
}));
|
|
const denied = await testCall(request, connectionId, scout, "sheets:list_rows");
|
|
expect(denied.decision).toBe("off");
|
|
expect(denied.error?.reasonCode).toBeTruthy();
|
|
|
|
await json(await request.patch(`/api/companies/${seed.companyId}/tools/policies/${block.id}`, {
|
|
data: { enabled: false },
|
|
}));
|
|
const allowed = await testCall(request, connectionId, scout, "sheets:list_rows");
|
|
expect(allowed.decision).toBe("allowed");
|
|
await expectAuditEvent(request, seed.companyId, { connectionId, agentId: scout.id, search: "sheets:list_rows" });
|
|
|
|
await page.goto(`/${seed.prefix}/apps/advanced/policies`);
|
|
await screenshot(page, "US-4", "01-policy-flip");
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
test(`${storyById("US-5").id} ${storyById("US-5").title} @mcp-runnable @mcp-us5`, async ({ page, request }) => {
|
|
const { seed, scout, mock, connectionId } = await seedConnectedFixture(request, "us5");
|
|
try {
|
|
const catalog = await json<{ catalog: Array<{ toolName: string }> }>(
|
|
await request.get(`/api/tool-connections/${connectionId}/catalog`),
|
|
);
|
|
expect(catalog.catalog.map((entry) => entry.toolName)).toEqual(expect.arrayContaining([
|
|
"sheets:list_rows",
|
|
"sheets:update_cell",
|
|
]));
|
|
const read = await testCall(request, connectionId, scout, "sheets:list_rows");
|
|
expect(read.decision).toBe("allowed");
|
|
|
|
await page.goto(`/${seed.prefix}/apps/connect`);
|
|
await screenshot(page, "US-5", "01-connect-your-own-entry");
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
test(`${storyById("US-6").id} ${storyById("US-6").title} @mcp-us6`, async () => {
|
|
test.skip(true, storyById("US-6").gate);
|
|
});
|
|
|
|
test(`${storyById("US-7").id} ${storyById("US-7").title} @mcp-us7`, async () => {
|
|
test.skip(true, storyById("US-7").gate);
|
|
});
|
|
|
|
test(`${storyById("US-8").id} ${storyById("US-8").title} @mcp-runnable @mcp-us8`, async ({ page, request }) => {
|
|
const { seed, mock, connectionId } = await seedConnectedFixture(request, "us8");
|
|
await mock.close();
|
|
|
|
const health = await request.post(`/api/tool-connections/${connectionId}/health-check`);
|
|
expect(health.status()).toBe(502);
|
|
await page.goto(`/${seed.prefix}/apps/connections`);
|
|
await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 30_000 });
|
|
await screenshot(page, "US-8", "01-needs-attention");
|
|
|
|
const recovered = await startMockMcp();
|
|
try {
|
|
await json(await request.patch(`/api/tool-connections/${connectionId}`, {
|
|
data: { config: { url: recovered.url } },
|
|
}));
|
|
await json(await request.post(`/api/tool-connections/${connectionId}/reconnect`, {
|
|
data: { credentialValues: { "credentials.authorization": "fresh-fixture-key" } },
|
|
}));
|
|
const after = await json<{ healthStatus: string }>(await request.get(`/api/tool-connections/${connectionId}`));
|
|
expect(after.healthStatus).toBe("ok");
|
|
await page.goto(`/${seed.prefix}/apps/${connectionId}`);
|
|
await screenshot(page, "US-8", "02-recovered");
|
|
} finally {
|
|
await recovered.close();
|
|
}
|
|
});
|
|
|
|
test(`${storyById("US-9").id} ${storyById("US-9").title} @mcp-runnable @mcp-us9`, async ({ page, request }) => {
|
|
const { seed, scout, mock, connectionId } = await seedConnectedFixture(request, "us9");
|
|
try {
|
|
for (const value of ["first", "second"]) {
|
|
const pending = await testCall(request, connectionId, scout, "sheets:update_cell", { cell: "B1", value });
|
|
expect(pending.decision).toBe("ask_first");
|
|
await page.goto(`/${seed.prefix}/apps/${connectionId}/review`);
|
|
await screenshot(page, "US-9", `review-${value}`);
|
|
await approveActionRequest(request, seed.companyId, pending.actionRequestId!);
|
|
await pollTestCall(request, connectionId, pending.actionRequestId!, "done");
|
|
}
|
|
expect(mock.captures.filter((capture) => capture.method === "tools/call" && capture.toolName === "sheets:update_cell")).toHaveLength(2);
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
test(`${storyById("US-10").id} ${storyById("US-10").title} @mcp-runnable @mcp-us10`, async ({ page, request }) => {
|
|
const { seed, mock, connectionId } = await seedConnectedFixture(request, "us10");
|
|
try {
|
|
await page.goto(`/${seed.prefix}/apps/${connectionId}`);
|
|
await expect(page.getByRole("heading", { name: /Sheets Fixture us10/i })).toBeVisible({ timeout: 30_000 });
|
|
await screenshot(page, "US-10", "01-apps-detail");
|
|
|
|
await page.goto(`/${seed.prefix}/apps/advanced`);
|
|
await expect(page.getByRole("heading", { name: "Advanced setup" })).toBeVisible({ timeout: 20_000 });
|
|
await screenshot(page, "US-10", "02-admin-depth");
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
});
|