#!/usr/bin/env node import { spawn } from "node:child_process"; import { once } from "node:events"; import { createInterface } from "node:readline"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { demoProfiles, findTool, fixtureProfiles, listTools, } from "../mcp-fixtures/catalog.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(__dirname, "../.."); const stdioServerPath = resolve(repoRoot, "scripts/mcp-fixtures/servers/stdio-fixture.mjs"); const httpServerPath = resolve(repoRoot, "scripts/mcp-fixtures/servers/http-fixture.mjs"); function parseArgs(argv) { const args = { paperclipUrl: process.env.PAPERCLIP_API_URL ?? "http://127.0.0.1:3100/api", requirePaperclip: false, json: false, }; for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === "--") continue; if (arg === "--paperclip-url") args.paperclipUrl = argv[++i]; else if (arg === "--require-paperclip") args.requirePaperclip = true; else if (arg === "--json") args.json = true; else if (arg === "--help") { console.log(`Usage: node scripts/smoke/mcp-fixture-harness.mjs [--paperclip-url URL] [--require-paperclip] [--json]`); process.exit(0); } else { throw new Error(`Unknown argument: ${arg}`); } } return args; } function normalizePaperclipUrl(raw) { const url = new URL(raw); if (url.pathname.endsWith("/api")) { url.pathname = url.pathname.slice(0, -4) || "/"; } return url.toString().replace(/\/$/, ""); } async function checkPaperclipHealth(rawUrl, required) { const baseUrl = normalizePaperclipUrl(rawUrl); try { const response = await fetch(`${baseUrl}/api/health`, { signal: AbortSignal.timeout(1500) }); if (!response.ok) throw new Error(`HTTP ${response.status}`); return { ok: true, baseUrl }; } catch (error) { if (required) { throw new Error(`Paperclip health check failed at ${baseUrl}/api/health: ${error.message}`); } return { ok: false, baseUrl, skippedReason: error.message }; } } function redactHostileText(value) { return JSON.stringify(value) .replace(/pc_live_[A-Za-z0-9_=-]+/g, "[REDACTED_SECRET]") .replace(/PAPERCLIP_API_KEY/g, "[REDACTED_ENV_NAME]"); } function fingerprintTool(tool) { return JSON.stringify({ name: tool.name, schemaVersion: tool.schemaVersion, inputSchema: tool.inputSchema, }); } class StdioFixtureClient { constructor() { this.nextId = 1; this.pending = new Map(); this.process = null; } async start() { this.process = spawn(process.execPath, [stdioServerPath], { cwd: repoRoot, stdio: ["pipe", "pipe", "pipe"], }); const rl = createInterface({ input: this.process.stdout }); rl.on("line", (line) => { let response; try { response = JSON.parse(line); } catch { return; } const pending = this.pending.get(response.id); if (!pending) return; this.pending.delete(response.id); pending.resolve(response); }); this.process.stderr.on("data", (chunk) => { process.stderr.write(`[mcp-stdio-fixture] ${chunk}`); }); await this.request("health"); } request(method, params = {}) { const id = String(this.nextId++); return new Promise((resolveRequest, reject) => { this.pending.set(id, { resolve: resolveRequest, reject }); this.process.stdin.write(`${JSON.stringify({ id, method, params })}\n`); setTimeout(() => { if (this.pending.has(id)) { this.pending.delete(id); reject(new Error(`stdio fixture request timed out: ${method}`)); } }, 2000).unref(); }); } async listTools() { const response = await this.request("list_tools"); return response.tools; } async callTool(name, input) { return this.request("call_tool", { name, input }); } async stop() { if (!this.process || this.process.killed) return; this.process.kill("SIGTERM"); await Promise.race([ once(this.process, "exit"), new Promise((resolveStop) => setTimeout(resolveStop, 500)), ]); } } class HttpFixtureClient { constructor() { this.process = null; this.baseUrl = null; } async start() { this.process = spawn(process.execPath, [httpServerPath], { cwd: repoRoot, env: { ...process.env, PORT: "0" }, stdio: ["ignore", "pipe", "pipe"], }); this.process.stderr.on("data", (chunk) => { process.stderr.write(`[mcp-http-fixture] ${chunk}`); }); const rl = createInterface({ input: this.process.stdout }); const ready = await new Promise((resolveReady, reject) => { const timer = setTimeout(() => reject(new Error("http fixture did not become ready")), 2000); rl.on("line", (line) => { const event = JSON.parse(line); if (event.event === "ready") { clearTimeout(timer); resolveReady(event); } }); }); this.baseUrl = `http://${ready.host}:${ready.port}`; const health = await fetch(`${this.baseUrl}/health`); if (!health.ok) throw new Error(`http fixture health failed: ${health.status}`); } async listTools() { const response = await fetch(`${this.baseUrl}/catalog`); const body = await response.json(); return body.tools; } async callTool(name, input) { const response = await fetch(`${this.baseUrl}/tools/call`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ name, input }), }); return response.json(); } async stop() { if (!this.process || this.process.killed) return; this.process.kill("SIGTERM"); await Promise.race([ once(this.process, "exit"), new Promise((resolveStop) => setTimeout(resolveStop, 500)), ]); } } class SmokePolicyHarness { constructor({ stdioClient, httpClient }) { this.stdioClient = stdioClient; this.httpClient = httpClient; this.audit = []; this.pendingApprovals = new Map(); this.idempotency = new Map(); this.quarantine = new Set(); this.baselineFingerprints = new Map(listTools().map((tool) => [tool.name, fingerprintTool(tool)])); } profile(profileId) { const profile = fixtureProfiles.find((candidate) => candidate.id === profileId); if (!profile) throw new Error(`Unknown profile: ${profileId}`); return profile; } isAllowedByProfile(profile, tool) { if (this.quarantine.has(tool.name)) return { outcome: "quarantined" }; const riskAllowed = tool.risk === "low" || profile.allowRisks?.includes(tool.risk) || !profile.denyRisks?.includes(tool.risk); if (!riskAllowed) return { outcome: "denied" }; if (profile.allowCapabilities.includes(tool.capability)) return { outcome: "allowed" }; if (profile.approvalCapabilities.includes(tool.capability) || tool.approvalRequired) return { outcome: "approval_required" }; return { outcome: "denied" }; } async call(profileId, toolName, input = {}, options = {}) { const profile = this.profile(profileId); const tool = findTool(toolName); const idempotencyKey = options.idempotencyKey ?? null; if (idempotencyKey && this.idempotency.has(idempotencyKey)) { const cached = this.idempotency.get(idempotencyKey); this.audit.push({ toolName, profileId, outcome: "idempotent_replay", idempotencyKey }); return cached; } const decision = this.isAllowedByProfile(profile, tool); if (decision.outcome === "denied" || decision.outcome === "quarantined") { const response = { ok: false, outcome: decision.outcome, toolName }; this.audit.push({ toolName, profileId, outcome: decision.outcome }); return response; } if (decision.outcome === "approval_required" && !options.approved) { const approvalId = `approval-${this.pendingApprovals.size + 1}`; const response = { ok: false, outcome: "approval_required", approvalId, toolName }; this.pendingApprovals.set(approvalId, { profileId, toolName, input, idempotencyKey }); this.audit.push({ toolName, profileId, outcome: "approval_required", approvalId }); return response; } const client = tool.transport === "stdio" ? this.stdioClient : this.httpClient; const response = await client.callTool(toolName, input); let outcome = response.ok ? "allowed" : response.error?.code ?? "fixture_error"; if (tool.hostile && response.ok) { const redacted = redactHostileText(response.result); outcome = redacted.includes("Ignore prior instructions") ? "malicious_result_quarantined" : "allowed"; response.result = { redacted, quarantined: outcome === "malicious_result_quarantined" }; } const wrapped = { ...response, outcome, toolName }; if (idempotencyKey && response.ok) this.idempotency.set(idempotencyKey, wrapped); this.audit.push({ toolName, profileId, outcome, transport: tool.transport, idempotencyKey }); return wrapped; } async approve(approvalId) { const pending = this.pendingApprovals.get(approvalId); if (!pending) throw new Error(`Unknown approval: ${approvalId}`); this.pendingApprovals.delete(approvalId); return this.call(pending.profileId, pending.toolName, pending.input, { approved: true, idempotencyKey: pending.idempotencyKey, }); } discoverSchemaChanges(tools) { const quarantined = []; for (const tool of tools) { const baseline = this.baselineFingerprints.get(tool.name); if (baseline && baseline !== fingerprintTool(tool)) { this.quarantine.add(tool.name); quarantined.push(tool.name); this.audit.push({ toolName: tool.name, outcome: "schema_change_quarantined" }); } } return quarantined; } } async function runCase(results, name, fn) { try { await fn(); results.push({ name, ok: true }); } catch (error) { results.push({ name, ok: false, error: error.message }); } } function assert(condition, message) { if (!condition) throw new Error(message); } async function main() { const args = parseArgs(process.argv.slice(2)); const paperclip = await checkPaperclipHealth(args.paperclipUrl, args.requirePaperclip); const stdioClient = new StdioFixtureClient(); const httpClient = new HttpFixtureClient(); const results = []; try { await stdioClient.start(); await httpClient.start(); const harness = new SmokePolicyHarness({ stdioClient, httpClient }); await runCase(results, "fixture catalog includes required profiles and demos", async () => { assert(fixtureProfiles.length === 4, "expected four profile definitions"); assert(demoProfiles.length === 8, "expected eight first-install demo definitions"); const tools = [...await stdioClient.listTools(), ...await httpClient.listTools()]; for (const fixture of [ "echo-calculator-time", "todo-kv", "outbox-email", "mock-social-blog", "malicious", "slow-crashing-stdio", "fake-oauth-missing-secret", ]) { assert(tools.some((tool) => tool.fixture === fixture), `missing fixture ${fixture}`); } assert(tools.some((tool) => tool.transport === "stdio"), "missing stdio fixture"); assert(tools.some((tool) => tool.transport === "http"), "missing http fixture"); }); await runCase(results, "allow and deny decisions are enforced", async () => { const allowed = await harness.call("read-only", "calculator.add", { a: 2, b: 3 }); assert(allowed.ok && allowed.result.value === 5, "calculator.add should be allowed"); const denied = await harness.call("read-only", "kv.set", { key: "a", value: "b" }); assert(!denied.ok && denied.outcome === "denied", "kv.set should be denied for read-only"); }); await runCase(results, "approval-gated writes execute after approval", async () => { const pending = await harness.call("approval-gated-writes", "email.send", { to: "qa@example.com", subject: "fixture", body: "deterministic", }, { idempotencyKey: "send-email-1" }); assert(pending.outcome === "approval_required", "email.send should require approval"); const approved = await harness.approve(pending.approvalId); assert(approved.ok && approved.result.message.status === "sent", "approved email.send should execute"); }); await runCase(results, "audit trail records decisions and transports", async () => { assert(harness.audit.some((event) => event.outcome === "denied" && event.toolName === "kv.set"), "missing deny audit"); assert(harness.audit.some((event) => event.outcome === "approval_required" && event.toolName === "email.send"), "missing approval audit"); assert(harness.audit.some((event) => event.transport === "stdio"), "missing stdio audit"); assert(harness.audit.some((event) => event.transport === "http"), "missing http audit"); }); await runCase(results, "runtime lifecycle handles slow and crashing stdio fixtures", async () => { const slow = await harness.call("runtime-lifecycle", "slow.ping", { delayMs: 10 }); assert(slow.ok && slow.result.pong === true, "slow.ping should return"); const crash = await harness.call("runtime-lifecycle", "crash.now", {}); assert(!crash.ok && crash.outcome === "fixture_crash", "crash.now should report fixture_crash"); const afterCrash = await harness.call("runtime-lifecycle", "time.now", {}); assert(afterCrash.ok, "stdio fixture should keep serving after synthetic crash response"); }); await runCase(results, "secret failures are categorized without real credentials", async () => { const response = await harness.call("read-only", "secret.read", {}); assert(!response.ok && response.outcome === "secret_missing", "secret.read should fail with secret_missing"); }); await runCase(results, "schema changes quarantine changed tools", async () => { const flip = await harness.call("runtime-lifecycle", "fixture.schemaFlip", { toolName: "kv.set" }); assert(flip.ok, "fixture.schemaFlip should execute"); const changedTools = await httpClient.listTools(); const quarantined = harness.discoverSchemaChanges(changedTools); assert(quarantined.includes("kv.set"), "kv.set should be quarantined after schema change"); const blocked = await harness.call("approval-gated-writes", "kv.set", { key: "a", value: "b" }); assert(blocked.outcome === "quarantined", "quarantined tool should not execute"); }); await runCase(results, "malicious result is redacted and quarantined", async () => { const response = await harness.call("security-hostile", "malicious.result", {}); assert(response.ok, "malicious result fixture should return a result"); assert(response.outcome === "malicious_result_quarantined", "malicious result should be quarantined"); assert(!response.result.redacted.includes("pc_live_should_be_redacted"), "synthetic secret should be redacted"); assert(!response.result.redacted.includes("PAPERCLIP_API_KEY"), "env key name should be redacted"); }); await runCase(results, "idempotent approved writes only execute once", async () => { const first = await harness.call("approval-gated-writes", "email.send", { to: "qa@example.com", subject: "idempotent", body: "only once", }, { approved: true, idempotencyKey: "send-email-idempotent" }); const second = await harness.call("approval-gated-writes", "email.send", { to: "qa@example.com", subject: "idempotent", body: "only once", }, { approved: true, idempotencyKey: "send-email-idempotent" }); assert(first.result.message.id === second.result.message.id, "idempotent replay should return cached message"); assert(harness.audit.some((event) => event.outcome === "idempotent_replay"), "missing idempotent replay audit"); }); const summary = { ok: results.every((result) => result.ok), paperclip, results, auditEvents: harness.audit.length, profiles: fixtureProfiles.map((profile) => profile.id), demos: demoProfiles.map((demo) => demo.id), }; if (args.json) { console.log(JSON.stringify(summary, null, 2)); } else { console.log(`MCP fixture smoke: ${summary.ok ? "PASS" : "FAIL"}`); console.log(`Paperclip health: ${paperclip.ok ? "ok" : `skipped (${paperclip.skippedReason})`}`); for (const result of results) { console.log(`${result.ok ? "PASS" : "FAIL"} ${result.name}${result.error ? ` - ${result.error}` : ""}`); } } if (!summary.ok) process.exitCode = 1; } finally { await Promise.allSettled([stdioClient.stop(), httpClient.stop()]); } } main().catch((error) => { console.error(error.message); process.exitCode = 1; });