paperclip/tests/runner-e2e/support.test.ts

734 lines
23 KiB
TypeScript

import { execFileSync } from "node:child_process";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { packageEvidence } from "./evidence.js";
import { RunnerApi } from "./api.js";
import { FixtureRegistry } from "./fixture-registry.js";
import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js";
import {
assertIsolatedServerEnvironment,
buildPaperclipServerEnvironment,
resolvePaperclipRunnerBinaryForHarness,
} from "./harness-env.js";
import { runnerExecutionById } from "./catalog.js";
import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js";
import { evaluateMatchers } from "./matchers.js";
import {
assertSecretFree,
findSecretLeak,
findSecretLeakInJsonValues,
findSecretLeakInDirectory,
isEphemeralCodexRuntimeAuthFile,
redactText,
sanitizeJson,
} from "./redaction.js";
import { parseDarwinSharedMemory } from "./shared-memory.js";
import {
isNonExecutingReviewFenceRun,
numberedPlanStepCount,
} from "./run-observations.js";
import { runnerE2EWebServerCommand } from "./web-server-command.js";
const cleanupDirectories: string[] = [];
afterEach(async () => {
vi.unstubAllEnvs();
vi.unstubAllGlobals();
await Promise.all(
cleanupDirectories
.splice(0)
.map((directory) => rm(directory, { recursive: true, force: true })),
);
});
describe("runner E2E local binary resolution", () => {
const localNativeExecution = runnerExecutionById(
"core-compatibility.runner-codex.local.message-marker",
);
it("uses the debug runner binary built by the E2E workflow", () => {
expect(
resolvePaperclipRunnerBinaryForHarness(
[localNativeExecution],
"/repository",
undefined,
"linux",
),
).toBe(
path.join(
"/repository",
"packages/paperclip-runner/runner/target/debug/paperclip-runnerd",
),
);
});
it("preserves an explicit runner binary override", () => {
expect(
resolvePaperclipRunnerBinaryForHarness(
[localNativeExecution],
"/repository",
"/custom/paperclip-runnerd",
"linux",
),
).toBe("/custom/paperclip-runnerd");
});
});
describe("runner E2E sensitive API boundary", () => {
it("keeps secret request bodies out of Playwright API tracing", async () => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "43123");
const playwrightPost = vi.fn();
const fetchMock = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ id: "secret-id" }), {
status: 201,
headers: { "content-type": "application/json" },
}),
);
vi.stubGlobal("fetch", fetchMock);
const api = new RunnerApi({ post: playwrightPost } as never);
await expect(
api.postSensitive("/api/companies/company-id/secrets", {
value: "fixture-secret-value",
}),
).resolves.toEqual({ id: "secret-id" });
expect(playwrightPost).not.toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalledWith(
new URL("http://127.0.0.1:43123/api/companies/company-id/secrets"),
expect.objectContaining({ method: "POST" }),
);
});
});
describe("runner E2E structured evidence scanning", () => {
it("does not invent a secret shape across JSON syntax boundaries", () => {
const value = {
OPENROUTER_API_KEY: "generated-secret-reference-id",
};
expect(findSecretLeak(JSON.stringify(value), [])).toBe(
"secret-shaped value",
);
expect(findSecretLeakInJsonValues(value, [])).toBeNull();
});
it("still rejects exact and provider-shaped values in nested JSON", () => {
expect(
findSecretLeakInJsonValues(
{ nested: ["safe", "campaign-secret-value"] },
["campaign-secret-value"],
),
).toBe("exact secret value");
expect(
findSecretLeakInJsonValues({ nested: "sk-proj-abcdefghijklmnop" }, []),
).toBe("secret-shaped value");
});
});
describe("runner E2E fixture registry", () => {
it("sets up in dependency order and tears down in reverse", async () => {
const events: string[] = [];
const registry = new FixtureRegistry()
.register({
id: "company",
setup: async () => {
events.push("setup-company");
return "c";
},
teardown: async () => {
events.push("teardown-company");
},
})
.register({
id: "agent",
dependencies: ["company"],
setup: async () => {
events.push("setup-agent");
return "a";
},
teardown: async () => {
events.push("teardown-agent");
},
});
const active = await registry.setupAll();
await active.teardown();
expect(events).toEqual([
"setup-company",
"setup-agent",
"teardown-agent",
"teardown-company",
]);
});
it("tears down partial setup after a failure", async () => {
const cleanup = vi.fn();
const registry = new FixtureRegistry()
.register({ id: "company", setup: async () => "c", teardown: cleanup })
.register({
id: "agent",
dependencies: ["company"],
setup: async () => {
throw new Error("boom");
},
});
await expect(registry.setupAll()).rejects.toThrow("boom");
expect(cleanup).toHaveBeenCalledOnce();
});
it("preserves setup and partial-cleanup failures", async () => {
const registry = new FixtureRegistry()
.register({
id: "company",
setup: async () => "c",
teardown: async () => {
throw new Error("cleanup transport returned 503");
},
})
.register({
id: "agent",
dependencies: ["company"],
setup: async () => {
throw new Error("agent setup failed");
},
});
await expect(registry.setupAll()).rejects.toMatchObject({
name: "AggregateError",
message: expect.stringContaining("cleanup failed"),
errors: expect.arrayContaining([
expect.objectContaining({ message: "agent setup failed" }),
expect.objectContaining({ message: "Fixture teardown failed" }),
]),
});
});
});
describe("runner E2E matchers", () => {
it("normalizes message text and evaluates state invariants", async () => {
const results = await evaluateMatchers(
[
{ kind: "message_contains", expected: "PAPERCLIP_E2E_OK_nonce" },
{
kind: "message_occurrences",
expected: "PAPERCLIP_E2E_OK_nonce",
count: 1,
},
{ kind: "issue_status", expected: "done" },
{ kind: "runtime_mode", expected: "native" },
],
{
message: " complete PAPERCLIP\\_E2E\\_OK\\_nonce ",
issueStatus: "done",
runtimeMode: "native",
},
);
expect(results.every((result) => result.passed)).toBe(true);
});
it("requires an exact number of complete visible markers", async () => {
const [result] = await evaluateMatchers(
[{ kind: "message_occurrences", expected: "FINAL_marker", count: 1 }],
{ message: "FINAL\\_marker\nFINAL_marker" },
);
expect(result).toMatchObject({ passed: false });
expect(result?.detail).toContain("observed 2");
});
it("normalizes ordered fragments and evaluates nested JSON Schema", async () => {
const results = await evaluateMatchers(
[
{
kind: "message_ordered",
expected: ["first marker", "second marker"],
},
{
kind: "json_schema",
schema: {
type: "object",
required: ["run"],
additionalProperties: false,
properties: {
run: {
type: "object",
required: ["status"],
properties: { status: { const: "succeeded" } },
},
},
},
},
],
{
message: "first marker\nsecond marker",
json: { run: { status: "succeeded" } },
},
);
expect(results.every((result) => result.passed)).toBe(true);
});
});
describe("runner E2E run observations", () => {
it("counts provider-equivalent numbered Plan step formats", () => {
expect(numberedPlanStepCount("1. First\n2) Second")).toBe(2);
expect(
numberedPlanStepCount(
"# Plan\n\nStep 1 — First\n\nStep 2 — Second\n\nStep 3: Verify",
),
).toBe(3);
expect(
numberedPlanStepCount("## **Step 1** — First\n- **2.** Second"),
).toBe(2);
});
it("excludes only queued continuations fenced while awaiting review", () => {
expect(
isNonExecutingReviewFenceRun({
status: "cancelled",
errorCode: "issue_continuation_waiting_on_review",
}),
).toBe(true);
expect(
isNonExecutingReviewFenceRun({
status: "failed",
errorCode: "issue_continuation_waiting_on_review",
}),
).toBe(false);
expect(
isNonExecutingReviewFenceRun({
status: "cancelled",
errorCode: "provider_failure",
}),
).toBe(false);
});
});
describe("runner E2E failure policy", () => {
it("retries only transient infrastructure failures", () => {
expect(
classifyFailure(new Error("Daytona preview connection timed out")),
).toBe("transient_infrastructure");
expect(
classifyFailure(
new Error(
"Browser bootstrap failed before task creation: New Task button timed out after a Vite 504",
),
),
).toBe("transient_infrastructure");
expect(
shouldRetryFailure(
classifyFailure(new Error("Daytona preview connection timed out")),
),
).toBe(true);
expect(
shouldRetryFailure(classifyFailure(new Error("marker matcher failed"))),
).toBe(false);
expect(
classifyFailure(
new Error(
"Timed out waiting for issue abc and heartbeat run terminal state",
),
),
).toBe("candidate_failure");
expect(
classifyFailure(
new Error("Provider request timed out during generation"),
),
).toBe("transient_infrastructure");
expect(
classifyFailure(
new Error(
"runner_ingress_unavailable: paperclip-runnerd: cumulative ACK cannot move beyond the produced source cursor",
),
),
).toBe("transient_infrastructure");
expect(
shouldRetryFailure(classifyFailure(new Error("invalid API key"))),
).toBe(false);
expect(
classifyFailure(
new Error("Daytona lease cleanup failed: provider returned 503"),
),
).toBe("transient_infrastructure");
expect(classifyFailure(new Error("cleanup invariant failure"))).toBe(
"cleanup_failure",
);
});
});
describe("runner E2E server isolation", () => {
it("strips database and paid-provider credentials from the Paperclip process", () => {
const env = buildPaperclipServerEnvironment(
{
PATH: "/bin",
DATABASE_URL: "postgres://existing",
DATABASE_MIGRATION_URL: "postgres://migration",
OPENAI_API_KEY: "openai",
ANTHROPIC_API_KEY: "anthropic",
OPENROUTER_API_KEY: "openrouter",
DAYTONA_API_KEY: "daytona",
OPENAI_ORG_ID: "also-provider-sensitive",
PAPERCLIP_API_KEY: "ambient-board-key",
PAPERCLIP_AGENT_API_KEY: "ambient-agent-key",
PAPERCLIP_TASK_BRIDGE_TOKEN: "ambient-task-token",
PAPERCLIP_SETUP_TOKEN: "ambient-setup-token",
PAPERCLIP_SECRETS_MASTER_KEY: "ambient-master-key",
PAPERCLIP_SECRETS_MASTER_KEY_FILE: "/outside/master.key",
PAPERCLIP_STORAGE_S3_BUCKET: "production-bucket",
},
{
PAPERCLIP_HOME: "/tmp/cell/paperclip-home",
PAPERCLIP_CONFIG: "/tmp/cell/paperclip-home/instances/e2e/config.json",
PAPERCLIP_AGENT_JWT_SECRET: "generated-agent-jwt",
PAPERCLIP_DECISION_SIGNING_SECRET: "generated-decision-key",
PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: "generated-tool-key",
BETTER_AUTH_SECRET: "generated-auth-key",
},
);
expect(env.PATH).toBe("/bin");
expect(env.DATABASE_URL).toBeUndefined();
expect(env.OPENAI_API_KEY).toBeUndefined();
expect(env.OPENAI_ORG_ID).toBeUndefined();
expect(env.PAPERCLIP_API_KEY).toBeUndefined();
expect(env.PAPERCLIP_AGENT_API_KEY).toBeUndefined();
expect(env.PAPERCLIP_AGENT_JWT_SECRET).toBe("generated-agent-jwt");
expect(env.PAPERCLIP_TASK_BRIDGE_TOKEN).toBeUndefined();
expect(env.PAPERCLIP_SETUP_TOKEN).toBeUndefined();
expect(env.PAPERCLIP_SECRETS_MASTER_KEY).toBeUndefined();
expect(env.PAPERCLIP_SECRETS_MASTER_KEY_FILE).toBeUndefined();
expect(env.PAPERCLIP_STORAGE_S3_BUCKET).toBeUndefined();
expect(() =>
assertIsolatedServerEnvironment(env, {
temporaryRoot: "/tmp/cell",
paperclipHome: "/tmp/cell/paperclip-home",
configPath: "/tmp/cell/paperclip-home/instances/e2e/config.json",
}),
).not.toThrow();
});
it("uses absolute repository paths for the Playwright web server", () => {
const command = runnerE2EWebServerCommand("/workspace/paperclip");
expect(command).toContain(
"'/workspace/paperclip/cli/node_modules/tsx/dist/cli.mjs'",
);
expect(command).toContain(
"'/workspace/paperclip/tests/runner-e2e/server.ts'",
);
});
it("accepts only embedded state paths beneath the temporary root", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-instance-isolation-test-"),
);
cleanupDirectories.push(root);
const database = path.join(root, "paperclip-home", "db");
const secretsKey = path.join(
root,
"paperclip-home",
"secrets",
"master.key",
);
const configPath = path.join(root, "paperclip-home", "config.json");
await mkdir(database, { recursive: true });
await mkdir(path.dirname(secretsKey), { recursive: true });
await writeFile(secretsKey, "generated-master-key");
const config = {
database: {
mode: "embedded-postgres",
embeddedPostgresDataDir: database,
backup: { dir: path.join(root, "backups") },
},
logging: { logDir: path.join(root, "logs") },
storage: {
provider: "local_disk",
localDisk: { baseDir: path.join(root, "storage") },
},
secrets: {
provider: "local_encrypted",
strictMode: true,
localEncrypted: { keyFilePath: secretsKey },
},
};
await writeFile(configPath, JSON.stringify(config));
await expect(
assertEmbeddedDatabaseIsolation(configPath, root),
).resolves.toBeUndefined();
await writeFile(
configPath,
JSON.stringify({
...config,
storage: {
...config.storage,
localDisk: { baseDir: "/outside/storage" },
},
}),
);
await expect(
assertEmbeddedDatabaseIsolation(configPath, root),
).rejects.toThrow("storage path escaped");
});
});
describe("runner E2E evidence redaction", () => {
const secret = "sk-proj-supersecretvalue123456";
it("redacts exact and shaped credentials recursively", () => {
expect(redactText(`token=${secret}`, [secret])).toBe("token=[REDACTED]");
expect(sanitizeJson({ nested: [secret] }, [secret])).toEqual({
nested: ["[REDACTED]"],
});
expect(sanitizeJson("paperclip.runner-e2e.evidence/v1", [secret])).toBe(
"paperclip.runner-e2e.evidence/v1",
);
});
it("detects leaks and accepts sanitized evidence", () => {
expect(findSecretLeak(Buffer.from(secret), [secret])).toBeTruthy();
expect(() => assertSecretFree("safe", [secret], "fixture")).not.toThrow();
expect(() =>
assertSecretFree(
"sk-proj-documentationfixture123456",
[secret],
"API source",
{ includeShapes: false },
),
).not.toThrow();
});
it("finds exact credentials across streamed persisted-state chunks", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-secret-scan-test-"),
);
cleanupDirectories.push(root);
await writeFile(
path.join(root, "database.bin"),
Buffer.concat([
Buffer.alloc(65_530, "x"),
Buffer.from(secret),
Buffer.alloc(32, "y"),
]),
);
await expect(
findSecretLeakInDirectory(root, [secret]),
).resolves.toMatchObject({ reason: "exact secret value" });
});
it("can ignore fake key shapes while scanning persisted package state", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-secret-shape-test-"),
);
cleanupDirectories.push(root);
await writeFile(
path.join(root, "provider-fixture.test.ts"),
'const fake = "sk-proj-documentationfixture123456";\n',
);
await expect(
findSecretLeakInDirectory(root, [secret], { includeShapes: false }),
).resolves.toBeNull();
});
it("can narrowly exclude a verified ephemeral runtime credential file", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-ephemeral-auth-test-"),
);
cleanupDirectories.push(root);
const runtimeAuth = path.join(root, "codex-home", "auth.json");
const forbiddenConfig = path.join(root, "config.json");
await mkdir(path.dirname(runtimeAuth), { recursive: true });
await writeFile(runtimeAuth, secret, { mode: 0o600 });
await writeFile(forbiddenConfig, secret);
await expect(
findSecretLeakInDirectory(root, [secret], {
includeShapes: false,
ignoreFile: (file) => file === runtimeAuth,
}),
).resolves.toMatchObject({ file: forbiddenConfig });
});
it("recognizes both managed and durable-session Codex runtime auth files", () => {
const root = path.join(os.tmpdir(), "paperclip-home");
expect(
isEphemeralCodexRuntimeAuthFile(
root,
path.join(
root,
"instances/instance-1/companies/company-1/agents/agent-1/codex-home/auth.json",
),
),
).toBe(true);
expect(
isEphemeralCodexRuntimeAuthFile(
root,
path.join(
root,
"instances/instance-1/runtime/paperclip-runner/durable-sessions/session-1/codex-home/auth.json",
),
),
).toBe(true);
expect(
isEphemeralCodexRuntimeAuthFile(
root,
path.join(root, "instances/instance-1/runtime/auth.json"),
),
).toBe(false);
expect(
isEphemeralCodexRuntimeAuthFile(
root,
path.join(
root,
"instances/instance-1/runtime/paperclip-runner/durable-sessions/session-1/codex-home/config.toml",
),
),
).toBe(false);
});
it("publishes only allowlisted sanitized files and reports source leaks", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-evidence-test-"),
);
cleanupDirectories.push(root);
const privateDir = path.join(root, "private");
const uploadDir = path.join(root, "upload");
await mkdir(privateDir, { recursive: true });
await writeFile(
path.join(privateDir, "result.json"),
JSON.stringify({ error: secret }),
);
await writeFile(path.join(privateDir, "database.sqlite"), secret);
const packaged = await packageEvidence({
privateDir,
uploadDir,
secrets: [secret],
expectPassScreenshot: false,
});
expect(packaged.leaks).toEqual([
{ file: "result.json", reason: "exact secret value" },
]);
expect(
await readFile(path.join(uploadDir, "result.json"), "utf8"),
).toContain("[REDACTED]");
await expect(
readFile(path.join(uploadDir, "database.sqlite")),
).rejects.toThrow();
});
it("keeps raster evidence private to CI and rejects active SVG content", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-visual-evidence-test-"),
);
cleanupDirectories.push(root);
const privateDir = path.join(root, "private");
const uploadDir = path.join(root, "upload");
const playwrightOutput = path.join(privateDir, "playwright-output");
await mkdir(playwrightOutput, { recursive: true });
await writeFile(path.join(privateDir, "final-state.png"), "png");
await writeFile(path.join(playwrightOutput, "failure.webm"), "webm");
await writeFile(
path.join(playwrightOutput, "active.svg"),
"<svg onload='alert(1)' />",
);
const packaged = await packageEvidence({
privateDir,
uploadDir,
secrets: [secret],
expectPassScreenshot: false,
});
expect(packaged.files).toEqual(
expect.arrayContaining([
"final-state.png",
path.join("playwright-output", "failure.webm"),
]),
);
expect(packaged.files).not.toContain(
path.join("playwright-output", "active.svg"),
);
await expect(
readFile(path.join(playwrightOutput, "active.svg"), "utf8"),
).resolves.toContain("onload");
await expect(
readFile(path.join(uploadDir, "playwright-output", "active.svg")),
).rejects.toThrow();
});
it("preserves valid JSON while redacting escaped command diagnostics", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-json-evidence-test-"),
);
cleanupDirectories.push(root);
const privateDir = path.join(root, "private");
const uploadDir = path.join(root, "upload");
await mkdir(path.join(privateDir, "snapshots"), { recursive: true });
await writeFile(
path.join(privateDir, "snapshots", "api-state.json"),
JSON.stringify({
log: String.raw`curl -H \"Authorization: Bearer temporary-run-token\" \\\n+ \"$PAPERCLIP_API_URL/api/issues\"`,
}),
);
await packageEvidence({
privateDir,
uploadDir,
secrets: [secret],
expectPassScreenshot: false,
});
const uploaded = await readFile(
path.join(uploadDir, "snapshots", "api-state.json"),
"utf8",
);
expect(() => JSON.parse(uploaded)).not.toThrow();
expect(uploaded).toContain("[REDACTED]");
});
it("streams large ZIP evidence and detects exact secrets", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-zip-evidence-test-"),
);
cleanupDirectories.push(root);
const privateDir = path.join(root, "private");
const uploadDir = path.join(root, "upload");
const blobDir = path.join(privateDir, "blob-report");
await mkdir(blobDir, { recursive: true });
await writeFile(
path.join(blobDir, "trace.txt"),
Buffer.concat([Buffer.alloc(2 * 1024 * 1024, "x"), Buffer.from(secret)]),
);
execFileSync("zip", ["-q", "report.zip", "trace.txt"], {
cwd: blobDir,
});
await rm(path.join(blobDir, "trace.txt"));
const packaged = await packageEvidence({
privateDir,
uploadDir,
secrets: [secret],
expectPassScreenshot: false,
});
expect(packaged.leaks).toEqual([
{
file: path.join("blob-report", "report.zip"),
reason: "exact secret value",
},
]);
await expect(
readFile(path.join(uploadDir, "blob-report", "report.zip")),
).rejects.toThrow();
});
});
describe("runner E2E macOS shared-memory cleanup", () => {
it("parses only shared-memory rows from ipcs output", () => {
expect(
parseDarwinSharedMemory(
`IPC status from <running system>\nT ID KEY MODE OWNER GROUP CREATOR CGROUP NATTCH SEGSZ CPID LPID\nm 327709 0x028ed0ab --rw------- dotta staff dotta staff 0 56 52172 52172\ns 123 0x0 --ra------- dotta staff\n`,
),
).toEqual([
{
id: "327709",
owner: "dotta",
attachments: 0,
creatorPid: 52172,
},
]);
});
});