1180 lines
38 KiB
TypeScript
1180 lines
38 KiB
TypeScript
import { randomBytes } from "node:crypto";
|
|
import { prepareCodexCiSandbox } from "./codex-ci-sandbox.js";
|
|
import { spawn } from "node:child_process";
|
|
import { createWriteStream } from "node:fs";
|
|
import { createRequire } from "node:module";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import {
|
|
access,
|
|
chmod,
|
|
cp,
|
|
lstat,
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
rm,
|
|
symlink,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { isImmutableDaytonaImage, runnerMatrix } from "./catalog.js";
|
|
import { renderRunnerE2EDashboard } from "./dashboard.js";
|
|
import { packageEvidence } from "./evidence.js";
|
|
import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js";
|
|
import { buildRunnerCampaign } from "./history.js";
|
|
import {
|
|
buildRunnerE2EProcessEnvironment,
|
|
resolvePaperclipRemoteRunnerBinaryForHarness,
|
|
resolvePaperclipRunnerBinaryForHarness,
|
|
} from "./harness-env.js";
|
|
import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js";
|
|
import {
|
|
assertSecretFree,
|
|
findSecretLeakInDirectory,
|
|
isEphemeralCodexRuntimeAuthFile,
|
|
isEphemeralPostgresPidFile,
|
|
normalizedSecrets,
|
|
sanitizeJson,
|
|
} from "./redaction.js";
|
|
import {
|
|
buildMatrixJobs,
|
|
parseRunnerSelectors,
|
|
RunnerSelectorError,
|
|
selectRunnerExecutions,
|
|
} from "./selectors.js";
|
|
import { resolveRunnerE2ESource } from "./source.js";
|
|
import {
|
|
CREDENTIAL_NAMES,
|
|
type MatrixExecution,
|
|
type RunnerE2EResult,
|
|
} from "./types.js";
|
|
import {
|
|
reapNewDetachedDarwinSharedMemory,
|
|
snapshotDarwinSharedMemory,
|
|
} from "./shared-memory.js";
|
|
import { reserveRunnerE2EServerPort } from "./ports.js";
|
|
import {
|
|
createResultExitGuard,
|
|
enforceResultProcessIntegrity,
|
|
} from "./result-exit-guard.js";
|
|
import {
|
|
observeDescendantProcessTree,
|
|
refreshContinuouslyLiveProcessGroups,
|
|
revalidateObservedProcessGroups,
|
|
safeProcessGroupTerminationOrder,
|
|
type ObservedProcessGroup,
|
|
type ProcessObservation,
|
|
} from "./process-tree.js";
|
|
|
|
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
|
|
const localEnvPath = path.join(repositoryRoot, ".env.runner-e2e.local");
|
|
const resultsRoot = path.join(repositoryRoot, "tests/runner-e2e/results");
|
|
const activeProcessGroups = new Set<number>();
|
|
const activeProcessCleanup = new Map<number, Promise<string | null>>();
|
|
const activeProcessTerminators = new Map<number, () => void>();
|
|
const completedResultExitGraceMs = 120_000;
|
|
const diagnosticProcessKinds = new Set([
|
|
"bash",
|
|
"chrome",
|
|
"codex",
|
|
"google-chrome",
|
|
"node",
|
|
"paperclip-runnerd",
|
|
"playwright",
|
|
"pnpm",
|
|
"postgres",
|
|
"sh",
|
|
"tsx",
|
|
]);
|
|
let cancelled = false;
|
|
|
|
function cleanId(value: string) {
|
|
const result = value
|
|
.replace(/[^A-Za-z0-9_.-]+/g, "-")
|
|
.replace(/^-+|-+$/g, "");
|
|
if (!result)
|
|
throw new Error(
|
|
`Invalid empty identifier derived from ${JSON.stringify(value)}`,
|
|
);
|
|
return result;
|
|
}
|
|
|
|
function secret(bytes = 32) {
|
|
return randomBytes(bytes).toString("base64url");
|
|
}
|
|
|
|
async function makeDirectoryTreeRemovable(directory: string): Promise<void> {
|
|
await chmod(directory, 0o700);
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
await Promise.all(
|
|
entries
|
|
.filter((entry) => entry.isDirectory() && !entry.isSymbolicLink())
|
|
.map((entry) =>
|
|
makeDirectoryTreeRemovable(path.join(directory, entry.name)),
|
|
),
|
|
);
|
|
}
|
|
|
|
function stopProcessGroup(pid: number, signal: NodeJS.Signals = "SIGTERM") {
|
|
try {
|
|
if (process.platform === "win32") process.kill(pid, signal);
|
|
else process.kill(-pid, signal);
|
|
} catch {
|
|
// The process tree already exited.
|
|
}
|
|
}
|
|
|
|
function processGroupIsAlive(pid: number) {
|
|
try {
|
|
process.kill(process.platform === "win32" ? pid : -pid, 0);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function terminateProcessGroup(pid: number) {
|
|
stopProcessGroup(pid, "SIGTERM");
|
|
const gracefulDeadline = Date.now() + 5_000;
|
|
while (processGroupIsAlive(pid) && Date.now() < gracefulDeadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
if (!processGroupIsAlive(pid)) return null;
|
|
stopProcessGroup(pid, "SIGKILL");
|
|
const forcedDeadline = Date.now() + 5_000;
|
|
while (processGroupIsAlive(pid) && Date.now() < forcedDeadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
return processGroupIsAlive(pid)
|
|
? `Paperclip/Playwright process group ${pid} survived SIGKILL`
|
|
: null;
|
|
}
|
|
|
|
function wait(milliseconds: number) {
|
|
return new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
|
}
|
|
|
|
interface ProcessTreeDiagnostic {
|
|
summary: string;
|
|
groups: ObservedProcessGroup[];
|
|
}
|
|
|
|
function observedGroupsSelectedForTermination(
|
|
groups: readonly ObservedProcessGroup[],
|
|
processGroupIds: readonly number[],
|
|
) {
|
|
const selected = new Set(processGroupIds);
|
|
return groups.filter((group) => selected.has(group.processGroupId));
|
|
}
|
|
|
|
async function readProcessTable(): Promise<ProcessObservation[] | null> {
|
|
if (process.platform === "win32") {
|
|
return null;
|
|
}
|
|
return await new Promise<ProcessObservation[] | null>((resolve) => {
|
|
const inspector = spawn(
|
|
"ps",
|
|
["-e", "-o", "pid=,ppid=,pgid=,lstart=,comm="],
|
|
{
|
|
env: { PATH: "/usr/bin:/bin", LANG: "C", LC_ALL: "C" },
|
|
stdio: ["ignore", "pipe", "ignore"],
|
|
},
|
|
);
|
|
let output = "";
|
|
let settled = false;
|
|
const finish = (value: ProcessObservation[] | null) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(inspectionTimeout);
|
|
resolve(value);
|
|
};
|
|
const inspectionTimeout = setTimeout(() => {
|
|
inspector.kill("SIGKILL");
|
|
finish(null);
|
|
}, 5_000);
|
|
inspectionTimeout.unref();
|
|
inspector.stdout?.setEncoding("utf8").on("data", (chunk: string) => {
|
|
output = `${output}${chunk}`.slice(-1024 * 1024);
|
|
});
|
|
inspector.once("error", () => finish(null));
|
|
inspector.once("close", () => {
|
|
const observations = output
|
|
.split(/\r?\n/)
|
|
.map((line) =>
|
|
/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+\s+\S+\s+\d+\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.+?)\s*$/.exec(
|
|
line,
|
|
),
|
|
)
|
|
.filter((match): match is RegExpExecArray => match !== null)
|
|
.map((match): ProcessObservation => {
|
|
// A target process can choose its own argv and process name. Emit a
|
|
// fixed category instead of target-controlled text so diagnostics
|
|
// can never turn that metadata into a secret-exfiltration channel.
|
|
const command = path.basename(match[5]!);
|
|
const kind = diagnosticProcessKinds.has(command) ? command : "other";
|
|
return {
|
|
pid: Number(match[1]),
|
|
parentPid: Number(match[2]),
|
|
processGroupId: Number(match[3]),
|
|
started: match[4]!,
|
|
kind,
|
|
};
|
|
});
|
|
finish(observations);
|
|
});
|
|
}).catch(() => null);
|
|
}
|
|
|
|
async function processTreeDiagnostic(
|
|
rootPid: number,
|
|
): Promise<ProcessTreeDiagnostic> {
|
|
const table = await readProcessTable();
|
|
if (!table) {
|
|
return {
|
|
summary: `process tree ${rootPid} (member inspection unavailable)`,
|
|
groups: [],
|
|
};
|
|
}
|
|
const observed = observeDescendantProcessTree(table, rootPid);
|
|
const members = observed.members
|
|
.slice(0, 64)
|
|
.map(
|
|
({ process: candidate, depth }) =>
|
|
`pid=${candidate.pid} ppid=${candidate.parentPid} pgid=${candidate.processGroupId} depth=${depth} kind=${candidate.kind}`,
|
|
);
|
|
const descendantGroupIds = observed.groups
|
|
.filter((group) => group.processGroupId !== rootPid)
|
|
.map((group) => group.processGroupId);
|
|
return {
|
|
summary:
|
|
members.length > 0
|
|
? `process tree ${rootPid}: ${members.join("; ")}; descendant pgids=${descendantGroupIds.join(",") || "none"}`
|
|
: `process tree ${rootPid}: no members reported`,
|
|
groups: observed.groups,
|
|
};
|
|
}
|
|
|
|
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
|
|
process.on(signal, () => {
|
|
cancelled = true;
|
|
for (const pid of activeProcessGroups) {
|
|
const terminate = activeProcessTerminators.get(pid);
|
|
if (terminate) {
|
|
terminate();
|
|
continue;
|
|
}
|
|
activeProcessCleanup.set(pid, terminateProcessGroup(pid));
|
|
}
|
|
});
|
|
}
|
|
|
|
async function loadLocalEnvironment(target: NodeJS.ProcessEnv) {
|
|
const contents = await readFile(localEnvPath, "utf8").catch(
|
|
(error: NodeJS.ErrnoException) => {
|
|
if (error.code === "ENOENT") return null;
|
|
throw error;
|
|
},
|
|
);
|
|
if (contents === null) return;
|
|
for (const [index, rawLine] of contents.split(/\r?\n/).entries()) {
|
|
const line = rawLine.trim();
|
|
if (!line || line.startsWith("#")) continue;
|
|
const match = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
if (!match)
|
|
throw new Error(
|
|
`Invalid ${path.basename(localEnvPath)} line ${index + 1}`,
|
|
);
|
|
if (target[match[1]] !== undefined) continue;
|
|
let value = match[2].trim();
|
|
if (
|
|
(value.startsWith('"') && value.endsWith('"')) ||
|
|
(value.startsWith("'") && value.endsWith("'"))
|
|
) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
target[match[1]] = value;
|
|
}
|
|
}
|
|
|
|
async function prepareProviderPath(
|
|
temporaryRoot: string,
|
|
inheritedPath: string | undefined,
|
|
) {
|
|
const toolBin = path.join(temporaryRoot, "provider-bin");
|
|
await mkdir(toolBin, { recursive: true });
|
|
const runnerRequire = createRequire(
|
|
path.join(repositoryRoot, "packages/paperclip-runner/package.json"),
|
|
);
|
|
const codexAcpPackage = runnerRequire.resolve(
|
|
"@agentclientprotocol/codex-acp/package.json",
|
|
);
|
|
const codexRequire = createRequire(codexAcpPackage);
|
|
const codexPackage = codexRequire.resolve("@openai/codex/package.json");
|
|
const codexManifest = JSON.parse(await readFile(codexPackage, "utf8")) as {
|
|
bin?: string | Record<string, string>;
|
|
};
|
|
const codexBin =
|
|
typeof codexManifest.bin === "string"
|
|
? codexManifest.bin
|
|
: codexManifest.bin?.codex;
|
|
if (!codexBin)
|
|
throw new Error(
|
|
"Production Codex ACP dependency does not expose its pinned Codex executable",
|
|
);
|
|
await symlink(
|
|
path.resolve(path.dirname(codexPackage), codexBin),
|
|
path.join(toolBin, "codex"),
|
|
);
|
|
const packageBin = path.join(
|
|
repositoryRoot,
|
|
"packages/paperclip-runner/node_modules/.bin",
|
|
);
|
|
return [toolBin, packageBin, inheritedPath]
|
|
.filter(Boolean)
|
|
.join(path.delimiter);
|
|
}
|
|
|
|
async function runProcess(
|
|
args: string[],
|
|
env: NodeJS.ProcessEnv,
|
|
timeoutMs: number | null,
|
|
logPath: string,
|
|
completionPaths: readonly string[],
|
|
interactive: boolean,
|
|
) {
|
|
const log = createWriteStream(logPath, { flags: "a", mode: 0o600 });
|
|
const child = spawn("pnpm", args, {
|
|
cwd: repositoryRoot,
|
|
env,
|
|
stdio: ["inherit", "pipe", "pipe"],
|
|
detached: process.platform !== "win32",
|
|
});
|
|
if (!child.pid) throw new Error("Failed to start Playwright");
|
|
activeProcessGroups.add(child.pid);
|
|
let outputTail = "";
|
|
const recordOutput = (chunk: Buffer, destination: NodeJS.WriteStream) => {
|
|
destination.write(chunk);
|
|
log.write(chunk);
|
|
outputTail += chunk.toString("utf8");
|
|
if (outputTail.length > 1024 * 1024)
|
|
outputTail = outputTail.slice(-1024 * 1024);
|
|
};
|
|
child.stdout?.on("data", (chunk: Buffer) =>
|
|
recordOutput(chunk, process.stdout),
|
|
);
|
|
child.stderr?.on("data", (chunk: Buffer) =>
|
|
recordOutput(chunk, process.stderr),
|
|
);
|
|
let childSettled = false;
|
|
let postResultStallError: string | null = null;
|
|
let boundedCleanup: Promise<string | null> | undefined;
|
|
const forceStopDirectChild = () => {
|
|
if (!childSettled && child.exitCode === null && child.signalCode === null) {
|
|
child.kill("SIGKILL");
|
|
}
|
|
};
|
|
const stopChildTree = (diagnostic?: ProcessTreeDiagnostic) => {
|
|
if (boundedCleanup) return;
|
|
const rootProcessGroupId = child.pid!;
|
|
boundedCleanup = (async () => {
|
|
const snapshot = diagnostic ?? (await processTreeDiagnostic(child.pid!));
|
|
const validationTable = await readProcessTable();
|
|
const currentProcessGroupId = validationTable
|
|
? (validationTable.find((candidate) => candidate.pid === process.pid)
|
|
?.processGroupId ?? null)
|
|
: null;
|
|
const observedGroups = validationTable
|
|
? revalidateObservedProcessGroups(snapshot.groups, validationTable)
|
|
: [];
|
|
const terminationOrder = safeProcessGroupTerminationOrder({
|
|
rootProcessGroupId,
|
|
currentProcessGroupId,
|
|
groups: observedGroups,
|
|
});
|
|
const verifiedGroups = observedGroupsSelectedForTermination(
|
|
observedGroups,
|
|
terminationOrder,
|
|
);
|
|
if (verifiedGroups.length === 0) {
|
|
forceStopDirectChild();
|
|
return "Could not revalidate an owned process group before cleanup";
|
|
}
|
|
for (const processGroupId of terminationOrder) {
|
|
stopProcessGroup(processGroupId, "SIGTERM");
|
|
}
|
|
|
|
let remainingGroups = verifiedGroups;
|
|
const gracefulDeadline = Date.now() + 5_000;
|
|
while (remainingGroups.length > 0 && Date.now() < gracefulDeadline) {
|
|
const table = await readProcessTable();
|
|
if (table) {
|
|
remainingGroups = refreshContinuouslyLiveProcessGroups(
|
|
remainingGroups,
|
|
table,
|
|
);
|
|
}
|
|
if (remainingGroups.length > 0) await wait(50);
|
|
}
|
|
const remainingGroupIds = new Set(
|
|
remainingGroups.map((group) => group.processGroupId),
|
|
);
|
|
const forcedOrder = safeProcessGroupTerminationOrder({
|
|
rootProcessGroupId,
|
|
currentProcessGroupId,
|
|
groups: remainingGroups,
|
|
}).filter((processGroupId) => remainingGroupIds.has(processGroupId));
|
|
for (const processGroupId of forcedOrder) {
|
|
stopProcessGroup(processGroupId, "SIGKILL");
|
|
}
|
|
|
|
const forcedDeadline = Date.now() + 5_000;
|
|
let forcedVerificationUnavailable = false;
|
|
while (Date.now() < forcedDeadline) {
|
|
const table = await readProcessTable();
|
|
if (!table) {
|
|
forcedVerificationUnavailable = true;
|
|
await wait(50);
|
|
continue;
|
|
}
|
|
forcedVerificationUnavailable = false;
|
|
remainingGroups = refreshContinuouslyLiveProcessGroups(
|
|
remainingGroups,
|
|
table,
|
|
);
|
|
if (remainingGroups.length === 0) return null;
|
|
await wait(50);
|
|
}
|
|
if (forcedVerificationUnavailable) {
|
|
return "Could not verify descendant process exit after SIGKILL";
|
|
}
|
|
return `Verified descendant process groups ${remainingGroups
|
|
.map((group) => group.processGroupId)
|
|
.join(",")} survived SIGKILL`;
|
|
})();
|
|
activeProcessCleanup.set(rootProcessGroupId, boundedCleanup);
|
|
};
|
|
activeProcessTerminators.set(child.pid, stopChildTree);
|
|
const resultExitGuard = createResultExitGuard({
|
|
resultPaths: completionPaths,
|
|
interactive,
|
|
graceMs: completedResultExitGraceMs,
|
|
now: Date.now,
|
|
pathExists: (candidate) =>
|
|
access(candidate).then(
|
|
() => true,
|
|
() => false,
|
|
),
|
|
onExpired: async () => {
|
|
const diagnostic = await processTreeDiagnostic(child.pid!);
|
|
if (childSettled) return;
|
|
postResultStallError = `Playwright remained alive for ${completedResultExitGraceMs}ms after every result was written`;
|
|
recordOutput(
|
|
Buffer.from(
|
|
`\n${postResultStallError}; forcing bounded cleanup. ${diagnostic.summary}\n`,
|
|
),
|
|
process.stderr,
|
|
);
|
|
stopChildTree(diagnostic);
|
|
},
|
|
});
|
|
const completionPoll = resultExitGuard.enabled
|
|
? setInterval(() => {
|
|
void resultExitGuard.poll().catch(() => {
|
|
if (childSettled || postResultStallError) return;
|
|
postResultStallError =
|
|
"Completed-result exit guard failed during bounded inspection";
|
|
recordOutput(
|
|
Buffer.from(`\n${postResultStallError}; forcing cleanup.\n`),
|
|
process.stderr,
|
|
);
|
|
stopChildTree();
|
|
});
|
|
}, 500)
|
|
: undefined;
|
|
completionPoll?.unref();
|
|
let timedOut = false;
|
|
const timer =
|
|
timeoutMs === null
|
|
? undefined
|
|
: setTimeout(() => {
|
|
timedOut = true;
|
|
stopChildTree();
|
|
}, timeoutMs);
|
|
timer?.unref();
|
|
let spawnError: string | null = null;
|
|
const exitCode = await new Promise<number>((resolve, reject) => {
|
|
child.once("error", reject);
|
|
child.once("exit", (code) => {
|
|
childSettled = true;
|
|
resolve(code ?? 1);
|
|
});
|
|
}).catch((error) => {
|
|
childSettled = true;
|
|
spawnError = error instanceof Error ? error.message : String(error);
|
|
return 1;
|
|
});
|
|
if (timer) clearTimeout(timer);
|
|
if (completionPoll) clearInterval(completionPoll);
|
|
const processCleanupError = child.pid
|
|
? await (boundedCleanup ??
|
|
activeProcessCleanup.get(child.pid) ??
|
|
terminateProcessGroup(child.pid))
|
|
: null;
|
|
if (child.pid) {
|
|
activeProcessGroups.delete(child.pid);
|
|
activeProcessCleanup.delete(child.pid);
|
|
activeProcessTerminators.delete(child.pid);
|
|
}
|
|
await new Promise<void>((resolve) => log.end(resolve));
|
|
return {
|
|
exitCode,
|
|
timedOut,
|
|
postResultStallError,
|
|
processCleanupError,
|
|
spawnError,
|
|
outputTail,
|
|
};
|
|
}
|
|
|
|
function syntheticResult(
|
|
execution: MatrixExecution,
|
|
attempt: number,
|
|
startedAtMs: number,
|
|
error: string,
|
|
failureClass: RunnerE2EResult["failureClass"] = "transient_infrastructure",
|
|
): RunnerE2EResult {
|
|
const finishedAtMs = Date.now();
|
|
return {
|
|
schema: "paperclip.runner-e2e.result/v2",
|
|
executionId: execution.id,
|
|
suiteId: execution.suite.id,
|
|
suiteDefinitionHash: execution.suiteDefinitionHash,
|
|
source: resolveRunnerE2ESource(),
|
|
...(execution.profile.ranking
|
|
? { rankingSnapshot: execution.profile.ranking }
|
|
: {}),
|
|
attempt,
|
|
status: "failed",
|
|
failureClass,
|
|
error,
|
|
profileId: execution.profile.id,
|
|
environmentId: execution.environment.id,
|
|
caseId: execution.task.id,
|
|
provider: execution.profile.provider,
|
|
model: execution.profile.model,
|
|
runtimeMode: execution.profile.expectedRuntimeMode,
|
|
startedAt: new Date(startedAtMs).toISOString(),
|
|
finishedAt: new Date(finishedAtMs).toISOString(),
|
|
durationMs: finishedAtMs - startedAtMs,
|
|
cleanup: "not_started",
|
|
};
|
|
}
|
|
|
|
async function readResult(resultPath: string, fallback: RunnerE2EResult) {
|
|
try {
|
|
return JSON.parse(await readFile(resultPath, "utf8")) as RunnerE2EResult;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
async function copySharedEvidence(privateRoot: string, casePrivateDir: string) {
|
|
for (const relative of [
|
|
"server.log",
|
|
"playwright.log",
|
|
"junit.xml",
|
|
"html-report",
|
|
"blob-report",
|
|
"playwright-output",
|
|
]) {
|
|
await cp(
|
|
path.join(privateRoot, relative),
|
|
path.join(casePrivateDir, relative),
|
|
{ recursive: true, force: true },
|
|
).catch((error: NodeJS.ErrnoException) => {
|
|
if (error.code !== "ENOENT") throw error;
|
|
});
|
|
}
|
|
}
|
|
|
|
async function runAttempt(input: {
|
|
executions: readonly MatrixExecution[];
|
|
attempt: number;
|
|
campaignId: string;
|
|
options: ReturnType<typeof parseRunnerSelectors>;
|
|
}): Promise<RunnerE2EResult[]> {
|
|
const { executions, attempt, campaignId, options } = input;
|
|
const execution = executions[0];
|
|
if (!execution) throw new Error("A runner E2E cell must contain a task case");
|
|
if (
|
|
executions.some(
|
|
(candidate) =>
|
|
candidate.profile.id !== execution.profile.id ||
|
|
candidate.environment.id !== execution.environment.id,
|
|
)
|
|
) {
|
|
throw new Error(
|
|
"One isolated runner E2E harness cannot mix profiles or environments",
|
|
);
|
|
}
|
|
const startedAtMs = Date.now();
|
|
const sharedMemoryBaseline = snapshotDarwinSharedMemory();
|
|
const temporaryRoot = await mkdtemp(
|
|
path.join(os.tmpdir(), "paperclip-runner-e2e-"),
|
|
);
|
|
const publishedResults: RunnerE2EResult[] = [];
|
|
const publishedResultPaths = new Map<string, string>();
|
|
let attemptSecrets: string[] = [];
|
|
try {
|
|
const paperclipHome = path.join(temporaryRoot, "paperclip-home");
|
|
const workspace = path.join(temporaryRoot, "workspace");
|
|
const privateDir = path.join(temporaryRoot, "artifacts-private");
|
|
const instanceId = `runner-e2e-${randomBytes(8).toString("hex")}`;
|
|
const configPath = path.join(
|
|
paperclipHome,
|
|
"instances",
|
|
instanceId,
|
|
"config.json",
|
|
);
|
|
const port = await reserveRunnerE2EServerPort();
|
|
await Promise.all([
|
|
mkdir(paperclipHome, { recursive: true }),
|
|
mkdir(workspace, { recursive: true }),
|
|
mkdir(privateDir, { recursive: true }),
|
|
]);
|
|
const providerPath = await prepareProviderPath(
|
|
temporaryRoot,
|
|
process.env.PATH,
|
|
);
|
|
if (execution.environment.id === "local" && execution.profile.id === "runner-codex") {
|
|
await prepareCodexCiSandbox(repositoryRoot, temporaryRoot);
|
|
}
|
|
const agentJwtSecret = secret(48);
|
|
const decisionSigningSecret = secret(48);
|
|
const toolActionSigningSecret = secret(48);
|
|
const betterAuthSecret = secret(48);
|
|
const credentials = normalizedSecrets([
|
|
...CREDENTIAL_NAMES.map((name) => process.env[name]),
|
|
agentJwtSecret,
|
|
decisionSigningSecret,
|
|
toolActionSigningSecret,
|
|
betterAuthSecret,
|
|
]);
|
|
attemptSecrets = credentials;
|
|
const runnerBinary = resolvePaperclipRunnerBinaryForHarness(
|
|
executions,
|
|
repositoryRoot,
|
|
);
|
|
const childEnv: NodeJS.ProcessEnv = {
|
|
...buildRunnerE2EProcessEnvironment(process.env, executions),
|
|
PATH: providerPath,
|
|
PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify(
|
|
executions.map((candidate) => candidate.id),
|
|
),
|
|
PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt),
|
|
PAPERCLIP_RUNNER_E2E_PORT: String(port),
|
|
PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot,
|
|
PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir,
|
|
PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace,
|
|
PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(privateDir, "server.log"),
|
|
PAPERCLIP_RUNNER_BINARY: runnerBinary,
|
|
PAPERCLIP_RUNNER_REMOTE_BINARY_PATH:
|
|
resolvePaperclipRemoteRunnerBinaryForHarness(executions, runnerBinary),
|
|
// Vite's optimized dependency cache embeds revision query strings. A
|
|
// private per-attempt cache prevents an earlier cell or local rebuild
|
|
// from producing `504 Outdated Optimize Dep` during browser bootstrap.
|
|
PAPERCLIP_VITE_CACHE_DIR: path.join(temporaryRoot, "vite-cache"),
|
|
PAPERCLIP_RUNNER_E2E_TEST_TIMEOUT_MS: String(
|
|
Math.max(
|
|
...executions.map(
|
|
(candidate) =>
|
|
candidate.task.attemptTimeoutMs[candidate.environment.id],
|
|
),
|
|
) + 90_000,
|
|
),
|
|
PAPERCLIP_HOME: paperclipHome,
|
|
PAPERCLIP_INSTANCE_ID: instanceId,
|
|
PAPERCLIP_CONFIG: configPath,
|
|
PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret,
|
|
PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret,
|
|
PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,
|
|
BETTER_AUTH_SECRET: betterAuthSecret,
|
|
};
|
|
// The database URLs are stripped here and again at the Playwright web-server
|
|
// boundary so a developer's shell can never redirect this paid test.
|
|
delete childEnv.DATABASE_URL;
|
|
delete childEnv.DATABASE_MIGRATION_URL;
|
|
|
|
const playwrightArgs = [
|
|
"exec",
|
|
"playwright",
|
|
"test",
|
|
"--config",
|
|
"tests/runner-e2e/playwright.config.ts",
|
|
...(options.headed ? ["--headed"] : []),
|
|
...(options.ui ? ["--ui"] : []),
|
|
...(options.debug ? ["--debug"] : []),
|
|
];
|
|
const watchdog =
|
|
options.ui || options.debug
|
|
? null
|
|
: executions.reduce(
|
|
(total, candidate) =>
|
|
total +
|
|
candidate.task.attemptTimeoutMs[candidate.environment.id] +
|
|
90_000,
|
|
0,
|
|
) +
|
|
5 * 60_000;
|
|
const processResult = await runProcess(
|
|
playwrightArgs,
|
|
childEnv,
|
|
watchdog,
|
|
path.join(privateDir, "playwright.log"),
|
|
executions.map((candidate) =>
|
|
path.join(privateDir, "cases", candidate.task.id, "result.json"),
|
|
),
|
|
options.ui || options.debug,
|
|
);
|
|
const processFailure = processResult.spawnError
|
|
? `Playwright failed to start: ${processResult.spawnError}`
|
|
: processResult.timedOut
|
|
? `Harness process exceeded ${Math.round((watchdog ?? 0) / 1000)} seconds`
|
|
: `Playwright exited ${processResult.exitCode} before writing a result`;
|
|
const processFailureClass =
|
|
processResult.spawnError || processResult.timedOut
|
|
? "transient_infrastructure"
|
|
: classifyFailure(
|
|
new Error(
|
|
processResult.outputTail
|
|
? `${processFailure}\n${processResult.outputTail}`
|
|
: processFailure,
|
|
),
|
|
);
|
|
const results = await Promise.all(
|
|
executions.map(async (candidate) => {
|
|
const resultPath = path.join(
|
|
privateDir,
|
|
"cases",
|
|
candidate.task.id,
|
|
"result.json",
|
|
);
|
|
const fallback = syntheticResult(
|
|
candidate,
|
|
attempt,
|
|
startedAtMs,
|
|
processFailure,
|
|
processFailureClass,
|
|
);
|
|
const result = await readResult(resultPath, fallback);
|
|
return enforceResultProcessIntegrity(result, processResult);
|
|
}),
|
|
);
|
|
let isolationError: unknown;
|
|
try {
|
|
await assertEmbeddedDatabaseIsolation(configPath, temporaryRoot);
|
|
} catch (error) {
|
|
isolationError = error;
|
|
}
|
|
let persistedStateError: unknown;
|
|
try {
|
|
const expectedEphemeralCredentials = new Set<string>();
|
|
for (const [label, directory] of [
|
|
["Paperclip home", paperclipHome],
|
|
["workspace", workspace],
|
|
] as const) {
|
|
while (true) {
|
|
// The managed Codex home may legitimately contain upstream source-code
|
|
// fixtures with fake `sk-*` strings. Reject exact campaign credentials.
|
|
const leak = await findSecretLeakInDirectory(directory, credentials, {
|
|
includeShapes: false,
|
|
ignoreFile: (file) => expectedEphemeralCredentials.has(file),
|
|
allowDisappearedFile: (file) =>
|
|
label === "Paperclip home" && isEphemeralPostgresPidFile(paperclipHome, file),
|
|
});
|
|
if (!leak) break;
|
|
const isManagedCodexRuntimeAuth =
|
|
label === "Paperclip home" &&
|
|
isEphemeralCodexRuntimeAuthFile(paperclipHome, leak.file);
|
|
if (isManagedCodexRuntimeAuth) {
|
|
const metadata = await lstat(leak.file);
|
|
if (metadata.isFile() && (metadata.mode & 0o777) === 0o600) {
|
|
// Codex CLI API-key mode requires this one runtime auth file. It
|
|
// lives only in the disposable cell root, is never published,
|
|
// must be owner-only, and is removed with the root below.
|
|
expectedEphemeralCredentials.add(leak.file);
|
|
continue;
|
|
}
|
|
}
|
|
throw new Error(
|
|
`Secret leak in persisted ${label} state at ${path.relative(temporaryRoot, leak.file)}: ${leak.reason}`,
|
|
);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
persistedStateError = error;
|
|
}
|
|
for (const [index, candidate] of executions.entries()) {
|
|
let result = results[index];
|
|
if (
|
|
isolationError &&
|
|
result.failureClass !== "cleanup_failure" &&
|
|
result.failureClass !== "secret_leak"
|
|
) {
|
|
const isolationMessage =
|
|
isolationError instanceof Error
|
|
? isolationError.message
|
|
: String(isolationError);
|
|
const configWasUnavailableDuringTransientBootstrap =
|
|
result.failureClass === "transient_infrastructure" &&
|
|
typeof isolationError === "object" &&
|
|
isolationError !== null &&
|
|
"code" in isolationError &&
|
|
isolationError.code === "ENOENT";
|
|
result = {
|
|
...result,
|
|
status: "failed",
|
|
failureClass: configWasUnavailableDuringTransientBootstrap
|
|
? result.failureClass
|
|
: "permanent_infrastructure",
|
|
error: configWasUnavailableDuringTransientBootstrap
|
|
? `${result.error}; isolated config could not be inspected after bootstrap failure: ${isolationMessage}`
|
|
: isolationMessage,
|
|
};
|
|
}
|
|
if (persistedStateError) {
|
|
const persistedStateMessage =
|
|
persistedStateError instanceof Error
|
|
? persistedStateError.message
|
|
: String(persistedStateError);
|
|
const persistedStateClass = classifyFailure(persistedStateError);
|
|
if (
|
|
persistedStateClass === "secret_leak" ||
|
|
(result.failureClass !== "secret_leak" &&
|
|
result.failureClass !== "cleanup_failure")
|
|
) {
|
|
result = {
|
|
...result,
|
|
status: "failed",
|
|
failureClass:
|
|
persistedStateClass === "secret_leak"
|
|
? "secret_leak"
|
|
: "permanent_infrastructure",
|
|
error: persistedStateMessage,
|
|
};
|
|
} else {
|
|
result = {
|
|
...result,
|
|
error: `${result.error}; persisted-state scan also failed: ${persistedStateMessage}`,
|
|
};
|
|
}
|
|
}
|
|
const casePrivateDir = path.join(privateDir, "cases", candidate.task.id);
|
|
const resultPath = path.join(casePrivateDir, "result.json");
|
|
const uploadDir = path.join(
|
|
resultsRoot,
|
|
campaignId,
|
|
candidate.suite.id,
|
|
candidate.profile.id,
|
|
candidate.environment.id,
|
|
candidate.task.id,
|
|
`attempt-${attempt}`,
|
|
);
|
|
await mkdir(casePrivateDir, { recursive: true });
|
|
await copySharedEvidence(privateDir, casePrivateDir);
|
|
await writeFile(
|
|
resultPath,
|
|
`${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
let evidence = await packageEvidence({
|
|
privateDir: casePrivateDir,
|
|
uploadDir,
|
|
secrets: credentials,
|
|
expectPassScreenshot: result.status === "passed",
|
|
});
|
|
if (evidence.leaks.length > 0 || evidence.missing.length > 0) {
|
|
result = {
|
|
...result,
|
|
status: "failed",
|
|
failureClass:
|
|
evidence.leaks.length > 0
|
|
? "secret_leak"
|
|
: "permanent_infrastructure",
|
|
error:
|
|
evidence.leaks.length > 0
|
|
? `Secret leak rejected from evidence: ${evidence.leaks.map((leak) => leak.file).join(", ")}`
|
|
: `Required evidence missing: ${evidence.missing.join(", ")}`,
|
|
};
|
|
await writeFile(
|
|
resultPath,
|
|
`${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
evidence = await packageEvidence({
|
|
privateDir: casePrivateDir,
|
|
uploadDir,
|
|
secrets: credentials,
|
|
expectPassScreenshot: false,
|
|
});
|
|
}
|
|
console.log(
|
|
`${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`,
|
|
);
|
|
publishedResults.push(result);
|
|
publishedResultPaths.set(
|
|
candidate.id,
|
|
path.join(uploadDir, "result.json"),
|
|
);
|
|
}
|
|
return [...publishedResults];
|
|
} finally {
|
|
reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline);
|
|
let cleanupError: unknown;
|
|
if (
|
|
temporaryRoot.startsWith(`${os.tmpdir()}${path.sep}paperclip-runner-e2e-`)
|
|
) {
|
|
for (let cleanupAttempt = 1; cleanupAttempt <= 3; cleanupAttempt += 1) {
|
|
try {
|
|
await rm(temporaryRoot, { recursive: true, force: true });
|
|
cleanupError = undefined;
|
|
break;
|
|
} catch (error) {
|
|
cleanupError = error;
|
|
if (cleanupAttempt < 3) {
|
|
await makeDirectoryTreeRemovable(temporaryRoot).catch(() => {});
|
|
await new Promise((resolve) =>
|
|
setTimeout(resolve, cleanupAttempt * 250),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
cleanupError = new Error(
|
|
`Refusing to remove unexpected temporary path ${temporaryRoot}`,
|
|
);
|
|
}
|
|
if (cleanupError) {
|
|
const message = `Temporary runner E2E state cleanup failed at ${temporaryRoot}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`;
|
|
for (const publishedResult of publishedResults) {
|
|
const publishedResultPath = publishedResultPaths.get(
|
|
publishedResult.executionId,
|
|
);
|
|
if (!publishedResultPath) continue;
|
|
Object.assign(publishedResult, {
|
|
status: "failed",
|
|
failureClass: "cleanup_failure",
|
|
error: message,
|
|
cleanup: "failed",
|
|
} satisfies Partial<RunnerE2EResult>);
|
|
const safeResult = `${JSON.stringify(
|
|
sanitizeJson(publishedResult, attemptSecrets),
|
|
null,
|
|
2,
|
|
)}\n`;
|
|
assertSecretFree(safeResult, attemptSecrets, publishedResultPath);
|
|
await writeFile(publishedResultPath, safeResult, "utf8");
|
|
}
|
|
// Cleanup failure is a failed cell, but must not suppress later cells in
|
|
// the same campaign. The result above carries the terminal failure.
|
|
console.error(message);
|
|
}
|
|
}
|
|
}
|
|
|
|
function printList(executions: readonly MatrixExecution[]) {
|
|
console.log("ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS");
|
|
for (const execution of executions) {
|
|
console.log(
|
|
[
|
|
execution.id,
|
|
execution.suite.id,
|
|
execution.profile.generation,
|
|
execution.profile.provider,
|
|
execution.profile.model,
|
|
execution.requiredCredentials.join(","),
|
|
].join("\t"),
|
|
);
|
|
}
|
|
}
|
|
|
|
async function runExecutionWithRetry(input: {
|
|
execution: MatrixExecution;
|
|
campaignId: string;
|
|
options: ReturnType<typeof parseRunnerSelectors>;
|
|
}): Promise<RunnerE2EResult> {
|
|
const { execution, campaignId, options } = input;
|
|
const [firstResult] = await runAttempt({
|
|
executions: [execution],
|
|
attempt: 1,
|
|
campaignId,
|
|
options,
|
|
});
|
|
if (!firstResult) throw new Error(`No result produced for ${execution.id}`);
|
|
if (
|
|
options.ui ||
|
|
options.debug ||
|
|
firstResult.status !== "failed" ||
|
|
!firstResult.failureClass ||
|
|
!shouldRetryFailure(firstResult.failureClass)
|
|
) {
|
|
return firstResult;
|
|
}
|
|
if (cancelled) throw new Error("Runner E2E campaign cancelled");
|
|
console.warn(
|
|
`Retrying ${execution.id} in a fresh isolated harness after ${firstResult.failureClass.replaceAll("_", " ")}`,
|
|
);
|
|
const [retryResult] = await runAttempt({
|
|
executions: [execution],
|
|
attempt: 2,
|
|
campaignId,
|
|
options,
|
|
});
|
|
if (!retryResult)
|
|
throw new Error(`No retry result produced for ${execution.id}`);
|
|
return retryResult;
|
|
}
|
|
|
|
async function runWithConcurrency<T, R>(
|
|
values: readonly T[],
|
|
concurrency: number,
|
|
worker: (value: T) => Promise<R>,
|
|
): Promise<R[]> {
|
|
const results = new Array<R>(values.length);
|
|
let cursor = 0;
|
|
const workers = Array.from(
|
|
{ length: Math.min(concurrency, values.length) },
|
|
async () => {
|
|
while (true) {
|
|
const index = cursor;
|
|
cursor += 1;
|
|
if (index >= values.length) return;
|
|
if (cancelled) throw new Error("Runner E2E campaign cancelled");
|
|
results[index] = await worker(values[index]!);
|
|
}
|
|
},
|
|
);
|
|
await Promise.all(workers);
|
|
return results;
|
|
}
|
|
|
|
async function main() {
|
|
let options: ReturnType<typeof parseRunnerSelectors>;
|
|
try {
|
|
options = parseRunnerSelectors(process.argv.slice(2));
|
|
} catch (error) {
|
|
if (error instanceof RunnerSelectorError)
|
|
throw new Error(`${error.message}\nUse --list to inspect valid cells.`);
|
|
throw error;
|
|
}
|
|
const executions = selectRunnerExecutions(options, runnerMatrix);
|
|
if (options.list) {
|
|
printList(executions);
|
|
return;
|
|
}
|
|
if (options.matrixJson) {
|
|
const jobs = buildMatrixJobs(executions);
|
|
console.log(
|
|
JSON.stringify({
|
|
include: jobs,
|
|
needsDaytona: jobs.some((job) => job.needsDaytona),
|
|
executionIds: executions.map((execution) => execution.id),
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
await loadLocalEnvironment(process.env);
|
|
const missingCredentials = [
|
|
...new Set(
|
|
executions.flatMap((execution) => execution.requiredCredentials),
|
|
),
|
|
].filter((name) => !process.env[name]?.trim());
|
|
if (missingCredentials.length > 0) {
|
|
throw new Error(
|
|
`Missing runner E2E credentials: ${missingCredentials.join(", ")}`,
|
|
);
|
|
}
|
|
if (
|
|
executions.some((execution) => execution.environment.id === "daytona") &&
|
|
!isImmutableDaytonaImage(process.env.PAPERCLIP_E2E_DAYTONA_IMAGE)
|
|
) {
|
|
throw new Error(
|
|
"PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image@sha256 digest for Daytona cells",
|
|
);
|
|
}
|
|
|
|
const campaignId = cleanId(
|
|
process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??
|
|
`local-${new Date().toISOString().replace(/[:.]/g, "-")}`,
|
|
);
|
|
const requestedParallelism =
|
|
options.headed || options.ui || options.debug ? 1 : options.maxParallel;
|
|
console.log(
|
|
`Running ${executions.length} isolated execution(s) with max parallelism ${requestedParallelism}`,
|
|
);
|
|
const finalResults = await runWithConcurrency(
|
|
executions,
|
|
requestedParallelism,
|
|
(execution) => runExecutionWithRetry({ execution, campaignId, options }),
|
|
);
|
|
|
|
const generatedAt = new Date().toISOString();
|
|
const campaign = buildRunnerCampaign({
|
|
campaignId,
|
|
generatedAt,
|
|
expected: executions.map((execution) => execution.id),
|
|
results: finalResults,
|
|
});
|
|
const summaryDir = path.join(resultsRoot, campaignId);
|
|
await mkdir(summaryDir, { recursive: true });
|
|
const campaignSecrets = normalizedSecrets(
|
|
CREDENTIAL_NAMES.map((name) => process.env[name]),
|
|
);
|
|
const safeCampaign = sanitizeJson(
|
|
campaign,
|
|
campaignSecrets,
|
|
) as typeof campaign;
|
|
const campaignText = `${JSON.stringify(safeCampaign, null, 2)}\n`;
|
|
assertSecretFree(campaignText, campaignSecrets, "campaign.json");
|
|
await writeFile(path.join(summaryDir, "campaign.json"), campaignText, "utf8");
|
|
const dashboard = renderRunnerE2EDashboard({
|
|
title: `Runner E2E · ${campaignId}`,
|
|
generatedAt,
|
|
expected: executions.map((execution) => execution.id),
|
|
catalog: runnerMatrix,
|
|
campaign: safeCampaign,
|
|
entries: safeCampaign.results.map((result) => ({
|
|
result,
|
|
valid: result.status === "passed" && result.cleanup === "passed",
|
|
errors:
|
|
result.status === "passed" && result.cleanup === "passed"
|
|
? []
|
|
: [
|
|
result.error ??
|
|
result.failureClass ??
|
|
`cleanup=${result.cleanup}`,
|
|
],
|
|
evidenceBaseHref: [
|
|
result.suiteId ?? "core-compatibility",
|
|
result.profileId,
|
|
result.environmentId,
|
|
result.caseId,
|
|
`attempt-${result.attempt}`,
|
|
].join("/"),
|
|
})),
|
|
});
|
|
assertSecretFree(dashboard, campaignSecrets, "dashboard.html");
|
|
await writeFile(path.join(summaryDir, "dashboard.html"), dashboard, "utf8");
|
|
console.log(
|
|
`Campaign ${campaignId}: ${safeCampaign.passed}/${safeCampaign.selected} passed`,
|
|
);
|
|
if (safeCampaign.failed > 0) process.exitCode = 1;
|
|
}
|
|
|
|
await main().catch((error) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exitCode = 1;
|
|
});
|