1319 lines
46 KiB
TypeScript
1319 lines
46 KiB
TypeScript
import { randomBytes } from "node:crypto";
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { expect, test, type Page } from "@playwright/test";
|
|
import { RunnerApi, pollUntil } from "./api.js";
|
|
import { buildRuntimeUsage, summarizeExecutionBilling } from "./billing.js";
|
|
import { runnerExecutionById } from "./catalog.js";
|
|
import { classifyFailure } from "./failure-classifier.js";
|
|
import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
|
|
import { evaluateMatchers, type MatcherResult } from "./matchers.js";
|
|
import {
|
|
isNonExecutingReviewFenceRun,
|
|
numberedPlanStepCount,
|
|
} from "./run-observations.js";
|
|
import {
|
|
assertSecretFree,
|
|
findSecretLeakInJsonValues,
|
|
normalizedSecrets,
|
|
sanitizeJson,
|
|
} from "./redaction.js";
|
|
import {
|
|
CREDENTIAL_NAMES,
|
|
type CredentialName,
|
|
type FailureClass,
|
|
type RunnerE2EResult,
|
|
} from "./types.js";
|
|
|
|
interface IssueRecord {
|
|
id: string;
|
|
identifier?: string | null;
|
|
companyId: string;
|
|
title: string;
|
|
status: string;
|
|
workMode?: string;
|
|
assigneeAgentId?: string | null;
|
|
executionRunId?: string | null;
|
|
checkoutRunId?: string | null;
|
|
}
|
|
|
|
interface CommentRecord {
|
|
id: string;
|
|
body?: string | null;
|
|
authorType?: string | null;
|
|
authorAgentId?: string | null;
|
|
createdByRunId?: string | null;
|
|
createdAt?: string;
|
|
}
|
|
|
|
interface RunRecord {
|
|
id: string;
|
|
companyId: string;
|
|
agentId: string;
|
|
status: string;
|
|
runtimeMode?: string;
|
|
continuationAttempt?: number;
|
|
retryOfRunId?: string | null;
|
|
runnerInstanceId?: string | null;
|
|
contextSnapshot?: Record<string, unknown> | null;
|
|
runnerProfileJson?: Record<string, unknown> | null;
|
|
usageJson?: Record<string, unknown> | null;
|
|
resultJson?: Record<string, unknown> | null;
|
|
error?: string | null;
|
|
errorCode?: string | null;
|
|
startedAt?: string | null;
|
|
finishedAt?: string | null;
|
|
}
|
|
|
|
interface EnvironmentLeaseRecord {
|
|
id: string;
|
|
issueId?: string | null;
|
|
heartbeatRunId?: string | null;
|
|
provider?: string | null;
|
|
acquiredAt?: string | null;
|
|
releasedAt?: string | null;
|
|
updatedAt?: string | null;
|
|
metadata?: Record<string, unknown> | null;
|
|
}
|
|
|
|
interface InteractionRecord {
|
|
id: string;
|
|
status: string;
|
|
kind?: string;
|
|
payload?: {
|
|
acceptLabel?: string;
|
|
rejectLabel?: string;
|
|
target?: {
|
|
type?: string;
|
|
key?: string;
|
|
revisionId?: string;
|
|
revisionNumber?: number;
|
|
};
|
|
questions?: Array<{
|
|
id: string;
|
|
prompt: string;
|
|
selectionMode: "single" | "multi";
|
|
required?: boolean;
|
|
options: Array<{ id: string; label: string }>;
|
|
}>;
|
|
};
|
|
}
|
|
interface IssueDocumentRecord {
|
|
id: string;
|
|
key: string;
|
|
body?: string | null;
|
|
latestRevisionId?: string | null;
|
|
latestRevisionNumber?: number;
|
|
}
|
|
interface RunEventRecord {
|
|
eventType?: string;
|
|
payload?: Record<string, unknown> | null;
|
|
sourceInstanceId?: string | null;
|
|
}
|
|
const TERMINAL_RUN_STATUSES = new Set([
|
|
"succeeded",
|
|
"interrupted",
|
|
"failed",
|
|
"cancelled",
|
|
"timed_out",
|
|
]);
|
|
const DEFINITIVE_FAILURE_RUN_STATUSES = new Set([
|
|
"failed",
|
|
"cancelled",
|
|
"timed_out",
|
|
]);
|
|
|
|
function definitiveRunFailure(runs: readonly RunRecord[]) {
|
|
const failed = runs.find((run) =>
|
|
DEFINITIVE_FAILURE_RUN_STATUSES.has(run.status),
|
|
);
|
|
if (!failed) return undefined;
|
|
return `heartbeat run ${failed.id} ended ${failed.status}${failed.errorCode ? ` (${failed.errorCode})` : ""}${failed.error ? `: ${failed.error}` : ""}`;
|
|
}
|
|
|
|
const executionIds = (() => {
|
|
const encoded = process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS;
|
|
if (encoded) {
|
|
const parsed = JSON.parse(encoded) as unknown;
|
|
if (
|
|
!Array.isArray(parsed) ||
|
|
parsed.length === 0 ||
|
|
parsed.some((value) => typeof value !== "string")
|
|
) {
|
|
throw new Error(
|
|
"PAPERCLIP_RUNNER_E2E_EXECUTION_IDS must be a non-empty JSON string array",
|
|
);
|
|
}
|
|
return parsed;
|
|
}
|
|
const single = process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_ID;
|
|
if (!single)
|
|
throw new Error("PAPERCLIP_RUNNER_E2E_EXECUTION_IDS is required");
|
|
return [single];
|
|
})();
|
|
const executions = executionIds.map(runnerExecutionById);
|
|
const attempt = Number(process.env.PAPERCLIP_RUNNER_E2E_ATTEMPT ?? "1");
|
|
const privateRoot = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR;
|
|
const workspacePath = process.env.PAPERCLIP_RUNNER_E2E_WORKSPACE;
|
|
if (!privateRoot || !workspacePath)
|
|
throw new Error("Runner E2E private/workspace paths are required");
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return value && typeof value === "object" && !Array.isArray(value)
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function credentialValues(): Partial<Record<CredentialName, string>> {
|
|
return Object.fromEntries(
|
|
CREDENTIAL_NAMES.flatMap((name) => {
|
|
const value = process.env[name]?.trim();
|
|
return value ? [[name, value]] : [];
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function writeSanitizedJson(
|
|
directory: string,
|
|
name: string,
|
|
value: unknown,
|
|
secrets: readonly string[],
|
|
) {
|
|
const raw = `${JSON.stringify(value)}\n`;
|
|
// Public API payloads may contain harmless provider-shaped fixtures or
|
|
// opaque generated tokens. Reject exact campaign credentials before writing,
|
|
// then sanitize all known shapes and assert the published JSON is clean.
|
|
assertSecretFree(raw, secrets, name, { includeShapes: false });
|
|
const safe = `${JSON.stringify(sanitizeJson(value, secrets), null, 2)}\n`;
|
|
const leak = findSecretLeakInJsonValues(JSON.parse(safe), secrets);
|
|
if (leak) throw new Error(`Secret leak in ${name}: ${leak}`);
|
|
await mkdir(directory, { recursive: true });
|
|
await writeFile(path.join(directory, name), safe, "utf8");
|
|
}
|
|
|
|
async function createTaskThroughUi(input: {
|
|
page: Page;
|
|
issuePrefix: string;
|
|
agentName: string;
|
|
title: string;
|
|
prompt: string;
|
|
workMode: "standard" | "planning" | "ask";
|
|
}) {
|
|
const issuesUrl = `/${encodeURIComponent(input.issuePrefix)}/issues`;
|
|
const newTask = input.page.getByRole("button", { name: "New Task" }).first();
|
|
let bootstrapError: unknown;
|
|
for (let bootstrapAttempt = 1; bootstrapAttempt <= 3; bootstrapAttempt += 1) {
|
|
try {
|
|
await input.page.goto(issuesUrl, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 30_000,
|
|
});
|
|
await newTask.waitFor({ state: "visible", timeout: 20_000 });
|
|
bootstrapError = undefined;
|
|
break;
|
|
} catch (error) {
|
|
bootstrapError = error;
|
|
if (bootstrapAttempt < 3) await input.page.waitForTimeout(1_000);
|
|
}
|
|
}
|
|
if (bootstrapError) {
|
|
throw new Error(
|
|
`Browser bootstrap failed before task creation: ${bootstrapError instanceof Error ? bootstrapError.message : String(bootstrapError)}`,
|
|
{ cause: bootstrapError },
|
|
);
|
|
}
|
|
await newTask.click();
|
|
await input.page.getByPlaceholder("Task title").fill(input.title);
|
|
await input.page
|
|
.getByRole("dialog")
|
|
.getByRole("textbox", { name: "editable markdown", exact: true })
|
|
.fill(input.prompt);
|
|
if (input.workMode !== "standard") {
|
|
await input.page
|
|
.getByRole("dialog")
|
|
.locator(`[data-issue-work-mode-chip="standard"]`)
|
|
.click();
|
|
await input.page
|
|
.locator(`[data-issue-work-mode="${input.workMode}"]`)
|
|
.click();
|
|
}
|
|
await input.page
|
|
.getByRole("button", { name: "Assignee", exact: true })
|
|
.click();
|
|
await input.page
|
|
.getByPlaceholder("Search assignees...")
|
|
.fill(input.agentName);
|
|
await input.page.getByText(input.agentName, { exact: true }).last().click();
|
|
await input.page
|
|
.getByRole("button", { name: "Create Task", exact: true })
|
|
.click();
|
|
}
|
|
|
|
function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
|
|
const explicit = new Set(
|
|
[issue.executionRunId, issue.checkoutRunId].filter(Boolean),
|
|
);
|
|
return runs.filter((run) => {
|
|
if (isNonExecutingReviewFenceRun(run)) return false;
|
|
const context = record(run.contextSnapshot);
|
|
return (
|
|
context.issueId === issue.id ||
|
|
context.taskId === issue.id ||
|
|
explicit.has(run.id)
|
|
);
|
|
});
|
|
}
|
|
|
|
function isPendingPlanConfirmation(interaction: InteractionRecord) {
|
|
return (
|
|
interaction.kind === "request_confirmation" &&
|
|
interaction.status === "pending" &&
|
|
interaction.payload?.target?.type === "issue_document" &&
|
|
interaction.payload.target.key === "plan" &&
|
|
typeof interaction.payload.target.revisionId === "string"
|
|
);
|
|
}
|
|
|
|
function isPendingQuestion(interaction: InteractionRecord) {
|
|
return (
|
|
interaction.kind === "ask_user_questions" &&
|
|
interaction.status === "pending" &&
|
|
Boolean(interaction.payload?.questions?.length)
|
|
);
|
|
}
|
|
|
|
function normalizePlanMarkdown(body: string | null | undefined) {
|
|
// Provider plan renderers may defensively escape underscores in plain-text
|
|
// markers. The rendered document and semantic marker are identical.
|
|
return (body ?? "").replaceAll("\\_", "_");
|
|
}
|
|
|
|
function renderedMarkerPattern(marker: string) {
|
|
return new RegExp(
|
|
marker
|
|
.split("_")
|
|
.map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
|
|
.join("\\\\?_"),
|
|
);
|
|
}
|
|
|
|
async function expectPlanStageMarkerVisible(page: Page, marker: string) {
|
|
const pattern = renderedMarkerPattern(marker);
|
|
// Depending on the interaction presentation, the task thread either
|
|
// expands the canonical Plan body inline or renders a compact `plan · vN`
|
|
// confirmation card while the agent's adjacent visible message describes
|
|
// the published revision. Backend assertions separately verify the exact
|
|
// canonical document body, revision ID, step count, and pending interaction.
|
|
await expect(
|
|
page
|
|
.getByTestId("task-chat-plan-preview")
|
|
.filter({ hasText: pattern })
|
|
.or(
|
|
page.getByTestId("task-chat-agent-bubble").filter({ hasText: pattern }),
|
|
)
|
|
.last(),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
}
|
|
|
|
for (const execution of executions) {
|
|
const privateDir = path.join(privateRoot, "cases", execution.task.id);
|
|
const resultPath = path.join(privateDir, "result.json");
|
|
const snapshotsDir = path.join(privateDir, "snapshots");
|
|
const deadlineMs = execution.task.attemptTimeoutMs[execution.environment.id];
|
|
|
|
test(`${execution.id} completes through the browser and public APIs`, async ({
|
|
page,
|
|
request,
|
|
}, testInfo) => {
|
|
test.setTimeout(deadlineMs + 90_000);
|
|
const startedAtMs = Date.now();
|
|
const startedAt = new Date(startedAtMs).toISOString();
|
|
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
|
|
const marker = execution.task.buildVisibleMarker(nonce);
|
|
const title = execution.task.buildTitle(nonce);
|
|
const prompt = execution.task.buildPrompt(nonce);
|
|
const credentials = credentialValues();
|
|
const secrets = normalizedSecrets(Object.values(credentials));
|
|
const api = new RunnerApi(request);
|
|
const consoleDiagnostics: Array<Record<string, unknown>> = [];
|
|
const networkDiagnostics: Array<Record<string, unknown>> = [];
|
|
let fixtures: LiveFixtureValues | undefined;
|
|
let issue: IssueRecord | undefined;
|
|
let selectedRuns: RunRecord[] = [];
|
|
let runtimeLeases: EnvironmentLeaseRecord[] = [];
|
|
let matcherResults: MatcherResult[] = [];
|
|
const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
|
|
let primaryError: unknown;
|
|
let failureClassOverride: FailureClass | undefined;
|
|
let cleanup: RunnerE2EResult["cleanup"] = "not_started";
|
|
|
|
const captureScreenshot = async (
|
|
id: string,
|
|
label: string,
|
|
file: string,
|
|
) => {
|
|
const screenshotPath = path.join(privateDir, file);
|
|
await page.screenshot({ path: screenshotPath, fullPage: true });
|
|
await testInfo.attach(id, {
|
|
path: screenshotPath,
|
|
contentType: "image/png",
|
|
});
|
|
screenshots.push({ id, label, file });
|
|
};
|
|
|
|
const captureRuntimeLeases = async () => {
|
|
if (!fixtures || execution.environment.id !== "daytona") return;
|
|
const listed = await api.get<EnvironmentLeaseRecord[]>(
|
|
`/api/environments/${fixtures.environment.id}/leases`,
|
|
);
|
|
const selectedRunIds = new Set(selectedRuns.map((run) => run.id));
|
|
const relevant = listed.filter(
|
|
(lease) =>
|
|
lease.provider === "daytona" &&
|
|
(selectedRunIds.size === 0 ||
|
|
(lease.heartbeatRunId &&
|
|
selectedRunIds.has(lease.heartbeatRunId)) ||
|
|
(issue && lease.issueId === issue.id)),
|
|
);
|
|
runtimeLeases = relevant.length > 0 ? relevant : listed;
|
|
};
|
|
|
|
const captureFailureApiState = async () => {
|
|
if (!fixtures || !issue) return;
|
|
const capture = async <T>(operation: () => Promise<T>) =>
|
|
operation().catch((error) => ({
|
|
evidenceCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
}));
|
|
const [currentIssue, listedRuns, comments, interactions] =
|
|
await Promise.all([
|
|
capture(() => api.get<IssueRecord>(`/api/issues/${issue!.id}`)),
|
|
capture(() =>
|
|
api.get<RunRecord[]>(
|
|
`/api/companies/${fixtures!.company.id}/heartbeat-runs?agentId=${fixtures!.agent.id}&limit=20`,
|
|
),
|
|
),
|
|
capture(() =>
|
|
api.get<CommentRecord[]>(
|
|
`/api/issues/${issue!.id}/comments?order=asc`,
|
|
),
|
|
),
|
|
capture(() =>
|
|
api.get<InteractionRecord[]>(
|
|
`/api/issues/${issue!.id}/interactions`,
|
|
),
|
|
),
|
|
]);
|
|
const taskRuns = Array.isArray(listedRuns)
|
|
? matchingRuns(listedRuns, "id" in currentIssue ? currentIssue : issue)
|
|
: [];
|
|
const detailedRuns = await Promise.all(
|
|
taskRuns.map((candidate) =>
|
|
capture(() =>
|
|
api.get<RunRecord>(`/api/heartbeat-runs/${candidate.id}`),
|
|
).then((value) => ("id" in value ? value : candidate)),
|
|
),
|
|
);
|
|
if (detailedRuns.length > 0) selectedRuns = detailedRuns;
|
|
const runEvidence = await Promise.all(
|
|
detailedRuns.map(async (candidate) => ({
|
|
runId: candidate.id,
|
|
log: await capture(() =>
|
|
api.get<unknown>(
|
|
`/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
|
|
),
|
|
),
|
|
events: await capture(() =>
|
|
api.get<RunEventRecord[]>(
|
|
`/api/heartbeat-runs/${candidate.id}/events?limit=1000`,
|
|
),
|
|
),
|
|
})),
|
|
);
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"api-state.json",
|
|
{
|
|
capturePhase: "failure",
|
|
issue: currentIssue,
|
|
runs: detailedRuns,
|
|
comments,
|
|
interactions,
|
|
runEvidence,
|
|
},
|
|
secrets,
|
|
);
|
|
};
|
|
|
|
page.on("console", (message) => {
|
|
if (["error", "warning"].includes(message.type())) {
|
|
consoleDiagnostics.push({
|
|
type: message.type(),
|
|
text: message.text(),
|
|
location: message.location(),
|
|
});
|
|
}
|
|
});
|
|
page.on("requestfailed", (requestEvent) => {
|
|
networkDiagnostics.push({
|
|
method: requestEvent.method(),
|
|
url: requestEvent.url(),
|
|
failure: requestEvent.failure()?.errorText ?? null,
|
|
});
|
|
});
|
|
page.on("response", (response) => {
|
|
if (response.status() >= 400) {
|
|
networkDiagnostics.push({
|
|
method: response.request().method(),
|
|
url: response.url(),
|
|
status: response.status(),
|
|
statusText: response.statusText(),
|
|
});
|
|
}
|
|
});
|
|
|
|
try {
|
|
const initialExperimental = await api.get<{
|
|
enableNativeRunner: boolean;
|
|
}>("/api/instance/settings/experimental");
|
|
expect(initialExperimental.enableNativeRunner).toBe(false);
|
|
await api.patch("/api/instance/settings/experimental", {
|
|
enableNativeRunner: true,
|
|
...(execution.profile.generation === "native" &&
|
|
execution.environment.id === "daytona"
|
|
? { enableRunnerPreviewIngress: true }
|
|
: {}),
|
|
});
|
|
|
|
fixtures = await setupLiveFixtures({
|
|
api,
|
|
execution,
|
|
executionNonce: nonce,
|
|
workspacePath,
|
|
credentials,
|
|
daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE,
|
|
});
|
|
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"fixtures.json",
|
|
{
|
|
executionId: execution.id,
|
|
companyId: fixtures.company.id,
|
|
environmentId: fixtures.environment.id,
|
|
agentId: fixtures.agent.id,
|
|
secretIds: Object.fromEntries(
|
|
Object.entries(fixtures.secretRefs).map(([name, ref]) => [
|
|
name,
|
|
ref?.secretId,
|
|
]),
|
|
),
|
|
persistedAgent: await api.get<unknown>(
|
|
`/api/agents/${fixtures.agent.id}`,
|
|
),
|
|
persistedEnvironment: await api.get<unknown>(
|
|
`/api/environments/${fixtures.environment.id}`,
|
|
),
|
|
},
|
|
secrets,
|
|
);
|
|
|
|
const issuePrefix = fixtures.company.issuePrefix;
|
|
if (!issuePrefix)
|
|
throw new Error(
|
|
"Created fixture company did not return an issue prefix",
|
|
);
|
|
await createTaskThroughUi({
|
|
page,
|
|
issuePrefix,
|
|
agentName: fixtures.agent.name,
|
|
title,
|
|
prompt,
|
|
workMode: execution.task.workMode,
|
|
});
|
|
|
|
const deadlineAt = startedAtMs + deadlineMs;
|
|
issue = await pollUntil({
|
|
label: `UI-created issue ${title}`,
|
|
deadlineAt,
|
|
load: async () => {
|
|
const issues = await api.get<IssueRecord[]>(
|
|
`/api/companies/${fixtures!.company.id}/issues?q=${encodeURIComponent(title)}&limit=50`,
|
|
);
|
|
return issues.find((candidate) => candidate.title === title);
|
|
},
|
|
accept: (candidate): candidate is IssueRecord => Boolean(candidate),
|
|
});
|
|
if (!issue) throw new Error(`Issue ${title} disappeared after creation`);
|
|
if (
|
|
issue.companyId !== fixtures.company.id ||
|
|
issue.assigneeAgentId !== fixtures.agent.id
|
|
) {
|
|
throw new Error(
|
|
"UI-created issue does not belong to the fixture company and agent",
|
|
);
|
|
}
|
|
if (issue.workMode !== execution.task.workMode) {
|
|
throw new Error(
|
|
`UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,
|
|
);
|
|
}
|
|
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
);
|
|
|
|
const loadTaskState = async () => {
|
|
const [currentIssue, runs, comments, interactions] = await Promise.all([
|
|
api.get<IssueRecord>(`/api/issues/${issue!.id}`),
|
|
api.get<RunRecord[]>(
|
|
`/api/companies/${fixtures!.company.id}/heartbeat-runs?agentId=${fixtures!.agent.id}&limit=20`,
|
|
),
|
|
api.get<CommentRecord[]>(
|
|
`/api/issues/${issue!.id}/comments?order=asc`,
|
|
),
|
|
api.get<InteractionRecord[]>(`/api/issues/${issue!.id}/interactions`),
|
|
]);
|
|
return {
|
|
currentIssue,
|
|
taskRuns: matchingRuns(runs, currentIssue),
|
|
comments,
|
|
interactions,
|
|
};
|
|
};
|
|
|
|
let planLifecycleEvidence: Record<string, unknown> | null = null;
|
|
let questionLifecycleEvidence: Record<string, unknown> | null = null;
|
|
if (execution.task.flow === "plan_revision_acceptance") {
|
|
const planMarkers = execution.task.buildPlanMarkers?.(nonce);
|
|
const revisionRequest = execution.task.buildRevisionRequest?.(nonce);
|
|
if (!planMarkers || !revisionRequest) {
|
|
throw new Error(
|
|
`Plan fixture ${execution.task.id} is missing lifecycle factories`,
|
|
);
|
|
}
|
|
const draftState = await pollUntil({
|
|
label: `initial plan confirmation for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 1 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(isPendingPlanConfirmation),
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
});
|
|
const draftInteraction = draftState.interactions.find(
|
|
isPendingPlanConfirmation,
|
|
)!;
|
|
const draftPlan = await api.get<IssueDocumentRecord>(
|
|
`/api/issues/${issue.id}/documents/plan`,
|
|
);
|
|
if (
|
|
!normalizePlanMarkdown(draftPlan.body).includes(planMarkers.draft)
|
|
) {
|
|
throw new Error(
|
|
`Initial Plan document did not contain ${planMarkers.draft}`,
|
|
);
|
|
}
|
|
if (numberedPlanStepCount(draftPlan.body) !== 2) {
|
|
throw new Error(
|
|
"Initial Plan must contain exactly two numbered steps",
|
|
);
|
|
}
|
|
if (
|
|
draftInteraction.payload?.target?.revisionId !==
|
|
draftPlan.latestRevisionId
|
|
) {
|
|
throw new Error(
|
|
"Initial plan confirmation did not target the latest Plan revision",
|
|
);
|
|
}
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expectPlanStageMarkerVisible(page, planMarkers.draft);
|
|
await captureScreenshot(
|
|
"plan-draft",
|
|
"Initial plan awaiting revision",
|
|
"plan-draft.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: draftInteraction.payload?.rejectLabel ?? "Reject",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
const revisionComposer = page
|
|
.getByTestId("plan-revision-composer")
|
|
.last();
|
|
await expect(revisionComposer).toBeVisible({ timeout: 10_000 });
|
|
await revisionComposer
|
|
.locator('[contenteditable="true"], textarea')
|
|
.first()
|
|
.fill(revisionRequest);
|
|
await page
|
|
.getByRole("button", {
|
|
name: draftInteraction.payload?.rejectLabel ?? "Reject",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
|
|
const revisedState = await pollUntil({
|
|
label: `revised plan confirmation for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 2 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(
|
|
(interaction) =>
|
|
isPendingPlanConfirmation(interaction) &&
|
|
interaction.id !== draftInteraction.id,
|
|
),
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
});
|
|
const revisedInteraction = revisedState.interactions.find(
|
|
(interaction) =>
|
|
isPendingPlanConfirmation(interaction) &&
|
|
interaction.id !== draftInteraction.id,
|
|
)!;
|
|
const revisedPlan = await api.get<IssueDocumentRecord>(
|
|
`/api/issues/${issue.id}/documents/plan`,
|
|
);
|
|
const normalizedRevisedPlan = normalizePlanMarkdown(revisedPlan.body);
|
|
if (
|
|
!normalizedRevisedPlan.includes(planMarkers.revised) ||
|
|
normalizedRevisedPlan.includes(planMarkers.draft)
|
|
) {
|
|
throw new Error(
|
|
`Revised Plan must replace ${planMarkers.draft} with ${planMarkers.revised}`,
|
|
);
|
|
}
|
|
if (numberedPlanStepCount(revisedPlan.body) !== 3) {
|
|
throw new Error(
|
|
"Revised Plan must contain exactly three numbered steps",
|
|
);
|
|
}
|
|
if (
|
|
revisedInteraction.payload?.target?.revisionId !==
|
|
revisedPlan.latestRevisionId ||
|
|
revisedPlan.latestRevisionId === draftPlan.latestRevisionId
|
|
) {
|
|
throw new Error(
|
|
"Revised confirmation did not target a new latest Plan revision",
|
|
);
|
|
}
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expectPlanStageMarkerVisible(page, planMarkers.revised);
|
|
await captureScreenshot(
|
|
"plan-revised",
|
|
"Revised plan awaiting acceptance",
|
|
"plan-revised.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: revisedInteraction.payload?.acceptLabel ?? "Approve",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
planLifecycleEvidence = {
|
|
draftInteraction,
|
|
draftPlan,
|
|
revisedInteraction,
|
|
revisedPlan,
|
|
revisionRequest,
|
|
};
|
|
} else if (execution.task.flow === "question_resume_completion") {
|
|
const expectedAnswer = execution.task.buildQuestionAnswer?.(nonce);
|
|
if (!expectedAnswer) {
|
|
throw new Error(
|
|
`Question fixture ${execution.task.id} is missing its answer factory`,
|
|
);
|
|
}
|
|
const pendingState = await pollUntil({
|
|
label: `pending user question for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 1 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(isPendingQuestion),
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
});
|
|
const questionInteraction =
|
|
pendingState.interactions.find(isPendingQuestion)!;
|
|
const labels =
|
|
questionInteraction.payload?.questions?.flatMap((question) =>
|
|
question.options.map((option) => option.label),
|
|
) ?? [];
|
|
if (!labels.includes(expectedAnswer.optionLabel)) {
|
|
throw new Error(
|
|
`Question interaction omitted ${expectedAnswer.optionLabel}`,
|
|
);
|
|
}
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expect(
|
|
page
|
|
.getByRole("button", {
|
|
name: expectedAnswer.optionLabel,
|
|
exact: true,
|
|
})
|
|
.last(),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
await captureScreenshot(
|
|
"question-pending",
|
|
"Structured question awaiting an answer",
|
|
"question-pending.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: expectedAnswer.optionLabel,
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
await page
|
|
.getByRole("button", { name: "Submit answers", exact: true })
|
|
.last()
|
|
.click();
|
|
questionLifecycleEvidence = {
|
|
interaction: questionInteraction,
|
|
answer: expectedAnswer.optionLabel,
|
|
expectedMarker: expectedAnswer.expectedMarker,
|
|
};
|
|
} else if (execution.task.flow === "plan_approval_completion") {
|
|
const planMarkers = execution.task.buildPlanMarkers?.(nonce);
|
|
if (!planMarkers) {
|
|
throw new Error(
|
|
`Plan fixture ${execution.task.id} is missing its marker factory`,
|
|
);
|
|
}
|
|
const pendingState = await pollUntil({
|
|
label: `pending plan approval for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 1 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(isPendingPlanConfirmation),
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
});
|
|
const interaction = pendingState.interactions.find(
|
|
isPendingPlanConfirmation,
|
|
)!;
|
|
const plan = await api.get<IssueDocumentRecord>(
|
|
`/api/issues/${issue.id}/documents/plan`,
|
|
);
|
|
if (!normalizePlanMarkdown(plan.body).includes(planMarkers.draft)) {
|
|
throw new Error(`Plan document did not contain ${planMarkers.draft}`);
|
|
}
|
|
if (numberedPlanStepCount(plan.body) !== 2) {
|
|
throw new Error("Plan must contain exactly two numbered steps");
|
|
}
|
|
if (interaction.payload?.target?.revisionId !== plan.latestRevisionId) {
|
|
throw new Error(
|
|
"Plan confirmation did not target the latest Plan revision",
|
|
);
|
|
}
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expectPlanStageMarkerVisible(page, planMarkers.draft);
|
|
await captureScreenshot(
|
|
"plan-pending",
|
|
"Plan awaiting approval",
|
|
"plan-pending.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: interaction.payload?.acceptLabel ?? "Approve",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
planLifecycleEvidence = { interaction, plan };
|
|
}
|
|
|
|
const terminal = await pollUntil({
|
|
label: `issue ${issue.id} and heartbeat run terminal state`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ currentIssue, taskRuns }) =>
|
|
currentIssue.status === execution.task.expectedTerminalState.issue &&
|
|
taskRuns.length >= execution.task.expectedRunCount &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)),
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
});
|
|
|
|
issue = terminal.currentIssue;
|
|
selectedRuns = terminal.taskRuns;
|
|
if (selectedRuns.length !== execution.task.expectedRunCount) {
|
|
const runLogs = await Promise.all(
|
|
selectedRuns.map(async (candidate) => ({
|
|
runId: candidate.id,
|
|
log: await api
|
|
.get<unknown>(
|
|
`/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
|
|
)
|
|
.catch((error) => ({
|
|
evidenceCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
})),
|
|
})),
|
|
);
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"api-state.json",
|
|
{ issue, runs: selectedRuns, runLogs, ...terminal },
|
|
secrets,
|
|
);
|
|
throw new Error(
|
|
`Expected exactly ${execution.task.expectedRunCount} task heartbeat run(s); observed ${selectedRuns.length}`,
|
|
);
|
|
}
|
|
// The company run-list endpoint intentionally returns only a compact,
|
|
// allowlisted context summary. Hydrate each selected run through the
|
|
// public detail endpoint before asserting environment/lease metadata.
|
|
selectedRuns = await Promise.all(
|
|
selectedRuns.map((candidate) =>
|
|
api.get<RunRecord>(`/api/heartbeat-runs/${candidate.id}`),
|
|
),
|
|
);
|
|
const run =
|
|
selectedRuns.find(
|
|
(candidate) => candidate.id === issue!.executionRunId,
|
|
) ?? selectedRuns[0];
|
|
const [persistedAgentValue, persistedEnvironmentValue, runLogs] =
|
|
await Promise.all([
|
|
api.get<unknown>(`/api/agents/${fixtures.agent.id}`),
|
|
api.get<unknown>(`/api/environments/${fixtures.environment.id}`),
|
|
Promise.all(
|
|
selectedRuns.map(async (candidate) => ({
|
|
runId: candidate.id,
|
|
log: await api
|
|
.get<unknown>(
|
|
`/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
|
|
)
|
|
.catch((error) => ({
|
|
evidenceCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
})),
|
|
})),
|
|
),
|
|
]);
|
|
const runLog =
|
|
runLogs.find((candidate) => candidate.runId === run.id)?.log ?? {};
|
|
const persistedAgent = record(persistedAgentValue);
|
|
const persistedEnvironment = record(persistedEnvironmentValue);
|
|
const agentComments = terminal.comments.filter(
|
|
(comment) =>
|
|
comment.authorAgentId === fixtures!.agent.id ||
|
|
selectedRuns.some(
|
|
(candidate) => comment.createdByRunId === candidate.id,
|
|
),
|
|
);
|
|
// Message matchers intentionally use persisted agent comments only.
|
|
// A semantic finish summary can differ from the actual user-facing text;
|
|
// accepting it here would let backend metadata mask a truncated UI
|
|
// response. The browser assertion below remains the visible source of
|
|
// truth after the comment projection has settled.
|
|
const message = agentComments
|
|
.map((comment) => comment.body ?? "")
|
|
.join("\n");
|
|
const pendingInteractions = terminal.interactions.filter(
|
|
(interaction) => interaction.status === "pending",
|
|
);
|
|
const invariantFailures: string[] = [];
|
|
if (pendingInteractions.length > 0)
|
|
invariantFailures.push(
|
|
`expected no unresolved interaction; observed ${pendingInteractions.length}`,
|
|
);
|
|
for (const candidate of selectedRuns) {
|
|
if (
|
|
(candidate.continuationAttempt ?? 0) !== 0 ||
|
|
candidate.retryOfRunId
|
|
)
|
|
invariantFailures.push(
|
|
`expected run ${candidate.id} without a recovery continuation`,
|
|
);
|
|
}
|
|
|
|
const context = record(run.contextSnapshot);
|
|
const environmentContext = record(context.paperclipEnvironment);
|
|
const workspaceContext = record(context.paperclipWorkspace);
|
|
const environmentDriver =
|
|
environmentContext.driver ?? persistedEnvironment.driver;
|
|
const observedEnvironment =
|
|
environmentDriver === "sandbox" ? "daytona" : environmentDriver;
|
|
const observedRuntimeMode =
|
|
run.runtimeMode ??
|
|
(persistedAgent.adapterType === "paperclip_runner"
|
|
? "native"
|
|
: "legacy");
|
|
matcherResults = await evaluateMatchers(
|
|
execution.task.buildMatchers(nonce, execution),
|
|
{
|
|
message,
|
|
issueStatus: issue.status,
|
|
runStatus: selectedRuns.every(
|
|
(candidate) =>
|
|
candidate.status === execution.task.expectedTerminalState.run,
|
|
)
|
|
? execution.task.expectedTerminalState.run
|
|
: selectedRuns.map((candidate) => candidate.status).join(","),
|
|
runtimeMode: observedRuntimeMode,
|
|
environment:
|
|
typeof observedEnvironment === "string"
|
|
? observedEnvironment
|
|
: undefined,
|
|
json: {
|
|
issue,
|
|
run,
|
|
comments: terminal.comments,
|
|
interactions: terminal.interactions,
|
|
},
|
|
},
|
|
);
|
|
const failedMatchers = matcherResults.filter((result) => !result.passed);
|
|
const observedEnvironmentId =
|
|
environmentContext.id ??
|
|
(execution.environment.id === "local"
|
|
? persistedAgent.defaultEnvironmentId
|
|
: undefined);
|
|
if (observedEnvironmentId !== fixtures.environment.id) {
|
|
invariantFailures.push(
|
|
`Expected environment ${fixtures.environment.id}; observed ${String(observedEnvironmentId)}`,
|
|
);
|
|
}
|
|
if (
|
|
execution.environment.id === "daytona" &&
|
|
typeof environmentContext.leaseId !== "string"
|
|
) {
|
|
invariantFailures.push(
|
|
"expected a Daytona sandbox lease on the run context",
|
|
);
|
|
}
|
|
if (execution.environment.id === "local") {
|
|
const runLogContent = String(record(runLog).content ?? "");
|
|
// The log endpoint returns NDJSON, so quotes inside each `chunk` are
|
|
// escaped. Accept both that wire representation and a decoded chunk.
|
|
const fallbackWorkspace =
|
|
/Using fallback workspace \\\"([^"\\]+)\\\"/.exec(
|
|
runLogContent,
|
|
)?.[1] ??
|
|
/Using fallback workspace "([^"]+)"/.exec(runLogContent)?.[1];
|
|
const cwd = String(workspaceContext.cwd ?? fallbackWorkspace ?? "");
|
|
const isolatedRoot = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "";
|
|
if (!isolatedRoot || !cwd.startsWith(`${isolatedRoot}/`)) {
|
|
invariantFailures.push(
|
|
`local run workspace escaped the isolated root: ${cwd}`,
|
|
);
|
|
}
|
|
}
|
|
let runEvents: RunEventRecord[] = [];
|
|
let runEventsCaptureError: string | null = null;
|
|
if (execution.profile.generation === "native") {
|
|
try {
|
|
runEvents = await api.get<RunEventRecord[]>(
|
|
`/api/heartbeat-runs/${run.id}/events?limit=1000`,
|
|
);
|
|
} catch (error) {
|
|
runEventsCaptureError =
|
|
error instanceof Error ? error.message : String(error);
|
|
invariantFailures.push(
|
|
`native run events query failed: ${runEventsCaptureError}`,
|
|
);
|
|
}
|
|
const runnerInstanceObserved =
|
|
Boolean(run.runnerInstanceId) ||
|
|
runEvents.some(
|
|
(event) =>
|
|
typeof event.sourceInstanceId === "string" &&
|
|
event.sourceInstanceId.length > 0 &&
|
|
!event.sourceInstanceId.endsWith(":control"),
|
|
);
|
|
if (!runnerInstanceObserved) {
|
|
invariantFailures.push(
|
|
"expected native run events from a runner instance",
|
|
);
|
|
}
|
|
const spanPayloads = runEvents
|
|
.filter((event) => event.eventType === "run.performance.span")
|
|
.map((event) => record(event.payload));
|
|
const selectedTransport = spanPayloads.find(
|
|
(payload) => payload.span === "runner.transport.selected",
|
|
);
|
|
const expectedTransport =
|
|
execution.environment.id === "daytona"
|
|
? "provider_ingress"
|
|
: "local_loopback";
|
|
if (selectedTransport?.mode !== expectedTransport) {
|
|
invariantFailures.push(
|
|
`Expected native runner transport ${expectedTransport}; observed ${String(selectedTransport?.mode)}`,
|
|
);
|
|
}
|
|
if (execution.environment.id === "daytona") {
|
|
const authenticated = spanPayloads.some(
|
|
(payload) =>
|
|
payload.span === "runner.prp.authenticate" &&
|
|
payload.outcome === "ok",
|
|
);
|
|
if (!authenticated) {
|
|
invariantFailures.push(
|
|
"expected authenticated native Daytona runner preview ingress",
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"api-state.json",
|
|
{
|
|
issue,
|
|
run,
|
|
comments: terminal.comments,
|
|
interactions: terminal.interactions,
|
|
planLifecycleEvidence,
|
|
questionLifecycleEvidence,
|
|
matcherResults,
|
|
invariantFailures,
|
|
runEvents,
|
|
runEventsCaptureError,
|
|
runLogs,
|
|
},
|
|
secrets,
|
|
);
|
|
|
|
// The backend polling above can observe a terminal transition before a
|
|
// websocket invalidation reaches the already-open task page. Reload the
|
|
// canonical task route so the screenshot and UI assertions prove the
|
|
// persisted final state, not a stale client cache.
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expect(
|
|
page
|
|
.getByTestId("task-chat-thread")
|
|
.getByTestId("task-chat-agent-bubble")
|
|
.filter({ hasText: renderedMarkerPattern(marker) })
|
|
.last(),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
await expect(
|
|
page.getByTestId("issue-detail-header").getByRole("button", {
|
|
name: "Change status (current: Done)",
|
|
exact: true,
|
|
}),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
await captureScreenshot(
|
|
"final-state",
|
|
"Final visible task state",
|
|
"final-state.png",
|
|
);
|
|
|
|
if (failedMatchers.length > 0) {
|
|
throw new Error(
|
|
`Matcher failure: ${failedMatchers.map((result) => result.detail).join("; ")}; run error: ${run.errorCode ?? "none"} ${run.error ?? "none"}`,
|
|
);
|
|
}
|
|
if (invariantFailures.length > 0) {
|
|
throw new Error(
|
|
`Runtime invariant failure: ${invariantFailures.join("; ")}`,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
primaryError = error;
|
|
try {
|
|
await captureFailureApiState();
|
|
} catch (captureError) {
|
|
const failureClass = classifyFailure(captureError);
|
|
if (failureClass === "secret_leak") {
|
|
failureClassOverride = "secret_leak";
|
|
primaryError = new AggregateError(
|
|
[primaryError, captureError],
|
|
`Failure evidence contained unsafe data: ${captureError instanceof Error ? captureError.message : String(captureError)}`,
|
|
);
|
|
} else {
|
|
networkDiagnostics.push({
|
|
evidenceCaptureError:
|
|
captureError instanceof Error
|
|
? captureError.message
|
|
: String(captureError),
|
|
});
|
|
}
|
|
}
|
|
if (!page.isClosed()) {
|
|
const failureScreenshot = path.join(privateDir, "failure.png");
|
|
await page
|
|
.screenshot({ path: failureScreenshot, fullPage: true })
|
|
.catch(() => undefined);
|
|
await testInfo
|
|
.attach("failure", {
|
|
path: failureScreenshot,
|
|
contentType: "image/png",
|
|
})
|
|
.catch(() => undefined);
|
|
}
|
|
} finally {
|
|
try {
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"browser-diagnostics.json",
|
|
{
|
|
console: consoleDiagnostics,
|
|
network: networkDiagnostics,
|
|
},
|
|
secrets,
|
|
);
|
|
} catch (error) {
|
|
primaryError = new AggregateError(
|
|
[primaryError, error].filter(Boolean),
|
|
`Browser diagnostics contained unsafe data: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
failureClassOverride = "secret_leak";
|
|
}
|
|
if (fixtures) {
|
|
await captureRuntimeLeases().catch((error) => {
|
|
networkDiagnostics.push({
|
|
runtimeBillingCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
});
|
|
});
|
|
try {
|
|
await fixtures.teardown();
|
|
cleanup = "passed";
|
|
} catch (error) {
|
|
cleanup = "failed";
|
|
const priorFailureClass = primaryError
|
|
? classifyFailure(primaryError)
|
|
: undefined;
|
|
const cleanupFailureClass = classifyFailure(error);
|
|
failureClassOverride =
|
|
priorFailureClass === "secret_leak"
|
|
? priorFailureClass
|
|
: cleanupFailureClass === "cleanup_failure"
|
|
? cleanupFailureClass
|
|
: (priorFailureClass ?? cleanupFailureClass);
|
|
primaryError = new AggregateError(
|
|
[primaryError, error].filter(Boolean),
|
|
`Cleanup failed after ${primaryError ? "test failure" : "test execution"}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
if (runtimeLeases.length > 0) {
|
|
runtimeLeases = await Promise.all(
|
|
runtimeLeases.map((lease) =>
|
|
api
|
|
.get<EnvironmentLeaseRecord>(
|
|
`/api/environment-leases/${lease.id}`,
|
|
)
|
|
.catch(() => lease),
|
|
),
|
|
);
|
|
}
|
|
} else {
|
|
cleanup =
|
|
primaryError &&
|
|
/(?:cleanup|teardown) failed/i.test(
|
|
primaryError instanceof Error
|
|
? primaryError.message
|
|
: String(primaryError),
|
|
)
|
|
? "failed"
|
|
: "passed";
|
|
}
|
|
|
|
const finishedAtMs = Date.now();
|
|
const runtimeUsage = buildRuntimeUsage({
|
|
environmentId: execution.environment.id,
|
|
runs: selectedRuns,
|
|
leases: runtimeLeases,
|
|
fallbackFinishedAt: new Date(finishedAtMs),
|
|
});
|
|
const resultWithoutBilling: RunnerE2EResult = {
|
|
schema: "paperclip.runner-e2e.result/v2",
|
|
executionId: execution.id,
|
|
suiteId: execution.suite.id,
|
|
suiteDefinitionHash: execution.suiteDefinitionHash,
|
|
source: {
|
|
sha: process.env.GITHUB_SHA ?? null,
|
|
ref: process.env.GITHUB_REF ?? null,
|
|
workflowRunUrl:
|
|
process.env.GITHUB_SERVER_URL &&
|
|
process.env.GITHUB_REPOSITORY &&
|
|
process.env.GITHUB_RUN_ID
|
|
? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
|
|
: null,
|
|
},
|
|
...(execution.profile.ranking
|
|
? { rankingSnapshot: execution.profile.ranking }
|
|
: {}),
|
|
attempt,
|
|
status: primaryError ? "failed" : "passed",
|
|
...(primaryError
|
|
? {
|
|
failureClass:
|
|
failureClassOverride ?? classifyFailure(primaryError),
|
|
error:
|
|
primaryError instanceof Error
|
|
? primaryError.message
|
|
: String(primaryError),
|
|
}
|
|
: {}),
|
|
profileId: execution.profile.id,
|
|
environmentId: execution.environment.id,
|
|
caseId: execution.task.id,
|
|
provider: execution.profile.provider,
|
|
model: execution.profile.model,
|
|
runtimeMode: execution.profile.expectedRuntimeMode,
|
|
issueId: issue?.id,
|
|
issueIdentifier: issue?.identifier ?? null,
|
|
runIds: selectedRuns.map((run) => run.id),
|
|
startedAt,
|
|
finishedAt: new Date(finishedAtMs).toISOString(),
|
|
durationMs: finishedAtMs - startedAtMs,
|
|
usage:
|
|
selectedRuns.length === 1
|
|
? (selectedRuns[0]?.usageJson ?? null)
|
|
: {
|
|
runs: selectedRuns.map((candidate) => ({
|
|
runId: candidate.id,
|
|
usage: candidate.usageJson ?? null,
|
|
})),
|
|
},
|
|
runtimeUsage,
|
|
matcherResults,
|
|
screenshots,
|
|
cleanup,
|
|
};
|
|
const result: RunnerE2EResult = {
|
|
...resultWithoutBilling,
|
|
billing: summarizeExecutionBilling(resultWithoutBilling),
|
|
};
|
|
await mkdir(privateDir, { recursive: true });
|
|
await writeFile(
|
|
resultPath,
|
|
`${JSON.stringify(sanitizeJson(result, secrets), null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
await testInfo.attach("runner-e2e-result", {
|
|
path: resultPath,
|
|
contentType: "application/json",
|
|
});
|
|
}
|
|
|
|
if (primaryError) throw primaryError;
|
|
});
|
|
}
|