test: verify scoped files in warm runner qualification
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
parent
761ac12cc0
commit
246a8edbdd
|
|
@ -30,6 +30,13 @@ Credentials are not recorded in Playwright reports. These API checks supplement
|
|||
the required browser walkthrough, two real 180-second intervals, and recovery
|
||||
scenarios; passing them alone is not staging acceptance.
|
||||
|
||||
The separate three-turn Daytona runner qualification writes into
|
||||
`PAPERCLIP_TASK_DIR` when the host supplies scoped folders. After each turn it
|
||||
checks that run's final save and reads the exact bytes through the task file API.
|
||||
It still checks the same sandbox, provider session, and ordered turn markers.
|
||||
Runs without a scoped manifest retain the existing host-workspace assertions.
|
||||
The test does not treat a missing host mirror as proof that a scoped file was lost.
|
||||
|
||||
The staging matrix covers legacy Codex and Claude with both CLI and ACP,
|
||||
legacy OpenCode and Pi, and native Codex, OpenCode, and ACPX Claude/Codex/Pi.
|
||||
Cursor, Gemini, Grok, and Kimi are excluded from this acceptance campaign by
|
||||
|
|
|
|||
|
|
@ -104,6 +104,11 @@ describe("runner E2E catalog", () => {
|
|||
const initialPrompt = daytonaWarmContinuityTask.buildPrompt("nonce");
|
||||
const followups =
|
||||
daytonaWarmContinuityTask.buildFollowupMessages?.("nonce") ?? [];
|
||||
for (const prompt of [initialPrompt, ...followups]) {
|
||||
expect(prompt).toContain(
|
||||
'if [ -n "${PAPERCLIP_TASK_DIR:-}" ]; then cd "$PAPERCLIP_TASK_DIR"; fi',
|
||||
);
|
||||
}
|
||||
expect(initialPrompt).toContain('"kind":"request_confirmation"');
|
||||
expect(initialPrompt).toContain(
|
||||
'"reviewInteractionId":"<returned interaction id>"',
|
||||
|
|
@ -270,7 +275,9 @@ describe("runner E2E catalog", () => {
|
|||
question?.buildPrompt("nonce"),
|
||||
...breadthTasks,
|
||||
]) {
|
||||
const terminalTextInstruction = prompt?.match(/then emit (?:exactly|only)/)?.[0];
|
||||
const terminalTextInstruction = prompt?.match(
|
||||
/then emit (?:exactly|only)/,
|
||||
)?.[0];
|
||||
expect(terminalTextInstruction).toBeDefined();
|
||||
expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan(
|
||||
prompt!.indexOf(terminalTextInstruction!),
|
||||
|
|
|
|||
|
|
@ -787,7 +787,7 @@ function warmTurnInstructions(turn: 1 | 2 | 3, nonce: string) {
|
|||
? `In a legacy runner, make exactly one public-API completion write after verification: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"done","comment":"${marker}"}. Include Authorization and X-Paperclip-Run-Id. Do not POST a separate comment.`
|
||||
: `In a legacy runner, after verification POST exactly one request_confirmation to /api/issues/$PAPERCLIP_TASK_ID/interactions with {"kind":"request_confirmation","idempotencyKey":"daytona-warm-review-T${turn}-${nonce}","resolverPolicy":"human_only","title":"Warm continuity turn ${turn}","summary":"Review completed warm continuity turn ${turn}.","continuationPolicy":"wake_assignee","payload":{"version":1,"prompt":"Is this warm continuity task ready to complete after turn ${turn}?","acceptLabel":"Approve completion","rejectLabel":"Continue work","rejectRequiresReason":true,"allowDeclineReason":true,"supersedeOnUserComment":false,"target":{"type":"custom","key":"daytona_warm_turn_${turn}","revisionId":"${nonce}-T${turn}","label":"Warm continuity turn ${turn}"}}}. Capture the returned interaction id. Then make exactly one issue PATCH with {"status":"in_review","comment":"${marker}","reviewInteractionId":"<returned interaction id>"}. Include Authorization and X-Paperclip-Run-Id on both writes. If the issue PATCH fails, retry only that PATCH and never create another interaction. Do not POST a separate comment. After both writes succeed, end the response and heartbeat immediately; do not wait or poll because the reviewer action will start the next turn.`;
|
||||
return [
|
||||
`This is warm Daytona continuity turn ${turn} of 3. Work only in the current execution workspace.`,
|
||||
`This is warm Daytona continuity turn ${turn} of 3. Before reading or writing the file, run: if [ -n "\${PAPERCLIP_TASK_DIR:-}" ]; then cd "$PAPERCLIP_TASK_DIR"; fi. Use the scoped task folder when supplied; otherwise keep the current execution workspace. Do this on every turn.`,
|
||||
turn === 1
|
||||
? `Create ${file} with exactly this one line followed by a newline: ${lines[0]}`
|
||||
: `Before changing anything, read ${file} and verify its content is exactly ${lines.slice(0, -1).join("\\n")} followed by a newline. Then append exactly ${lines.at(-1)} followed by a newline.`,
|
||||
|
|
|
|||
|
|
@ -37,12 +37,13 @@ describe("runner E2E Daytona image contract", () => {
|
|||
expect(normalizedDockerfile).not.toContain(
|
||||
"COPY packages/paperclip-runner ./packages/paperclip-runner",
|
||||
);
|
||||
// Branch images need the full manifest graph for workspace patches. The
|
||||
// resolved lock is verified before the frozen provider dependency install.
|
||||
expect(dockerfile).toContain("COPY packages ./packages");
|
||||
expect(dockerfile).toContain(
|
||||
"COPY packages/paperclip-eval-kernel/src ./packages/paperclip-eval-kernel/src",
|
||||
);
|
||||
expect(dockerfile).toContain(
|
||||
"COPY packages/paperclip-runner/src ./packages/paperclip-runner/src",
|
||||
"pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile",
|
||||
);
|
||||
expect(dockerfile).toContain("sha256sum -c /tmp/provider-lock.sha256");
|
||||
expect(dockerfile).toContain(
|
||||
"/opt/paperclip-runner/provider-pack/provider-pack.json",
|
||||
);
|
||||
|
|
@ -137,20 +138,24 @@ describe("runner E2E Daytona image contract", () => {
|
|||
workflow.indexOf(`--format '{{json .Image}}'`),
|
||||
);
|
||||
const providerInstall = dockerfile.indexOf(
|
||||
"RUN pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...'",
|
||||
);
|
||||
const runnerSourceCopy = dockerfile.indexOf(
|
||||
"COPY packages/paperclip-runner/src ./packages/paperclip-runner/src",
|
||||
"pnpm install --frozen-lockfile --filter '@paperclipai/paperclip-runner...'",
|
||||
);
|
||||
const runnerSourceCopy = dockerfile.indexOf("COPY packages ./packages");
|
||||
const providerRevisionArg = dockerfile.indexOf(
|
||||
"ARG PAPERCLIP_RUNNER_SOURCE_REVISION",
|
||||
);
|
||||
const cliInstall = dockerfile.indexOf("RUN npm install -g");
|
||||
const cliInstall = dockerfile.indexOf("npm install -g");
|
||||
const finalMetadataArgs = dockerfile.lastIndexOf(
|
||||
"ARG PAPERCLIP_RUNNER_CONTENT_ID",
|
||||
);
|
||||
expect(providerInstall).toBeGreaterThan(0);
|
||||
expect(providerInstall).toBeLessThan(runnerSourceCopy);
|
||||
expect(runnerSourceCopy).toBeGreaterThan(0);
|
||||
expect(runnerSourceCopy).toBeLessThan(providerInstall);
|
||||
const lockVerification = dockerfile.indexOf(
|
||||
"sha256sum -c /tmp/provider-lock.sha256",
|
||||
);
|
||||
expect(lockVerification).toBeGreaterThan(runnerSourceCopy);
|
||||
expect(lockVerification).toBeLessThan(providerInstall);
|
||||
expect(providerInstall).toBeLessThan(providerRevisionArg);
|
||||
expect(cliInstall).toBeGreaterThan(0);
|
||||
expect(cliInstall).toBeLessThan(finalMetadataArgs);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {
|
|||
providerSessionContinuityFailures,
|
||||
} from "./run-observations.js";
|
||||
import { resolveRunnerE2ESource } from "./source.js";
|
||||
import { readWarmWorkspaceFile } from "./warm-workspace.js";
|
||||
import {
|
||||
isPublicRunnerScreenshotRoute,
|
||||
PUBLIC_RUNNER_SCREENSHOT_MARKER,
|
||||
|
|
@ -1338,10 +1339,6 @@ for (const execution of executions) {
|
|||
`Warm fixture ${execution.task.id} is missing its project or follow-up messages`,
|
||||
);
|
||||
}
|
||||
const workspaceFile = path.join(
|
||||
workspacePath,
|
||||
`daytona-warm-${nonce}.txt`,
|
||||
);
|
||||
const turnEvidence: Array<Record<string, unknown>> = [];
|
||||
for (const completedTurn of [1, 2] as const) {
|
||||
const turnDeadlineAt = Math.min(
|
||||
|
|
@ -1376,10 +1373,16 @@ for (const execution of executions) {
|
|||
{ length: completedTurn },
|
||||
(_, index) => `T${index + 1}-${nonce}`,
|
||||
).join("\n")}\n`;
|
||||
const hostContent = await readFile(workspaceFile, "utf8");
|
||||
if (hostContent !== expectedPrefix) {
|
||||
const fileObservation = await readWarmWorkspaceFile({
|
||||
api,
|
||||
run: sortRunsChronologically(waitingState.taskRuns).at(-1)!,
|
||||
issueId: issue.id,
|
||||
workspacePath,
|
||||
filename: `daytona-warm-${nonce}.txt`,
|
||||
});
|
||||
if (fileObservation.content !== expectedPrefix) {
|
||||
throw new Error(
|
||||
`Host workspace was not finalized after warm turn ${completedTurn}: expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(hostContent)}`,
|
||||
`Warm workspace was not finalized after turn ${completedTurn} (${fileObservation.source}): expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(fileObservation.content)}`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
|
|
@ -1475,7 +1478,7 @@ for (const execution of executions) {
|
|||
turn: completedTurn,
|
||||
issue: waitingState.currentIssue,
|
||||
run: chronologicalRuns.at(-1),
|
||||
hostContent,
|
||||
fileObservation,
|
||||
leases: completedLeases,
|
||||
});
|
||||
await page.goto(
|
||||
|
|
@ -1836,12 +1839,22 @@ for (const execution of executions) {
|
|||
)
|
||||
.map(async (matcher) => [
|
||||
matcher.path,
|
||||
await readFile(
|
||||
path.isAbsolute(matcher.path)
|
||||
? matcher.path
|
||||
: path.join(workspacePath, matcher.path),
|
||||
"utf8",
|
||||
).catch(() => undefined),
|
||||
execution.task.flow === "warm_three_turn"
|
||||
? (
|
||||
await readWarmWorkspaceFile({
|
||||
api,
|
||||
run: finalRun,
|
||||
issueId: issue!.id,
|
||||
workspacePath,
|
||||
filename: matcher.path,
|
||||
})
|
||||
).content
|
||||
: await readFile(
|
||||
path.isAbsolute(matcher.path)
|
||||
? matcher.path
|
||||
: path.join(workspacePath, matcher.path),
|
||||
"utf8",
|
||||
).catch(() => undefined),
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { readWarmWorkspaceFile } from "./warm-workspace.js";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const workspacePath = await mkdtemp(
|
||||
path.join(os.tmpdir(), "warm-workspace-test-"),
|
||||
);
|
||||
temporaryDirectories.push(workspacePath);
|
||||
const binding = {
|
||||
version: 1,
|
||||
runId: "run",
|
||||
companyId: "company",
|
||||
agentId: "agent",
|
||||
taskId: "task",
|
||||
folders: { task: "folder" },
|
||||
};
|
||||
const saved = {
|
||||
runId: "run",
|
||||
state: "saved",
|
||||
active: false,
|
||||
lastSavedAt: "2026-09-09T04:00:00Z",
|
||||
};
|
||||
const response = {
|
||||
ok: () => true,
|
||||
status: () => 200,
|
||||
text: async () => "T1-nonce\n",
|
||||
};
|
||||
const api = {
|
||||
get: vi.fn().mockResolvedValue([saved]),
|
||||
request: { get: vi.fn().mockResolvedValue(response) },
|
||||
};
|
||||
return {
|
||||
input: {
|
||||
api,
|
||||
run: {
|
||||
id: "run",
|
||||
companyId: "company",
|
||||
agentId: "agent",
|
||||
contextSnapshot: { paperclipWorkFolders: binding },
|
||||
},
|
||||
issueId: "task",
|
||||
workspacePath,
|
||||
filename: "daytona-warm-nonce.txt",
|
||||
},
|
||||
saved,
|
||||
};
|
||||
}
|
||||
|
||||
describe("warm workspace persistence observation", () => {
|
||||
it("reads the saved scoped file without requiring a mirrored host file", async () => {
|
||||
const { input } = await fixture();
|
||||
expect(await readWarmWorkspaceFile(input)).toEqual({
|
||||
source: "task-cache",
|
||||
content: "T1-nonce\n",
|
||||
});
|
||||
expect(input.api.get).toHaveBeenCalledWith(
|
||||
"/api/companies/company/work-folders/task/task/sync",
|
||||
);
|
||||
expect(input.api.request.get).toHaveBeenCalledWith(
|
||||
"/api/companies/company/work-folders/task/task/content?path=daytona-warm-nonce.txt",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not substitute a stale host copy for a missing cached file", async () => {
|
||||
const { input } = await fixture();
|
||||
await writeFile(
|
||||
path.join(input.workspacePath, input.filename),
|
||||
"T1-nonce\n",
|
||||
);
|
||||
input.api.request.get.mockResolvedValue({
|
||||
ok: () => false,
|
||||
status: () => 404,
|
||||
});
|
||||
await expect(readWarmWorkspaceFile(input)).rejects.toThrow(
|
||||
"download returned 404",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ runId: "older-run" },
|
||||
{ state: "failed" },
|
||||
{ state: "saving" },
|
||||
{ active: true },
|
||||
{ lastSavedAt: null },
|
||||
])(
|
||||
"rejects incomplete or unrelated checkpoint evidence: %j",
|
||||
async (override) => {
|
||||
const { input, saved } = await fixture();
|
||||
input.api.get.mockResolvedValue([{ ...saved, ...override }]);
|
||||
await expect(readWarmWorkspaceFile(input)).rejects.toThrow(
|
||||
"successful final file save",
|
||||
);
|
||||
expect(input.api.request.get).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["runId", "companyId", "agentId", "taskId"] as const)(
|
||||
"rejects a manifest with a different %s",
|
||||
async (field) => {
|
||||
const { input } = await fixture();
|
||||
input.run.contextSnapshot.paperclipWorkFolders[field] = "other";
|
||||
await expect(readWarmWorkspaceFile(input)).rejects.toThrow();
|
||||
expect(input.api.get).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the host workspace contract when the run has no scoped manifest", async () => {
|
||||
const { input } = await fixture();
|
||||
await writeFile(
|
||||
path.join(input.workspacePath, input.filename),
|
||||
"T1-local\n",
|
||||
);
|
||||
expect(
|
||||
await readWarmWorkspaceFile({
|
||||
...input,
|
||||
run: { ...input.run, contextSnapshot: null },
|
||||
}),
|
||||
).toEqual({ source: "host-workspace", content: "T1-local\n" });
|
||||
expect(input.api.get).not.toHaveBeenCalled();
|
||||
expect(input.api.request.get).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { WorkFolderSyncStatus } from "../../packages/shared/src/work-folders.js";
|
||||
import type { APIResponse } from "@playwright/test";
|
||||
import type { RunnerApi } from "./api.js";
|
||||
|
||||
/** Observe the persistence contract selected by the host for this exact run. */
|
||||
export async function readWarmWorkspaceFile(input: {
|
||||
api: Pick<RunnerApi, "get"> & {
|
||||
request: {
|
||||
get(path: string): Promise<Pick<APIResponse, "ok" | "status" | "text">>;
|
||||
};
|
||||
};
|
||||
run: {
|
||||
id: string;
|
||||
companyId: string;
|
||||
agentId: string;
|
||||
contextSnapshot?: Record<string, unknown> | null;
|
||||
};
|
||||
issueId: string;
|
||||
workspacePath: string;
|
||||
filename: string;
|
||||
}): Promise<{ source: "task-cache" | "host-workspace"; content: string }> {
|
||||
const { api, run, issueId, workspacePath, filename } = input;
|
||||
assert(
|
||||
/^[a-zA-Z0-9-]+\.txt$/.test(filename),
|
||||
"Invalid warm fixture filename",
|
||||
);
|
||||
const manifest = run.contextSnapshot?.paperclipWorkFolders;
|
||||
if (manifest === undefined || manifest === null) {
|
||||
return {
|
||||
source: "host-workspace",
|
||||
content: await readFile(path.join(workspacePath, filename), "utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
assert(typeof manifest === "object", "Invalid host work-folder manifest");
|
||||
const binding = manifest as Record<string, unknown>;
|
||||
assert.equal(binding.version, 1);
|
||||
assert.equal(binding.runId, run.id);
|
||||
assert.equal(binding.companyId, run.companyId);
|
||||
assert.equal(binding.agentId, run.agentId);
|
||||
assert.equal(binding.taskId, issueId);
|
||||
assert(binding.folders && typeof binding.folders === "object");
|
||||
assert(typeof (binding.folders as Record<string, unknown>).task === "string");
|
||||
|
||||
const base = `/api/companies/${encodeURIComponent(run.companyId)}/work-folders/task/${encodeURIComponent(issueId)}`;
|
||||
const statuses = await api.get<WorkFolderSyncStatus[]>(`${base}/sync`);
|
||||
const saved = statuses.find((status) => status.runId === run.id);
|
||||
assert(
|
||||
saved && saved.state === "saved" && !saved.active && saved.lastSavedAt,
|
||||
"The completed warm turn must have a successful final file save",
|
||||
);
|
||||
const response = await api.request.get(
|
||||
`${base}/content?path=${encodeURIComponent(filename)}`,
|
||||
);
|
||||
assert(
|
||||
response.ok(),
|
||||
`Warm task file download returned ${response.status()}`,
|
||||
);
|
||||
return { source: "task-cache", content: await response.text() };
|
||||
}
|
||||
Loading…
Reference in New Issue