RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test(runner): harden breadth terminal responses

This commit is contained in:
Dotta 2026-09-03 08:07:42 -05:00
parent ed4a59eeeb
commit 23834d77fe
3 changed files with 37 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tests/runner-e2e/catalog.test.ts
View File

@ -118,12 +118,16 @@ describe("runner E2E catalog", () => {
const question = localIntegrityTasks.find(
(task) => task.id === "structured-question-resume",
);
const breadthTasks = openRouterBreadthTasks.map((task) =>
task.buildPrompt("nonce"),
);
for (const prompt of [
message?.buildPrompt("nonce"),
ask?.buildPrompt("nonce"),
plan?.buildPrompt("nonce"),
question?.buildPrompt("nonce"),
...breadthTasks,
]) {
expect(prompt).toContain("first emit");
expect(prompt!.indexOf("first emit")).toBeLessThan(
@ -133,6 +137,19 @@ describe("runner E2E catalog", () => {
"Do not wait for the terminal tool result before emitting the response",
);
}
for (const taskId of [
"question-resume-complete",
"plan-approve-complete",
]) {
const prompt = openRouterBreadthTasks
.find((task) => task.id === taskId)
?.buildPrompt("nonce");
expect(prompt).toContain(
"do not spell, quote, repeat, announce, or include",
);
expect(prompt).toContain("refer to it only as “the terminal marker.”");
}
});
it("uses only declared secret references in generated payloads", () => {

9
tests/runner-e2e/catalog.ts
View File

@ -635,8 +635,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [
buildPrompt: (nonce) =>
[
"Complete this deterministic hello task in one turn.",
`Return ${breadthMarker("H", nonce)} as the complete visible response.`,
`Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]} and mark the task Done.`,
`In the same provider turn first emit exactly ${breadthMarker("H", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`,
"Do not create files, plans, interactions, or additional work.",
].join("\n"),
buildMatchers: (nonce, execution) =>
@ -661,8 +660,9 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [
[
"Ask the user one structured question before completing this task.",
`Call request_human_input exactly once with idempotencyKey \`question-${nonce}\`, interactionKind \`questions\`, title \`Verification word\`, prompt \`Choose the verification word\`, continuationPolicy \`wake_assignee\`, and payload {version:1,questions:[{id:\`verification-word\`,prompt:\`Choose the verification word.\`,selectionMode:\`single\`,required:true,options:[{id:\`cobalt\`,label:\`Cobalt\`},{id:\`amber\`,label:\`Amber\`}]}]}.`,
`Before the answer arrives, do not spell, quote, repeat, announce, or include ${breadthMarker("Q_C", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`,
"Do not call paperclip_finish while the question is pending.",
`After the answer arrives, if it is Cobalt, return ${breadthMarker("Q_C", nonce)} visibly and call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("Q_C", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}.`,
`After the answer arrives, if it is Cobalt, in the same provider turn first emit exactly ${breadthMarker("Q_C", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("Q_C", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`,
"Do not create files, plans, or additional work.",
].join("\n"),
buildMatchers: (nonce, execution) =>
@ -688,8 +688,9 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [
"Create a canonical Plan with exactly two numbered steps and request approval; do not implement before approval.",
`The Plan body must contain ${breadthMarker("P_READY", nonce)}.`,
"Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`.",
`Before that exact Plan revision is accepted, do not spell, quote, repeat, announce, or include ${breadthMarker("P_OK", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`,
"Do not call paperclip_finish while confirmation is pending.",
`After that exact Plan revision is accepted, return ${breadthMarker("P_OK", nonce)} visibly and call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("P_OK", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}.`,
`After that exact Plan revision is accepted, in the same provider turn first emit exactly ${breadthMarker("P_OK", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("P_OK", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`,
"Do not create files, child tasks, or unrelated work.",
].join("\n"),
buildMatchers: (nonce, execution) =>

15
tests/runner-e2e/runner.spec.ts
View File

@ -1443,6 +1443,21 @@ for (const execution of executions) {
// fresh harness. A repeated near miss remains a failed cell.
failureClassOverride = "provider_variance";
}
if (
execution.suite.id === "openrouter-model-breadth" &&
execution.task.id === "question-resume-complete" &&
exactMessageMatcher?.kind === "message_exact" &&
selectedRuns.every((candidate) => candidate.status === "succeeded") &&
issue.status === "done" &&
finalRunMessage !== exactMessageMatcher.expected &&
record(finalRun.resultJson).summary === exactMessageMatcher.expected
) {
// A breadth model can occasionally satisfy the durable terminal
// contract while paraphrasing the visible final response. Preserve
// the exact persisted-message and DOM assertions, but classify this
// narrowly proven provider variance for one fresh-harness retry.
failureClassOverride = "provider_variance";
}
const observedEnvironmentId =
environmentContext.id ??
(execution.environment.id === "local"