From 2311f084224b2b634db58980918c6fb79e1930c6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 3 Sep 2026 09:02:08 -0500 Subject: [PATCH] test(runner): align terminal fixture ordering --- tests/runner-e2e/catalog.test.ts | 12 +++++------- tests/runner-e2e/catalog.ts | 14 +++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6dbcb23090..4116f8ec1c 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -113,7 +113,7 @@ describe("runner E2E catalog", () => { expect(plan?.buildPrompt("nonce")).toContain("exactly two numbered steps"); }); - it("emits native terminal text before invoking the terminal tool", () => { + it("emits native terminal text after the terminal tool succeeds", () => { const message = runnerTasks.find((task) => task.id === "message-marker"); const ask = runnerTasks.find((task) => task.id === "ask-question"); const plan = runnerTasks.find((task) => task.id === "plan-revise-accept"); @@ -131,13 +131,11 @@ describe("runner E2E catalog", () => { question?.buildPrompt("nonce"), ...breadthTasks, ]) { - expect(prompt).toContain("first emit"); - expect(prompt!.indexOf("first emit")).toBeLessThan( - prompt!.indexOf("paperclip_finish exactly once"), - ); - expect(prompt).toContain( - "Do not wait for the terminal tool result before emitting the response", + expect(prompt).toContain("then emit exactly"); + expect(prompt!.indexOf("paperclip_finish exactly once")).toBeLessThan( + prompt!.indexOf("then emit exactly"), ); + expect(prompt).toContain("Wait for that tool call to succeed"); } for (const taskId of [ diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 492d4e2310..5e7039b895 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -390,7 +390,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ "Complete this task in a single run.", `The exact marker also appears unescaped in the task title: PAPERCLIP_E2E_OK_${nonce}`, `Your final visible task-thread response must be exactly this marker: PAPERCLIP_E2E_OK_${nonce}`, - `In a native runner, in the same provider turn first emit that exact marker once as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_OK_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, + `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_OK_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly PAPERCLIP_E2E_OK_${nonce} once as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `In a legacy runner, make exactly one public-API write containing the marker: PATCH /api/issues/$PAPERCLIP_TASK_ID with {"status":"done","comment":"PAPERCLIP_E2E_OK_${nonce}"}. Do not POST to /comments, and do not include the marker in any other write.`, "The visible task-thread response is asserted; hidden reasoning or provider terminal output alone does not count.", "Use underscore characters exactly as shown and do not insert backslashes.", @@ -457,7 +457,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ "Only after the revised plan is accepted, implement it by posting one final visible task-thread response containing exactly " + `PAPERCLIP_E2E_PLAN_DONE_${nonce}` + " and mark the task Done.", - `For a native runner, remain in the requested planning collaboration mode. Call write_document for key \`plan\`, then call request_human_input exactly once with interactionKind \`confirmation\`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy \`wake_assignee\`. Do not call paperclip_finish while waiting for either Plan confirmation. After the revised Plan is accepted, in the same provider turn first emit exactly PAPERCLIP_E2E_PLAN_DONE_${nonce} as the complete final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_PLAN_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, + `For a native runner, remain in the requested planning collaboration mode. Call write_document for key \`plan\`, then call request_human_input exactly once with interactionKind \`confirmation\`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy \`wake_assignee\`. Do not call paperclip_finish while waiting for either Plan confirmation. After the revised Plan is accepted, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_PLAN_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly PAPERCLIP_E2E_PLAN_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `For a legacy runner, use the public Paperclip API. The first PUT of the \`plan\` issue document creates it. For every later PUT, first GET the current document and set \`baseRevisionId\` to its \`latestRevisionId\`; a 409 means you must GET again and retry with the new latest revision. Create a \`request_confirmation\` targeting the successful PUT response's \`latestRevisionId\` with \`continuationPolicy: wake_assignee\`, and move the issue to \`in_review\` while waiting. After the revised Plan is accepted, write PAPERCLIP_E2E_PLAN_DONE_${nonce} exactly once through one atomic issue PATCH with status \`done\` and that exact comment; do not POST a separate comment or perform a second write.`, "Do not create files, child tasks, or unrelated work, and do not expose credentials.", ].join("\n"), @@ -508,7 +508,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ `Your final visible task-thread answer must be exactly this complete marker, including its final suffix: E2E_ASK_12_${nonce}.`, "Do not create or modify files, do not create a plan or additional work, and do not expose credentials.", "After posting the direct answer, mark this task Done.", - `In a native runner, in the same provider turn first emit exactly E2E_ASK_12_${nonce} as the complete final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"E2E_ASK_12_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, + `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"E2E_ASK_12_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly E2E_ASK_12_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `In a legacy runner, post exactly E2E_ASK_12_${nonce} as the task comment body and mark the task Done through the public API.`, ].join("\n"), buildMatchers(nonce, execution) { @@ -586,7 +586,7 @@ const structuredQuestionResumeTask = { `In a native runner, call request_human_input exactly once with idempotencyKey \`question-${nonce}\`, interactionKind \`questions\`, title \`Verification word\`, prompt \`Choose the verification word\`, continuationPolicy \`wake_assignee\`, and payload {version:1,questions:[{id:\`verification-word\`,prompt:\`Choose the verification word.\`,selectionMode:\`single\`,required:true,options:[{id:\`cobalt\`,label:\`Cobalt\`},{id:\`amber\`,label:\`Amber\`}]}]}.`, "In a legacy runner, POST /api/issues/$PAPERCLIP_TASK_ID/interactions with {kind:`ask_user_questions`,continuationPolicy:`wake_assignee`,payload:{version:1,questions:[{id:`verification-word`,prompt:`Choose the verification word.`,selectionMode:`single`,required:true,options:[{id:`cobalt`,label:`Cobalt`},{id:`amber`,label:`Amber`}]}]}} using Authorization and X-Paperclip-Run-Id, then move the issue to `in_review`.", `After the answer arrives, if it is Cobalt, publish exactly PAPERCLIP_E2E_QUESTION_DONE_${nonce} once as the complete visible response and mark the task Done.`, - `In a native runner, after the answer arrives, in the same provider turn first emit exactly PAPERCLIP_E2E_QUESTION_DONE_${nonce} as the complete final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_QUESTION_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, + `In a native runner, after the answer arrives, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_QUESTION_DONE_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly PAPERCLIP_E2E_QUESTION_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `In a legacy runner, post exactly PAPERCLIP_E2E_QUESTION_DONE_${nonce} as the task comment body and mark the task Done through the public API.`, "Do not create files, plans, child tasks, or unrelated work, and do not expose credentials.", ].join("\n"), @@ -644,7 +644,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ buildPrompt: (nonce) => [ "Complete this deterministic hello task in one turn.", - `In the same provider turn first emit exactly ${breadthMarker("H", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, + `Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${breadthMarker("H", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, "Do not create files, plans, interactions, or additional work.", ].join("\n"), buildMatchers: (nonce, execution) => @@ -671,7 +671,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ `Call request_human_input exactly once with idempotencyKey \`question-${nonce}\`, interactionKind \`questions\`, title \`Verification word\`, prompt \`Choose the verification word\`, continuationPolicy \`wake_assignee\`, and payload {version:1,questions:[{id:\`verification-word\`,prompt:\`Choose the verification word.\`,selectionMode:\`single\`,required:true,options:[{id:\`cobalt\`,label:\`Cobalt\`},{id:\`amber\`,label:\`Amber\`}]}]}.`, `Before the answer arrives, do not spell, quote, repeat, announce, or include ${breadthMarker("Q_C", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, "Do not call paperclip_finish while the question is pending.", - `After the answer arrives, if it is Cobalt, in the same provider turn first emit exactly ${breadthMarker("Q_C", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("Q_C", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, + `After the answer arrives, if it is Cobalt, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("Q_C", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${breadthMarker("Q_C", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, "Do not create files, plans, or additional work.", ].join("\n"), buildMatchers: (nonce, execution) => @@ -699,7 +699,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ "Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`.", `Before that exact Plan revision is accepted, do not spell, quote, repeat, announce, or include ${breadthMarker("P_OK", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, "Do not call paperclip_finish while confirmation is pending.", - `After that exact Plan revision is accepted, in the same provider turn first emit exactly ${breadthMarker("P_OK", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("P_OK", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, + `After that exact Plan revision is accepted, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("P_OK", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${breadthMarker("P_OK", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, "Do not create files, child tasks, or unrelated work.", ].join("\n"), buildMatchers: (nonce, execution) =>