From 23834d77fe28c8ed28cd917b445908bb56713e79 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 3 Sep 2026 08:07:42 -0500 Subject: [PATCH] test(runner): harden breadth terminal responses --- tests/runner-e2e/catalog.test.ts | 17 +++++++++++++++++ tests/runner-e2e/catalog.ts | 9 +++++---- tests/runner-e2e/runner.spec.ts | 15 +++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index cf032cc808..c81914b7c2 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -118,12 +118,16 @@ describe("runner E2E catalog", () => { const question = localIntegrityTasks.find( (task) => task.id === "structured-question-resume", ); + const breadthTasks = openRouterBreadthTasks.map((task) => + task.buildPrompt("nonce"), + ); for (const prompt of [ message?.buildPrompt("nonce"), ask?.buildPrompt("nonce"), plan?.buildPrompt("nonce"), question?.buildPrompt("nonce"), + ...breadthTasks, ]) { expect(prompt).toContain("first emit"); expect(prompt!.indexOf("first emit")).toBeLessThan( @@ -133,6 +137,19 @@ describe("runner E2E catalog", () => { "Do not wait for the terminal tool result before emitting the response", ); } + + for (const taskId of [ + "question-resume-complete", + "plan-approve-complete", + ]) { + const prompt = openRouterBreadthTasks + .find((task) => task.id === taskId) + ?.buildPrompt("nonce"); + expect(prompt).toContain( + "do not spell, quote, repeat, announce, or include", + ); + expect(prompt).toContain("refer to it only as “the terminal marker.”"); + } }); it("uses only declared secret references in generated payloads", () => { diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index cfbc40dae5..53c859c553 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -635,8 +635,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ buildPrompt: (nonce) => [ "Complete this deterministic hello task in one turn.", - `Return ${breadthMarker("H", nonce)} as the complete visible response.`, - `Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]} and mark the task Done.`, + `In the same provider turn first emit exactly ${breadthMarker("H", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, "Do not create files, plans, interactions, or additional work.", ].join("\n"), buildMatchers: (nonce, execution) => @@ -661,8 +660,9 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ [ "Ask the user one structured question before completing this task.", `Call request_human_input exactly once with idempotencyKey \`question-${nonce}\`, interactionKind \`questions\`, title \`Verification word\`, prompt \`Choose the verification word\`, continuationPolicy \`wake_assignee\`, and payload {version:1,questions:[{id:\`verification-word\`,prompt:\`Choose the verification word.\`,selectionMode:\`single\`,required:true,options:[{id:\`cobalt\`,label:\`Cobalt\`},{id:\`amber\`,label:\`Amber\`}]}]}.`, + `Before the answer arrives, do not spell, quote, repeat, announce, or include ${breadthMarker("Q_C", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, "Do not call paperclip_finish while the question is pending.", - `After the answer arrives, if it is Cobalt, return ${breadthMarker("Q_C", nonce)} visibly and call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("Q_C", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}.`, + `After the answer arrives, if it is Cobalt, in the same provider turn first emit exactly ${breadthMarker("Q_C", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("Q_C", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, "Do not create files, plans, or additional work.", ].join("\n"), buildMatchers: (nonce, execution) => @@ -688,8 +688,9 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ "Create a canonical Plan with exactly two numbered steps and request approval; do not implement before approval.", `The Plan body must contain ${breadthMarker("P_READY", nonce)}.`, "Call write_document for key `plan`, then call request_human_input exactly once with interactionKind `confirmation`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy `wake_assignee`.", + `Before that exact Plan revision is accepted, do not spell, quote, repeat, announce, or include ${breadthMarker("P_OK", nonce)} in any visible response, comment, or tool payload; refer to it only as “the terminal marker.”`, "Do not call paperclip_finish while confirmation is pending.", - `After that exact Plan revision is accepted, return ${breadthMarker("P_OK", nonce)} visibly and call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("P_OK", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}.`, + `After that exact Plan revision is accepted, in the same provider turn first emit exactly ${breadthMarker("P_OK", nonce)} as the complete user-facing final response, then call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("P_OK", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Do not wait for the terminal tool result before emitting the response, and do not call another tool.`, "Do not create files, child tasks, or unrelated work.", ].join("\n"), buildMatchers: (nonce, execution) => diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 64846ec0ba..2ac70bd877 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1443,6 +1443,21 @@ for (const execution of executions) { // fresh harness. A repeated near miss remains a failed cell. failureClassOverride = "provider_variance"; } + if ( + execution.suite.id === "openrouter-model-breadth" && + execution.task.id === "question-resume-complete" && + exactMessageMatcher?.kind === "message_exact" && + selectedRuns.every((candidate) => candidate.status === "succeeded") && + issue.status === "done" && + finalRunMessage !== exactMessageMatcher.expected && + record(finalRun.resultJson).summary === exactMessageMatcher.expected + ) { + // A breadth model can occasionally satisfy the durable terminal + // contract while paraphrasing the visible final response. Preserve + // the exact persisted-message and DOM assertions, but classify this + // narrowly proven provider variance for one fresh-harness retry. + failureClassOverride = "provider_variance"; + } const observedEnvironmentId = environmentContext.id ?? (execution.environment.id === "local"