diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 56d3cdeb37..7c4e5620d7 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -196,6 +196,16 @@ describe("runner E2E catalog", () => { ); expect(prompt).toContain("refer to it only as “the terminal marker.”"); } + + const breadthHello = openRouterBreadthTasks + .find((task) => task.id === "hello-complete") + ?.buildPrompt("nonce"); + expect(breadthHello).toContain( + "Your first response action must be the paperclip_finish tool call", + ); + expect(breadthHello).toContain( + "Do not emit any assistant text, acknowledgement, or preamble before calling it", + ); }); it("uses only declared secret references in generated payloads", () => { diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 1045336f33..28c34e99c8 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -651,6 +651,7 @@ export const openRouterBreadthTasks: readonly RunnerTaskFixture[] = [ buildPrompt: (nonce) => [ "Complete this deterministic hello task in one turn.", + "Your first response action must be the paperclip_finish tool call. Do not emit any assistant text, acknowledgement, or preamble before calling it.", `Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"${breadthMarker("H", nonce)}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly ${breadthMarker("H", nonce)} as the complete user-facing final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, "Do not create files, plans, interactions, or additional work.", ].join("\n"), diff --git a/tests/runner-e2e/run-observations.ts b/tests/runner-e2e/run-observations.ts index 90d4312f19..74938a3ac7 100644 --- a/tests/runner-e2e/run-observations.ts +++ b/tests/runner-e2e/run-observations.ts @@ -15,6 +15,29 @@ export interface ObservableProviderSessionRun { contextSnapshot?: Record | null; } +export interface ObservableMatcherResult { + matcher: { + kind?: string; + expected?: unknown; + count?: unknown; + }; + passed: boolean; +} + +export interface OpenRouterHelloTerminalVarianceObservation { + suiteId: string; + profileId: string; + taskId: string; + expectedMarker: string; + finalRunMessage: string; + allAgentMessages: string; + semanticSummary: unknown; + issueStatus: string; + runStatuses: readonly string[]; + matcherResults: readonly ObservableMatcherResult[]; + invariantFailures: readonly string[]; +} + export function acceptedPlanSessionResetFailures( provider: "codex" | "opencode" | "acpx", previousSessionId: string | null | undefined, @@ -59,6 +82,66 @@ export function isNonExecutingReviewFenceRun(run: ObservableRunState) { ); } +function normalizeMessage(value: string) { + return value + .replace(/\r\n/g, "\n") + .replace(/\\_/g, "_") + .replace(/[ \t]+/g, " ") + .trim(); +} + +function countOccurrences(value: string, expected: string) { + if (!expected) return 0; + let count = 0; + let cursor = 0; + while (cursor <= value.length - expected.length) { + const index = value.indexOf(expected, cursor); + if (index < 0) break; + count += 1; + cursor = index + expected.length; + } + return count; +} + +export function isOpenRouterDeepSeekHelloTerminalVariance( + observation: OpenRouterHelloTerminalVarianceObservation, +) { + const marker = normalizeMessage(observation.expectedMarker); + const finalRunMessage = normalizeMessage(observation.finalRunMessage); + const allAgentMessages = normalizeMessage(observation.allAgentMessages); + const failedMatchers = observation.matcherResults.filter( + (result) => !result.passed, + ); + const hasExpectedExactFailure = failedMatchers.some( + (result) => + result.matcher.kind === "message_exact" && + result.matcher.expected === observation.expectedMarker, + ); + const hasExpectedOccurrenceFailure = failedMatchers.some( + (result) => + result.matcher.kind === "message_occurrences" && + result.matcher.expected === observation.expectedMarker && + result.matcher.count === 1, + ); + + return ( + observation.suiteId === "openrouter-model-breadth" && + observation.profileId === "openrouter-deepseek-deepseek-v4-flash-0731" && + observation.taskId === "hello-complete" && + observation.issueStatus === "done" && + observation.runStatuses.length === 1 && + observation.runStatuses[0] === "succeeded" && + observation.semanticSummary === observation.expectedMarker && + finalRunMessage.length > 0 && + countOccurrences(finalRunMessage, marker) === 0 && + countOccurrences(allAgentMessages, marker) === 0 && + observation.invariantFailures.length === 0 && + failedMatchers.length === 2 && + hasExpectedExactFailure && + hasExpectedOccurrenceFailure + ); +} + export function isControlPlaneGovernedResponseWait( events: readonly ObservableRunEvent[], ) { diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 6386b864f7..d85ca19939 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -13,6 +13,7 @@ import { acceptedPlanSessionResetFailures, isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, + isOpenRouterDeepSeekHelloTerminalVariance, numberedPlanStepCount, providerSessionContinuityFailures, } from "./run-observations.js"; @@ -1646,6 +1647,29 @@ for (const execution of executions) { secrets, ); + if ( + exactMessageMatcher?.kind === "message_exact" && + isOpenRouterDeepSeekHelloTerminalVariance({ + suiteId: execution.suite.id, + profileId: execution.profile.id, + taskId: execution.task.id, + expectedMarker: exactMessageMatcher.expected, + finalRunMessage, + allAgentMessages: message, + semanticSummary: record(finalRun.resultJson).summary, + issueStatus: issue.status, + runStatuses: selectedRuns.map((candidate) => candidate.status), + matcherResults, + invariantFailures, + }) + ) { + // DeepSeek can occasionally complete the semantic finish correctly but + // expose its pre-tool acknowledgement as the visible final answer. Keep + // exact persisted-message, global occurrence, and DOM checks strict, + // while allowing one fresh-harness retry only for the zero-marker form. + failureClassOverride = "provider_variance"; + } + // The backend polling above can observe a terminal transition before a // websocket invalidation reaches the already-open task page. Reload the // canonical task route so the screenshot and UI assertions prove the diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index f1fe348abf..c40b6a605d 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -36,6 +36,7 @@ import { acceptedPlanSessionResetFailures, isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, + isOpenRouterDeepSeekHelloTerminalVariance, numberedPlanStepCount, providerSessionContinuityFailures, } from "./run-observations.js"; @@ -336,6 +337,71 @@ describe("runner E2E matchers", () => { }); describe("runner E2E run observations", () => { + it("retries only the zero-marker DeepSeek hello terminal emission variance", () => { + const expectedMarker = "PC_H_nonce-1"; + const observation = { + suiteId: "openrouter-model-breadth", + profileId: "openrouter-deepseek-deepseek-v4-flash-0731", + taskId: "hello-complete", + expectedMarker, + finalRunMessage: + "I'll complete this deterministic hello task by calling paperclip_finish once.", + allAgentMessages: + "I'll complete this deterministic hello task by calling paperclip_finish once.", + semanticSummary: expectedMarker, + issueStatus: "done", + runStatuses: ["succeeded"], + matcherResults: [ + { + matcher: { kind: "message_exact", expected: expectedMarker }, + passed: false, + }, + { + matcher: { + kind: "message_occurrences", + expected: expectedMarker, + count: 1, + }, + passed: false, + }, + { matcher: { kind: "issue_status", expected: "done" }, passed: true }, + ], + invariantFailures: [], + }; + + expect(isOpenRouterDeepSeekHelloTerminalVariance(observation)).toBe(true); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + allAgentMessages: `${expectedMarker}\n${expectedMarker}`, + }), + ).toBe(false); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + semanticSummary: "different-summary", + }), + ).toBe(false); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + invariantFailures: ["missing native terminal event"], + }), + ).toBe(false); + expect( + isOpenRouterDeepSeekHelloTerminalVariance({ + ...observation, + matcherResults: [ + ...observation.matcherResults, + { + matcher: { kind: "environment", expected: "local" }, + passed: false, + }, + ], + }), + ).toBe(false); + }); + it("counts provider-equivalent numbered Plan step formats", () => { expect(numberedPlanStepCount("1. First\n2) Second")).toBe(2); expect(