From da1e4030221eab4a30c3176a6caeda3b5d1d97fa Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:02:55 -0500 Subject: [PATCH] test(runner): harden native OpenCode paid fixtures (#12833) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Paid runner E2E tests verify the full control-plane path for supported providers. > - Native OpenCode could write the reserved terminal marker through progress and final output. > - The restart fixture also waited for all development assets after the recovered UI was already usable. > - These behaviors made two valid local runner paths fail qualification. > - This pull request makes the OpenCode write contract explicit and uses the visible UI as the restart readiness gate. > - The benefit is reliable local OpenCode qualification without weaker duplicate detection. ## Linked Issues or Issue Description Refs #12769 Refs #12829 Refs #12828 **What happened?** The native OpenCode ask fixture allowed a progress tool call before the final response. OpenCode could write the reserved terminal marker in both places. The structured restart fixture could also time out while it waited for `DOMContentLoaded` after the recovered UI was visible and usable. **Expected behavior** The ask fixture must write the reserved marker once. The restart fixture must continue when the recovered UI and interaction API prove that the application is ready. **Steps to reproduce** 1. Run the local native OpenCode ask-question paid cell. 2. Observe a run that calls `report_progress`, calls `paperclip_finish`, and then emits the exact marker. 3. Run the local native OpenCode structured-question restart-resume cell with a fresh Vite graph. 4. Observe that the page is usable before the navigation lifecycle event completes. **Paperclip version or commit** The failures reproduced at `06cdf88bd9ac0fad82588025d23a68e810b20fd0`. The fixes are at `f7e044e71df11a0582eafe28d2fd52ea7cd07948`. **Deployment mode** Local dev. **Installation method** Built from source. **Agent adapter(s) involved** OpenCode through the native runner. ## What Changed - Require `paperclip_finish` to be the only tool call in the native ask fixture. - Forbid `report_progress` and other tool calls in that fixture. - Wait for navigation commit after a server restart. - Keep the explicit recovered UI and interaction API readiness checks. - Add prompt contract assertions. ## Verification - The exact two-cell paid run passed both affected cells on the AWS runner fleet: https://github.com/paperclipai/paperclip/actions/runs/33883334853 - Native OpenCode ask-question passed in job https://github.com/paperclipai/paperclip/actions/runs/33883334853/job/101058952662 - Native OpenCode structured restart-resume passed in job https://github.com/paperclipai/paperclip/actions/runs/33883334853/job/101058952823 - Prettier passed for all three changed files. - `git diff --check` passed. - The run-level aggregate failed only because the workflow source still used the pre-repair lockfile on `master`. PR #12828 repairs that lockfile. ## Risks Low risk. The prompt change affects native ask fixtures across provider profiles. The navigation change remains guarded by explicit UI and API assertions. ## Model Used OpenAI Codex, `gpt-5.6-sol`, extended reasoning, tool use, and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- tests/runner-e2e/catalog.test.ts | 6 ++++++ tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/runner.spec.ts | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 7c4e5620d7..dd87dd3494 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -206,6 +206,12 @@ describe("runner E2E catalog", () => { expect(breadthHello).toContain( "Do not emit any assistant text, acknowledgement, or preamble before calling it", ); + + const nativeAsk = ask?.buildPrompt("nonce"); + expect(nativeAsk).toContain("paperclip_finish must be your only tool call"); + expect(nativeAsk).toContain( + "never call report_progress or any other tool before or after it", + ); }); it("uses only declared secret references in generated payloads", () => { diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 28c34e99c8..4752beab55 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -512,7 +512,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ `Your final visible task-thread answer must be exactly this complete marker, including its final suffix: E2E_ASK_12_${nonce}.`, "Do not create or modify files, do not create a plan or additional work, and do not expose credentials.", "After posting the direct answer, mark this task Done.", - `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"E2E_ASK_12_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit exactly E2E_ASK_12_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, + `In a native runner, call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"E2E_ASK_12_${nonce}",completionClaim:{contractRevision:"1",objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. For the entire run, paperclip_finish must be your only tool call: never call report_progress or any other tool before or after it. Wait for that tool call to succeed, then emit exactly E2E_ASK_12_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds.`, `In a legacy runner, post exactly E2E_ASK_12_${nonce} as the task comment body and mark the task Done through the public API.`, ].join("\n"), buildMatchers(nonce, execution) { diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index d85ca19939..298108d0e0 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1026,7 +1026,11 @@ for (const execution of executions) { }); await page.goto( `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`, - { waitUntil: "domcontentloaded" }, + // A restarted Vite dev server may keep loading its fresh module + // graph after the task UI is already usable. Bind navigation only + // to the committed canonical route, then let the explicit UI and + // API assertions below prove readiness and preserved state. + { waitUntil: "commit" }, ); await expect( page