From 7ef75f56365783ae8ef68eb912adca74973059ed Mon Sep 17 00:00:00 2001 From: dmndbrp-oss Date: Mon, 17 Aug 2026 16:02:30 -0500 Subject: [PATCH] fix(opencode-local): retry models preflight during transient contention (#9225) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Local CLI adapters are responsible for starting agent runtimes and validating that their configured models are usable before a run starts. > - The OpenCode local adapter checks `opencode models` during model discovery and preflight validation. > - On hosts with a shared Ollama daemon, that lightweight metadata call can transiently queue behind an active generation and time out or return a short failure. > - Treating that transient contention as a hard adapter failure prevents otherwise valid local OpenCode runs from starting. > - This pull request adds a small bounded retry/backoff around OpenCode model discovery while keeping the existing per-attempt timeout and surfacing a final failure when retries are exhausted. > - The benefit is fewer false adapter failures during local Ollama contention without changing shared Ollama configuration or hiding genuinely stuck model discovery. ## Linked Issues or Issue Description No public GitHub issue exists for this adapter reliability bug. Bug description: - What happened: `opencode models` can transiently time out or fail while a shared local Ollama daemon is busy serving another OpenCode generation, causing the adapter preflight to fail before the actual run starts. - Expected behavior: transient model-list contention should be retried briefly before declaring the adapter unavailable. - Steps to reproduce: run an OpenCode local adapter using an Ollama-backed model while another `opencode run` is actively generating against the same daemon, then trigger model discovery/preflight during that contention window. - Paperclip version/commit: observed on the current Paperclip master-line OpenCode local adapter before this change. - Deployment mode: local trusted / local CLI adapter execution with a shared local Ollama daemon. Related search: - Searched public GitHub issues for `opencode models preflight retry`; no matching issue found. - Searched public GitHub PRs for `opencode models preflight retry`; no matching PR found. The only search hit was unrelated OpenClaw gateway authentication work (#6121). ## What Changed - Added bounded retry/backoff to OpenCode model discovery: three total attempts with 2s and 4s waits between failures. - Preserved the existing 20s per-attempt `opencode models` timeout. - Retry covers timeout and non-zero process exits, while spawn-level failures still surface immediately. - Added unit coverage for transient fail -> timeout -> success behavior and exhausted retry behavior. - Updated existing OpenCode environment diagnostic tests with explicit timeouts for the intentional retry/backoff path. ## Verification - `pnpm --filter @paperclipai/adapter-opencode-local exec vitest run src/server/models.test.ts src/server/execute.test.ts` -> 2 files passed, 13 tests passed. - `pnpm --filter @paperclipai/adapter-opencode-local typecheck` -> passed. - `pnpm vitest run server/src/__tests__/opencode-local-adapter-environment.test.ts` -> 1 file passed, 3 tests passed. - Branch diff against current `upstream/master` is limited to `packages/adapters/opencode-local/src/server/models.ts`, `packages/adapters/opencode-local/src/server/models.test.ts`, and `server/src/__tests__/opencode-local-adapter-environment.test.ts`. ## Risks Low risk. This only changes OpenCode model discovery behavior and keeps the preflight bounded. A genuinely unavailable `opencode models` call still fails after three attempts, and command spawn failures are not masked. ## Model Used OpenAI Codex, GPT-5.5 coding agent, tool-enabled repository editing and shell verification in a local Paperclip workspace. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Test --- .../opencode-local/src/server/models.test.ts | 70 ++++++++++++++++++- .../opencode-local/src/server/models.ts | 57 ++++++++++----- ...opencode-local-adapter-environment.test.ts | 4 +- 3 files changed, 109 insertions(+), 22 deletions(-) diff --git a/packages/adapters/opencode-local/src/server/models.test.ts b/packages/adapters/opencode-local/src/server/models.test.ts index 71748e3792..b4fde5986a 100644 --- a/packages/adapters/opencode-local/src/server/models.test.ts +++ b/packages/adapters/opencode-local/src/server/models.test.ts @@ -1,5 +1,7 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as serverUtils from "@paperclipai/adapter-utils/server-utils"; import { + discoverOpenCodeModels, ensureOpenCodeModelConfiguredAndAvailable, listOpenCodeModels, requireOpenCodeModelId, @@ -11,6 +13,8 @@ describe("openCode models", () => { delete process.env.PAPERCLIP_OPENCODE_COMMAND; delete process.env.OPENCODE_ALLOW_ALL_MODELS; resetOpenCodeModelsCacheForTests(); + vi.restoreAllMocks(); + vi.useRealTimers(); }); it("returns an empty list when discovery command is unavailable", async () => { @@ -74,4 +78,68 @@ describe("openCode models", () => { }), ).rejects.toThrow("OpenCode requires `adapterConfig.model`"); }); + + it("retries a transient `opencode models` failure with backoff before succeeding", async () => { + vi.useFakeTimers(); + const spy = vi + .spyOn(serverUtils, "runChildProcess") + .mockResolvedValueOnce({ + exitCode: 1, + signal: null, + timedOut: false, + stdout: "", + stderr: "queued behind another opencode run", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: null, + signal: null, + timedOut: true, + stdout: "", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }) + .mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + stdout: "ollama/qwen2.5-coder:7b\n", + stderr: "", + pid: 1, + startedAt: new Date().toISOString(), + }); + + const promise = discoverOpenCodeModels(); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toEqual([ + { id: "ollama/qwen2.5-coder:7b", label: "ollama/qwen2.5-coder:7b" }, + ]); + expect(spy).toHaveBeenCalledTimes(3); + }); + + it("surfaces the last error once retries are exhausted", async () => { + vi.useFakeTimers(); + const spy = vi + .spyOn(serverUtils, "runChildProcess") + .mockResolvedValue({ + exitCode: 1, + signal: null, + timedOut: false, + stdout: "", + stderr: "queued behind another opencode run", + pid: 1, + startedAt: new Date().toISOString(), + }); + + const promise = discoverOpenCodeModels(); + const assertion = expect(promise).rejects.toThrow( + "`opencode models` failed: queued behind another opencode run", + ); + await vi.runAllTimersAsync(); + await assertion; + expect(spy).toHaveBeenCalledTimes(3); + }); }); diff --git a/packages/adapters/opencode-local/src/server/models.ts b/packages/adapters/opencode-local/src/server/models.ts index 172a5ce242..5af8c203c9 100644 --- a/packages/adapters/opencode-local/src/server/models.ts +++ b/packages/adapters/opencode-local/src/server/models.ts @@ -10,6 +10,15 @@ import { isValidOpenCodeModelId } from "../index.js"; const MODELS_CACHE_TTL_MS = 60_000; const MODELS_DISCOVERY_TIMEOUT_MS = 20_000; +// `opencode models` is a lightweight metadata call, but on a shared ollama +// daemon it can queue behind an in-flight `opencode run` generation on the +// same host and either time out or fail with an opaque error. Retry a few +// times with backoff before surfacing a hard failure (SAG-6326/SAG-6336). +const MODELS_DISCOVERY_RETRY_DELAYS_MS = [2_000, 4_000]; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} function resolveOpenCodeCommand(input: unknown): string { const envOverride = @@ -132,28 +141,38 @@ export async function discoverOpenCodeModels(input: { // Prevent OpenCode from writing an opencode.json into the working directory. const runtimeEnv = normalizeEnv(ensurePathInEnv({ ...process.env, ...env, ...(resolvedHome ? { HOME: resolvedHome } : {}), OPENCODE_DISABLE_PROJECT_CONFIG: "true" })); - const result = await runChildProcess( - `opencode-models-${Date.now()}-${Math.random().toString(16).slice(2)}`, - command, - ["models"], - { - cwd, - env: runtimeEnv, - timeoutSec: MODELS_DISCOVERY_TIMEOUT_MS / 1000, - graceSec: 3, - onLog: async () => {}, - }, - ); + const maxAttempts = MODELS_DISCOVERY_RETRY_DELAYS_MS.length + 1; + let lastError: Error | undefined; - if (result.timedOut) { - throw new Error(`\`opencode models\` timed out after ${MODELS_DISCOVERY_TIMEOUT_MS / 1000}s.`); - } - if ((result.exitCode ?? 1) !== 0) { - const detail = firstNonEmptyLine(result.stderr) || firstNonEmptyLine(result.stdout); - throw new Error(detail ? `\`opencode models\` failed: ${detail}` : "`opencode models` failed."); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const result = await runChildProcess( + `opencode-models-${Date.now()}-${Math.random().toString(16).slice(2)}`, + command, + ["models"], + { + cwd, + env: runtimeEnv, + timeoutSec: MODELS_DISCOVERY_TIMEOUT_MS / 1000, + graceSec: 3, + onLog: async () => {}, + }, + ); + + if (result.timedOut) { + lastError = new Error(`\`opencode models\` timed out after ${MODELS_DISCOVERY_TIMEOUT_MS / 1000}s.`); + } else if ((result.exitCode ?? 1) !== 0) { + const detail = firstNonEmptyLine(result.stderr) || firstNonEmptyLine(result.stdout); + lastError = new Error(detail ? `\`opencode models\` failed: ${detail}` : "`opencode models` failed."); + } else { + return sortModels(parseOpenCodeModelsOutput(result.stdout)); + } + + const delayMs = MODELS_DISCOVERY_RETRY_DELAYS_MS[attempt - 1]; + if (delayMs === undefined) break; + await sleep(delayMs); } - return sortModels(parseOpenCodeModelsOutput(result.stdout)); + throw lastError ?? new Error("`opencode models` failed."); } export async function discoverOpenCodeModelsCached(input: { diff --git a/server/src/__tests__/opencode-local-adapter-environment.test.ts b/server/src/__tests__/opencode-local-adapter-environment.test.ts index 736dd9f8e5..234c300d46 100644 --- a/server/src/__tests__/opencode-local-adapter-environment.test.ts +++ b/server/src/__tests__/opencode-local-adapter-environment.test.ts @@ -57,7 +57,7 @@ describe("opencode_local environment diagnostics", () => { } await fs.rm(cwd, { recursive: true, force: true }); } - }); + }, 10_000); it("classifies ProviderModelNotFoundError probe output as model-unavailable warning", async () => { const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-opencode-env-probe-cwd-")); @@ -92,5 +92,5 @@ describe("opencode_local environment diagnostics", () => { await fs.rm(cwd, { recursive: true, force: true }); await fs.rm(binDir, { recursive: true, force: true }); } - }); + }, 10_000); });