diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsx index bbaa9cf67f..18bbaff7df 100644 --- a/ui/src/components/AgentConfigForm.tsx +++ b/ui/src/components/AgentConfigForm.tsx @@ -27,6 +27,7 @@ import { import { Button } from "@/components/ui/button"; import { FolderOpen, Heart, ChevronDown, X } from "lucide-react"; import { asBoolean, asFiniteNumber, asObject, cn } from "../lib/utils"; +import { resolveAdapterTestEnvironmentId } from "../lib/adapter-test-environment"; import { extractModelName, extractProviderId } from "../lib/model-utils"; import { queryKeys } from "../lib/queryKeys"; import { useCompany } from "../context/CompanyContext"; @@ -662,7 +663,37 @@ export function AgentConfigForm(props: AgentConfigFormProps) { const adapterConfigPatch = flushedEnv ? { env: flushedEnv } : undefined; const primaryModel = currentModelId.trim() || null; const cheapTestCase = getCheapModelTestCase(adapterConfigPatch); - const environmentId = currentDefaultEnvironmentId || null; + // Probe where a real run would actually execute: the agent's own + // environment, else the instance default. Testing the host for an + // agent that runs in the instance-default sandbox reports failures + // (e.g. a CLI that only exists in the sandbox image) a real run would + // never hit. The raw id is sent even for a local environment — the + // server resolves the driver and probes the host in that case. + // + // Test can be clicked before the settings query settles (or after it + // failed with retry:false), so when the agent relies on the instance + // default, resolve the settings here rather than trusting the + // render-time cache. A fetch that still fails FAILS the test with an + // honest diagnostic — silently probing the host instead would report + // the exact false command-not-found failure this resolution exists to + // fix. Agents with their own environment never need the settings. + let settings = instanceSettings; + if (!rawCurrentDefaultEnvironmentId && settings === undefined) { + try { + settings = await queryClient.ensureQueryData({ + queryKey: queryKeys.instance.settings, + queryFn: () => instanceSettingsApi.get(), + }); + } catch { + throw new Error( + "Could not load instance settings to determine which environment to test in. Retry the test.", + ); + } + } + const environmentId = resolveAdapterTestEnvironmentId({ + agentDefaultEnvironmentId: rawCurrentDefaultEnvironmentId || null, + instanceDefaultEnvironmentId: settings?.defaultEnvironmentId ?? null, + }); const testResults: Array<{ label: string; model: string | null; result: AdapterEnvironmentTestResult }> = [ { label: "Primary model", diff --git a/ui/src/lib/adapter-test-environment.test.ts b/ui/src/lib/adapter-test-environment.test.ts new file mode 100644 index 0000000000..856d085ced --- /dev/null +++ b/ui/src/lib/adapter-test-environment.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; + +import { resolveAdapterTestEnvironmentId } from "./adapter-test-environment"; + +describe("resolveAdapterTestEnvironmentId", () => { + it("prefers the agent's own environment", () => { + expect( + resolveAdapterTestEnvironmentId({ + agentDefaultEnvironmentId: "agent-env", + instanceDefaultEnvironmentId: "instance-env", + }), + ).toBe("agent-env"); + }); + + it("falls back to the instance default when the agent has none", () => { + // The regression this pins: an agent relying on the instance default + // (e.g. a managed sandbox with extra CLIs baked into its image) must be + // tested inside that environment, not on the Paperclip host where the + // CLI does not exist. + expect( + resolveAdapterTestEnvironmentId({ + agentDefaultEnvironmentId: null, + instanceDefaultEnvironmentId: "instance-env", + }), + ).toBe("instance-env"); + expect( + resolveAdapterTestEnvironmentId({ + agentDefaultEnvironmentId: "", + instanceDefaultEnvironmentId: "instance-env", + }), + ).toBe("instance-env"); + }); + + it("returns null (host probe) when neither is set", () => { + expect( + resolveAdapterTestEnvironmentId({ + agentDefaultEnvironmentId: undefined, + instanceDefaultEnvironmentId: undefined, + }), + ).toBeNull(); + expect( + resolveAdapterTestEnvironmentId({ + agentDefaultEnvironmentId: "", + instanceDefaultEnvironmentId: null, + }), + ).toBeNull(); + }); +}); diff --git a/ui/src/lib/adapter-test-environment.ts b/ui/src/lib/adapter-test-environment.ts new file mode 100644 index 0000000000..541f320c57 --- /dev/null +++ b/ui/src/lib/adapter-test-environment.ts @@ -0,0 +1,18 @@ +/** + * Which environment should an adapter "Test" probe? + * + * Mirrors the server's run-time resolution + * (`resolveExecutionWorkspaceEnvironmentId`): the agent's own environment + * wins, otherwise the instance default, otherwise none (the server probes + * the Paperclip host). Without the instance-default fallback, the Test + * button probes the host for agents that rely on the instance default and + * fails on commands that only exist inside the default environment — for + * example a sandbox image with an extra CLI installed — even though a real + * run would have resolved to that environment and succeeded. + */ +export function resolveAdapterTestEnvironmentId(input: { + agentDefaultEnvironmentId: string | null | undefined; + instanceDefaultEnvironmentId: string | null | undefined; +}): string | null { + return input.agentDefaultEnvironmentId || input.instanceDefaultEnvironmentId || null; +}