diff --git a/ui/src/components/AgentConfigForm.render.test.tsx b/ui/src/components/AgentConfigForm.render.test.tsx index 9d94ed5073..f28a1d826a 100644 --- a/ui/src/components/AgentConfigForm.render.test.tsx +++ b/ui/src/components/AgentConfigForm.render.test.tsx @@ -9,9 +9,11 @@ import { TooltipProvider } from "@/components/ui/tooltip"; import { AgentConfigForm } from "./AgentConfigForm"; const mockAgentsApi = vi.hoisted(() => ({ + adapterModelProfiles: vi.fn(), adapterModels: vi.fn(), detectModel: vi.fn(), list: vi.fn(), + testEnvironment: vi.fn(), })); const mockEnvironmentsApi = vi.hoisted(() => ({ @@ -155,7 +157,11 @@ function makeEnvironment(overrides: Partial): Environment { }; } -async function renderForm(environments: Environment[], agentOverrides: Partial = {}) { +async function renderForm( + environments: Environment[], + agentOverrides: Partial = {}, + options: { showAdapterTestEnvironmentButton?: boolean } = {}, +) { mockEnvironmentsApi.list.mockResolvedValue(environments); const container = document.createElement("div"); @@ -178,7 +184,7 @@ async function renderForm(environments: Environment[], agentOverrides: Partial , @@ -193,9 +199,16 @@ describe("AgentConfigForm environment selector", () => { let roots: Root[] = []; beforeEach(() => { + mockAgentsApi.adapterModelProfiles.mockResolvedValue([]); mockAgentsApi.adapterModels.mockResolvedValue([]); mockAgentsApi.detectModel.mockResolvedValue(null); mockAgentsApi.list.mockResolvedValue([]); + mockAgentsApi.testEnvironment.mockResolvedValue({ + adapterType: "codex_local", + status: "pass", + checks: [], + testedAt: new Date(0).toISOString(), + }); mockInstanceSettingsApi.get.mockResolvedValue({ defaultEnvironmentId: null }); mockInstanceSettingsApi.getExperimental.mockResolvedValue({ enableEnvironments: true }); mockInstanceSettingsApi.getGeneral.mockResolvedValue({ executionMode: "any" }); @@ -269,4 +282,83 @@ describe("AgentConfigForm environment selector", () => { expect(selector?.textContent).toContain("Default: Local"); expect(selector?.textContent).toContain("Fake Sandbox ยท sandbox"); }); + + it("tests both the primary and cheap models when a cheap profile is configured", async () => { + const result = await renderForm([ + makeEnvironment({ id: "local-1", name: "Local", driver: "local" }), + ], { + adapterConfig: { model: "gpt-5.4" }, + runtimeConfig: { + modelProfiles: { + cheap: { + enabled: true, + adapterConfig: { + model: "gpt-5.4-mini", + baseUrl: "https://cheap-models.example.test", + provider: "budget-provider", + }, + }, + }, + }, + }, { + showAdapterTestEnvironmentButton: true, + }); + roots.push(result.root); + + const testButton = Array.from(result.container.querySelectorAll("button")).find( + (button) => button.textContent?.trim() === "Test", + ); + expect(testButton).toBeTruthy(); + + await act(async () => { + testButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + await flushReact(); + + expect(mockAgentsApi.testEnvironment).toHaveBeenCalledTimes(2); + expect(mockAgentsApi.testEnvironment.mock.calls[0]?.[2]).toMatchObject({ + adapterConfig: expect.objectContaining({ model: "gpt-5.4" }), + }); + expect(mockAgentsApi.testEnvironment.mock.calls[1]?.[2]).toMatchObject({ + adapterConfig: expect.objectContaining({ + model: "gpt-5.4-mini", + baseUrl: "https://cheap-models.example.test", + provider: "budget-provider", + }), + }); + }); + + it("surfaces request failures instead of converting them into model test checks", async () => { + mockAgentsApi.testEnvironment.mockRejectedValueOnce(new Error("Network unavailable")); + + const result = await renderForm([ + makeEnvironment({ id: "local-1", name: "Local", driver: "local" }), + ], { + adapterConfig: { model: "gpt-5.4" }, + runtimeConfig: { + modelProfiles: { + cheap: { + enabled: true, + adapterConfig: { model: "gpt-5.4-mini" }, + }, + }, + }, + }, { + showAdapterTestEnvironmentButton: true, + }); + roots.push(result.root); + + const testButton = Array.from(result.container.querySelectorAll("button")).find( + (button) => button.textContent?.trim() === "Test", + ); + expect(testButton).toBeTruthy(); + + await act(async () => { + testButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + await flushReact(); + + expect(mockAgentsApi.testEnvironment).toHaveBeenCalledTimes(1); + expect(result.container.textContent).toContain("Network unavailable"); + }); }); diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsx index a673f2ca07..06f12521e9 100644 --- a/ui/src/components/AgentConfigForm.tsx +++ b/ui/src/components/AgentConfigForm.tsx @@ -516,12 +516,19 @@ export function AgentConfigForm(props: AgentConfigFormProps) { return typeof value === "string" ? value : ""; }, [adapterCheapDefault]); - function buildAdapterConfigForTest(): Record { + function buildAdapterConfigForTest(adapterConfigPatch?: Record): Record { if (isCreate) { - return uiAdapter.buildAdapterConfig(val!); + const next = uiAdapter.buildAdapterConfig(val!); + if (adapterConfigPatch) { + Object.assign(next, adapterConfigPatch); + } + return next; } const base = config as Record; const next = { ...base, ...overlay.adapterConfig }; + if (adapterConfigPatch) { + Object.assign(next, adapterConfigPatch); + } if (adapterType === "hermes_local") { const hermesCommand = typeof next.hermesCommand === "string" && next.hermesCommand.length > 0 @@ -536,15 +543,123 @@ export function AgentConfigForm(props: AgentConfigFormProps) { return next; } + function buildCheapAdapterConfigForTest(): Record { + const adapterDefaultConfig = asObject(adapterCheapDefault?.adapterConfig); + const createCheapModel = isCreate ? (val!.cheapModel ?? "").trim() : ""; + const cheapAdapterConfig = isCreate + ? { + ...adapterDefaultConfig, + ...(createCheapModel ? { model: createCheapModel } : {}), + } + : { + ...adapterDefaultConfig, + ...cheapProfileFromAgent.adapterConfig, + ...asObject(cheapOverlay?.adapterConfig), + }; + return buildAdapterConfigForTest(cheapAdapterConfig); + } + + function getCheapModelTestCase(): { model: string; adapterConfig: Record } | null { + if (!currentCheapEnabled) return null; + const adapterConfig = buildCheapAdapterConfigForTest(); + const configModel = typeof adapterConfig.model === "string" ? adapterConfig.model.trim() : ""; + const model = configModel || currentCheapModel.trim(); + if (!model) return null; + adapterConfig.model = model; + return { model, adapterConfig }; + } + + function prefixEnvironmentTestChecks( + result: AdapterEnvironmentTestResult, + label: string, + model: string | null, + ): AdapterEnvironmentTestResult { + const modelLabel = model ? ` (${model})` : ""; + return { + ...result, + checks: [ + { + code: `${label.toLowerCase().replace(/[^a-z0-9]+/g, "_")}_test_started`, + level: "info", + message: `${label} test${modelLabel}`, + }, + ...result.checks.map((check) => ({ + ...check, + message: `${label} test${modelLabel}: ${check.message}`, + })), + ], + }; + } + + async function runEnvironmentTestCase( + label: string, + model: string | null, + adapterConfig: Record, + environmentId: string | null, + ): Promise { + const result = await agentsApi.testEnvironment(selectedCompanyId!, adapterType, { + adapterConfig, + environmentId, + }); + return prefixEnvironmentTestChecks(result, label, model); + } + + function mergeEnvironmentTestResults( + results: AdapterEnvironmentTestResult[], + ): AdapterEnvironmentTestResult { + const checks = results.flatMap((result) => result.checks); + const status = results.some((result) => result.status === "fail") + ? "fail" + : results.some((result) => result.status === "warn") + ? "warn" + : "pass"; + const testedAt = results[results.length - 1]?.testedAt ?? new Date().toISOString(); + + return { + adapterType, + status, + checks, + testedAt, + }; + } + const testEnvironment = useMutation({ mutationFn: async () => { if (!selectedCompanyId) { throw new Error("Select a company to test adapter environment"); } - return agentsApi.testEnvironment(selectedCompanyId, adapterType, { - adapterConfig: buildAdapterConfigForTest(), - environmentId: currentDefaultEnvironmentId || null, - }); + const primaryModel = currentModelId.trim() || null; + const cheapTestCase = getCheapModelTestCase(); + const environmentId = currentDefaultEnvironmentId || null; + const testResults: Array<{ label: string; model: string | null; result: AdapterEnvironmentTestResult }> = [ + { + label: "Primary model", + model: primaryModel, + result: await runEnvironmentTestCase( + "Primary model", + primaryModel, + buildAdapterConfigForTest(), + environmentId, + ), + }, + ]; + + if (cheapTestCase) { + testResults.push({ + label: "Cheap model", + model: cheapTestCase.model, + result: await runEnvironmentTestCase( + "Cheap model", + cheapTestCase.model, + cheapTestCase.adapterConfig, + environmentId, + ), + }); + } + + return testResults.length > 1 + ? mergeEnvironmentTestResults(testResults.map(({ result }) => result)) + : testResults[0]!.result; }, }); const [testActionPending, setTestActionPending] = useState(false); @@ -692,9 +807,10 @@ export function AgentConfigForm(props: AgentConfigFormProps) { const cheapProfileFromAgent = useMemo(() => { const profiles = (runtimeConfig.modelProfiles ?? {}) as Record; const cheap = (profiles.cheap ?? {}) as Record; - const cheapAdapterConfig = (cheap.adapterConfig ?? {}) as Record; + const cheapAdapterConfig = asObject(cheap.adapterConfig); return { enabled: cheap.enabled !== false, + adapterConfig: cheapAdapterConfig, model: typeof cheapAdapterConfig.model === "string" ? cheapAdapterConfig.model : "", }; }, [runtimeConfig]);