From d815c8b8dc8f2debdde820f6f636dc7b183708e3 Mon Sep 17 00:00:00 2001 From: Alexei Vedernikov <51241875+alvedder@users.noreply.github.com> Date: Tue, 4 Aug 2026 23:43:00 +0700 Subject: [PATCH] fix(llm): support per-request provider timeouts (#832) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(llm): support per-request provider timeouts * fix(llm): convert Gemini timeout to milliseconds * fix(llm): validate Gemini HTTP options * test(llm): type Anthropic stream context args * test(llm): live per-request timeout coverage for all providers Two live checks per provider: a generous timeout asserted at the SDK call boundary, and a tight timeout that must abort well under the 600s client default. Gemini's async transport can be aiohttp, so its tight timeout surfaces as asyncio.TimeoutError rather than httpx. Co-Authored-By: Claude Fable 5 * style(tests): drop extra blank line in anthropic backend test Co-Authored-By: Claude Fable 5 * fix(llm): validate provider_params.timeout at config load Move the timeout coercion into src.config as coerce_provider_timeout and run it from a field validator on ModelOverrideSettings.provider_params, so a bad value in config.toml/env fails at startup with the exact config path instead of surfacing per-request as a retried 500. Good values normalize to float seconds at load. The per-request guard in src.llm.backend now delegates to the same coercion (wrapping ValueError in ValidationException) and continues to cover extra_params passed programmatically. Co-Authored-By: Claude Fable 5 * docs: document provider_params.timeout load-time validation and gotchas Co-Authored-By: Claude Fable 5 * refactor(llm): address review nits on timeout plumbing Apply eisene's review feedback: - Rename PROVIDER_TIMEOUT_ERROR → PROVIDER_TIMEOUT_ERROR_TEXT - Move request_timeout_from_extra_params from backend.py (pure dataclasses) to request_builder.py (request assembly) - Add comment explaining Gemini's ms timeout conversion - Generalize _normalize_extra_params with _strip_none_params helper Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Aakash Kattelu Co-authored-by: Claude Fable 5 --- config.toml.example | 1 + docs/v3/contributing/configuration.mdx | 21 ++++ src/config.py | 40 ++++++++ src/llm/backends/anthropic.py | 14 ++- src/llm/backends/gemini.py | 29 +++++- src/llm/backends/openai.py | 9 +- src/llm/request_builder.py | 49 ++++++++- tests/live_llm/test_live_timeouts.py | 117 ++++++++++++++++++++++ tests/llm/test_backends/test_anthropic.py | 79 +++++++++++++++ tests/llm/test_backends/test_gemini.py | 36 ++++++- tests/llm/test_backends/test_openai.py | 73 ++++++++++++++ tests/llm/test_request_builder.py | 50 +++++++++ tests/test_config.py | 47 +++++++++ 13 files changed, 557 insertions(+), 8 deletions(-) create mode 100644 tests/live_llm/test_live_timeouts.py diff --git a/config.toml.example b/config.toml.example index e64d90f0..42146da7 100644 --- a/config.toml.example +++ b/config.toml.example @@ -138,6 +138,7 @@ model = "gpt-5.4-mini" # api_key_env = "DERIVER_CUSTOM_BACKUP_API_KEY" # [deriver.model_config.overrides.provider_params] # verbosity = "low" +# timeout = 3600.0 # Peer card settings [peer_card] diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index 8f4fb443..384c7841 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -189,8 +189,29 @@ Each model config supports an `overrides.provider_params` dict for passing arbit [deriver.model_config.overrides.provider_params] # These are passed directly to the provider SDK verbosity = "low" +# Per-request timeout in seconds; useful for queued workers that can wait longer +timeout = 3600.0 ``` +Because provider params live on each model config, background workers such as +the Deriver and Dreamer can use longer request timeouts while synchronous +chat paths keep tighter defaults. + +`timeout` gotchas: + +- The value is validated **at config load**: it must coerce to a positive, + finite number of seconds (numbers or numeric strings like `"3600"`), or the + process refuses to start with an error naming the offending config path. + This applies to both the primary model config and its `fallback.overrides`. +- The unit is always **seconds**, regardless of transport. OpenAI and + Anthropic receive it as the SDK's `timeout` kwarg; Gemini has no such + kwarg, so Honcho converts it to milliseconds on `http_options.timeout`. +- When unset, nothing is forwarded and each SDK's default applies — adding + this key is opt-in and changes no existing behavior. +- A too-tight timeout doesn't fail once: the aborted request goes through the + normal retry/fallback chain before the caller sees an error, so the + observed latency is several multiples of the timeout. + #### Transport passthrough keys Three keys inside `provider_params` are recognized as request-level escape hatches and forwarded to the underlying transport. Where a transport actually validates and merges one of these keys, its value must be a mapping — a non-mapping value raises a configuration error (see the per-transport behavior below; a key a transport ignores is not validated): diff --git a/src/config.py b/src/config.py index 84eedecd..177bef05 100644 --- a/src/config.py +++ b/src/config.py @@ -1,4 +1,5 @@ import logging +import math import os from pathlib import Path from typing import Annotated, Any, ClassVar, Literal, cast @@ -66,6 +67,37 @@ ThinkingEffortLevel = Literal[ StructuredOutputMode = Literal["json_schema", "json_object"] +PROVIDER_TIMEOUT_ERROR_TEXT = ( + "provider_params.timeout must be a positive number of seconds" +) + + +def coerce_provider_timeout(value: Any) -> float: + """Coerce a `provider_params.timeout` value to positive, finite seconds. + + Canonical implementation shared by config-load validation (here) and + per-request validation (`src.llm.request_builder.request_timeout_from_extra_params`, + which translates the ValueError into a ValidationException). Lives in + config.py because src.exceptions imports src.config, so config validators + cannot raise Honcho exception types. + """ + if isinstance(value, bool): + raise ValueError(PROVIDER_TIMEOUT_ERROR_TEXT) + if isinstance(value, int | float): + timeout = float(value) + elif isinstance(value, str): + try: + timeout = float(value.strip()) + except ValueError as exc: + raise ValueError(PROVIDER_TIMEOUT_ERROR_TEXT) from exc + else: + raise ValueError(PROVIDER_TIMEOUT_ERROR_TEXT) + + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError(PROVIDER_TIMEOUT_ERROR_TEXT) + return timeout + + class ModelOverrideSettings(BaseModel): """Advanced module-level transport overrides.""" @@ -91,6 +123,14 @@ class ModelOverrideSettings(BaseModel): ), ) + @field_validator("provider_params") + @classmethod + def _validate_provider_timeout(cls, v: dict[str, Any]) -> dict[str, Any]: + """Reject bad `timeout` values at config load; normalize good ones to float.""" + if "timeout" not in v: + return v + return {**v, "timeout": coerce_provider_timeout(v["timeout"])} + class PromptCachePolicy(BaseModel): """Per-call prompt-caching configuration. diff --git a/src/llm/backends/anthropic.py b/src/llm/backends/anthropic.py index 614a2e32..9f58af07 100644 --- a/src/llm/backends/anthropic.py +++ b/src/llm/backends/anthropic.py @@ -9,7 +9,10 @@ from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock from pydantic import BaseModel, ValidationError from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult -from src.llm.request_builder import apply_sdk_passthroughs +from src.llm.request_builder import ( + apply_sdk_passthroughs, + request_timeout_from_extra_params, +) from src.llm.structured_output import repair_response_model_json, schema_instruction @@ -74,6 +77,10 @@ class AnthropicBackend: # from ModelConfig.provider_params. Shallow merge with operator-wins. apply_sdk_passthroughs(params, extra_params) + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + params["timeout"] = timeout + # The '{' prefill forces a JSON-first response, which suppresses # tool_use blocks — skip it when tools are available and rely on the # conditional instruction + repair fallback instead. @@ -157,6 +164,11 @@ class AnthropicBackend: # Operator escape hatch: forward Anthropic SDK passthrough kwargs # from ModelConfig.provider_params. Shallow merge with operator-wins. apply_sdk_passthroughs(params, extra_params) + + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + params["timeout"] = timeout + # See complete(): no '{' prefill when tools are available, so # tool_use blocks stay reachable on the streamed path too. use_json_prefill = ( diff --git a/src/llm/backends/gemini.py b/src/llm/backends/gemini.py index c114196e..3f6ade64 100644 --- a/src/llm/backends/gemini.py +++ b/src/llm/backends/gemini.py @@ -4,6 +4,7 @@ from collections.abc import AsyncIterator from datetime import datetime, timedelta, timezone from typing import Any, ClassVar, cast +from google.genai import types as genai_types from pydantic import BaseModel from src.exceptions import LLMError, ValidationException @@ -14,7 +15,10 @@ from src.llm.caching import ( build_cache_key, gemini_cache_store, ) -from src.llm.request_builder import coerce_passthrough_mapping +from src.llm.request_builder import ( + coerce_passthrough_mapping, + request_timeout_from_extra_params, +) from src.llm.structured_output import repair_response_model_json, schema_instruction GEMINI_BLOCKED_FINISH_REASONS = { @@ -289,19 +293,38 @@ class GeminiBackend: # extra_query has no SDK-level equivalent and is ignored. Shallow # merge with operator-wins. Operators are responsible for not setting # unknown fields that google-genai's validation will reject. + http_options: genai_types.HttpOptions | None = None if extra_params: operator_extra_body = extra_params.get("extra_body") if operator_extra_body: config.update( coerce_passthrough_mapping("extra_body", operator_extra_body) ) + raw_http_options = config.get("http_options") + if isinstance(raw_http_options, genai_types.HttpOptions): + http_options = raw_http_options + elif isinstance(raw_http_options, dict): + http_options = genai_types.HttpOptions.model_validate( + raw_http_options + ) operator_extra_headers = extra_params.get("extra_headers") if operator_extra_headers: - http_options = config.setdefault("http_options", {}) - existing_headers = http_options.setdefault("headers", {}) + if http_options is None: + http_options = genai_types.HttpOptions() + existing_headers = dict(http_options.headers or {}) existing_headers.update( coerce_passthrough_mapping("extra_headers", operator_extra_headers) ) + http_options.headers = existing_headers + + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + if http_options is None: + http_options = genai_types.HttpOptions() + # Gemini has no native timeout kwarg; set the httpx-level value in ms. + http_options.timeout = int(timeout * 1000) + if http_options is not None: + config["http_options"] = http_options return config def _normalize_response( diff --git a/src/llm/backends/openai.py b/src/llm/backends/openai.py index d5d0ed73..672709ff 100644 --- a/src/llm/backends/openai.py +++ b/src/llm/backends/openai.py @@ -11,7 +11,10 @@ from pydantic import BaseModel, ValidationError from src.exceptions import ValidationException from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult -from src.llm.request_builder import apply_sdk_passthroughs +from src.llm.request_builder import ( + apply_sdk_passthroughs, + request_timeout_from_extra_params, +) from src.llm.structured_output import ( StructuredOutputError, empty_structured_output, @@ -397,6 +400,10 @@ class OpenAIBackend: # if the operator supplies `extra_body.reasoning`, it replaces any # value Honcho auto-injected above. apply_sdk_passthroughs(params, extra_params) + + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + params["timeout"] = timeout return params def _normalize_response( diff --git a/src/llm/request_builder.py b/src/llm/request_builder.py index 3da437b5..7eadab71 100644 --- a/src/llm/request_builder.py +++ b/src/llm/request_builder.py @@ -11,10 +11,14 @@ from typing import Any, cast from pydantic import BaseModel -from src.config import ModelConfig, PromptCachePolicy +from src.config import ModelConfig, PromptCachePolicy, coerce_provider_timeout from src.exceptions import ValidationException -from .backend import CompletionResult, ProviderBackend, StreamChunk +from .backend import ( + CompletionResult, + ProviderBackend, + StreamChunk, +) # Operator escape-hatch keys recognized inside ModelConfig.provider_params. PASSTHROUGH_KEYS = ("extra_body", "extra_headers", "extra_query") @@ -99,6 +103,45 @@ def build_config_extra_params(config: ModelConfig) -> dict[str, Any]: return extra_params +def request_timeout_from_extra_params( + extra_params: dict[str, Any] | None, +) -> float | None: + """Return a validated per-request provider timeout from extra params. + + Config-sourced timeouts are already validated and normalized at config + load (`coerce_provider_timeout` in src.config); this guards extra_params + passed programmatically at call time. + """ + if not extra_params or "timeout" not in extra_params: + return None + + try: + return coerce_provider_timeout(extra_params["timeout"]) + except ValueError as exc: + raise ValidationException(str(exc)) from exc + + +def _strip_none_params( + params: dict[str, Any], + keys: tuple[str, ...], +) -> dict[str, Any]: + """Remove specified keys from extra params when their values are None.""" + return {k: v for k, v in params.items() if not (k in keys and v is None)} + + +def _normalize_extra_params(extra_params: dict[str, Any]) -> dict[str, Any]: + """Normalize and clean shared extra params before they reach backends. + + Centralizes per-key coercion and null-stripping so new keys are added + here rather than spawning one-off normalizers. + """ + result = dict(extra_params) + timeout = request_timeout_from_extra_params(result) + if timeout is not None: + result["timeout"] = timeout + return _strip_none_params(result, ("timeout",)) + + async def execute_completion( backend: ProviderBackend, config: ModelConfig, @@ -120,6 +163,7 @@ async def execute_completion( **build_config_extra_params(config), **(extra_params or {}), } + merged_extra_params = _normalize_extra_params(merged_extra_params) if cache_policy is not None: merged_extra_params["cache_policy"] = cache_policy @@ -158,6 +202,7 @@ async def execute_stream( **build_config_extra_params(config), **(extra_params or {}), } + merged_extra_params = _normalize_extra_params(merged_extra_params) if cache_policy is not None: merged_extra_params["cache_policy"] = cache_policy diff --git a/tests/live_llm/test_live_timeouts.py b/tests/live_llm/test_live_timeouts.py new file mode 100644 index 00000000..28bf6143 --- /dev/null +++ b/tests/live_llm/test_live_timeouts.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import time +from typing import Any + +import anthropic +import httpx +import openai +import pytest + +from src.llm.request_builder import execute_completion + +from .conftest import make_backend, require_provider_key, wrap_async_method +from .model_matrix import LiveModelSpec, ProviderName, get_live_model_specs + +pytestmark = [pytest.mark.live_llm] + +GENEROUS_TIMEOUT_SECONDS = 120 +TIGHT_TIMEOUT_SECONDS = 0.01 +# Well under the 600s client default; generous enough to absorb SDK retries. +TIGHT_TIMEOUT_WALL_CLOCK_LIMIT_SECONDS = 30 + +TIMEOUT_EXCEPTIONS: dict[ProviderName, tuple[type[BaseException], ...]] = { + "anthropic": (anthropic.APITimeoutError,), + "openai": (openai.APITimeoutError,), + # google-genai raises httpx or aiohttp timeouts depending on its transport; + # aiohttp surfaces as asyncio.TimeoutError (== builtins.TimeoutError). + "gemini": (httpx.TimeoutException, TimeoutError), +} + +PROVIDER_MARKS = { + "anthropic": pytest.mark.requires_anthropic, + "openai": pytest.mark.requires_openai, + "gemini": pytest.mark.requires_gemini, +} + + +def representative_specs() -> list[Any]: + """One spec per provider — timeout plumbing is transport-level, not model-level.""" + params: list[Any] = [] + for provider in ("anthropic", "openai", "gemini"): + specs = get_live_model_specs(provider=provider) + if not specs: + continue + params.append( + pytest.param(specs[0], marks=PROVIDER_MARKS[provider], id=specs[0].id) + ) + return params + + +def assert_timeout_reached_sdk( + model_spec: LiveModelSpec, call_kwargs: dict[str, Any], timeout_seconds: float +) -> None: + if model_spec.provider == "gemini": + http_options = call_kwargs["config"]["http_options"] + assert http_options.timeout == int(timeout_seconds * 1000) + else: + assert call_kwargs["timeout"] == timeout_seconds + + +def sdk_call_target(backend: Any, model_spec: LiveModelSpec) -> tuple[Any, str]: + if model_spec.provider == "gemini": + return backend._client.aio.models, "generate_content" + if model_spec.provider == "anthropic": + return backend._client.messages, "create" + return backend._client.chat.completions, "create" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_spec", representative_specs()) +async def test_live_provider_timeout_reaches_the_wire( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend( + model_spec, provider_params={"timeout": GENEROUS_TIMEOUT_SECONDS} + ) + target, attribute = sdk_call_target(backend, model_spec) + calls = wrap_async_method(monkeypatch, target, attribute) + + result = await execute_completion( + backend, + config, + messages=[{"role": "user", "content": "Reply with the single word: ok"}], + max_tokens=256, + ) + + assert isinstance(result.content, str) + assert result.content.strip() + assert len(calls) == 1 + assert_timeout_reached_sdk(model_spec, calls[0]["kwargs"], GENEROUS_TIMEOUT_SECONDS) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_spec", representative_specs()) +async def test_live_tight_provider_timeout_aborts_request( + model_spec: LiveModelSpec, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend( + model_spec, provider_params={"timeout": TIGHT_TIMEOUT_SECONDS} + ) + + started = time.monotonic() + with pytest.raises(TIMEOUT_EXCEPTIONS[model_spec.provider]): + await execute_completion( + backend, + config, + messages=[{"role": "user", "content": "Reply with the single word: ok"}], + max_tokens=256, + ) + elapsed = time.monotonic() - started + + assert ( + elapsed < TIGHT_TIMEOUT_WALL_CLOCK_LIMIT_SECONDS + ), f"tight timeout took {elapsed:.1f}s — per-request timeout likely not applied" diff --git a/tests/llm/test_backends/test_anthropic.py b/tests/llm/test_backends/test_anthropic.py index 13255ec7..a226f626 100644 --- a/tests/llm/test_backends/test_anthropic.py +++ b/tests/llm/test_backends/test_anthropic.py @@ -449,3 +449,82 @@ async def test_anthropic_backend_stream_no_prefill_when_tools_present() -> None: "If not responding with a tool call, respond with valid JSON" in call["messages"][0]["content"] ) + + +@pytest.mark.asyncio +async def test_anthropic_backend_passes_timeout_to_completion_request() -> None: + """Anthropic completion requests receive per-request provider timeout.""" + client = Mock() + client.messages.create = AsyncMock( + return_value=SimpleNamespace( + content=[TextBlock(type="text", text="ok")], + usage=SimpleNamespace( + input_tokens=10, + output_tokens=5, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ), + stop_reason="end_turn", + ) + ) + + backend = AnthropicBackend(client) + await backend.complete( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": 45}, + ) + + await_args = client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic create call") + assert await_args.kwargs["timeout"] == 45.0 + + +@pytest.mark.asyncio +async def test_anthropic_backend_passes_timeout_to_stream_request() -> None: + """Anthropic stream requests receive per-request provider timeout.""" + + class FakeStream: + """Minimal async stream manager for Anthropic streaming tests.""" + + async def __aenter__(self): + """Return the stream object used by the backend.""" + return self + + async def __aexit__(self, *_args: object) -> bool: + """Do not suppress stream errors.""" + return False + + def __aiter__(self): + """Return the async iterator used by the backend.""" + return self + + async def __anext__(self): + """End the fake stream immediately.""" + raise StopAsyncIteration + + async def get_final_message(self): + """Return the final message required by the backend.""" + return SimpleNamespace( + usage=SimpleNamespace(output_tokens=1), + stop_reason="end_turn", + ) + + client = Mock() + client.messages.stream = Mock(return_value=FakeStream()) + + backend = AnthropicBackend(client) + chunks = [ + chunk + async for chunk in backend.stream( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": "60"}, + ) + ] + + assert chunks[-1].is_done is True + assert client.messages.stream.call_args.kwargs["timeout"] == 60.0 diff --git a/tests/llm/test_backends/test_gemini.py b/tests/llm/test_backends/test_gemini.py index 1e3933b4..8295ab26 100644 --- a/tests/llm/test_backends/test_gemini.py +++ b/tests/llm/test_backends/test_gemini.py @@ -98,6 +98,40 @@ async def test_gemini_backend_maps_thinking_effort_to_thinking_level() -> None: assert call["config"]["thinking_config"] == {"thinking_level": "low"} +@pytest.mark.asyncio +async def test_gemini_backend_maps_timeout_to_http_options() -> None: + """Gemini requests receive provider timeout through config http_options.""" + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=[SimpleNamespace(text="ok")]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": "90"}, + ) + + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + assert await_args.kwargs["config"]["http_options"].timeout == 90_000 + + @pytest.mark.asyncio async def test_gemini_backend_rejects_budget_and_effort_together() -> None: backend = GeminiBackend(Mock()) @@ -385,7 +419,7 @@ async def test_gemini_backend_forwards_provider_params_extra_headers() -> None: if await_args is None: raise AssertionError("Expected Gemini generate_content call") call = await_args.kwargs - assert call["config"]["http_options"]["headers"] == {"X-Trace-Id": "abc123"} + assert call["config"]["http_options"].headers == {"X-Trace-Id": "abc123"} @pytest.mark.asyncio diff --git a/tests/llm/test_backends/test_openai.py b/tests/llm/test_backends/test_openai.py index 8567e034..fbd8e719 100644 --- a/tests/llm/test_backends/test_openai.py +++ b/tests/llm/test_backends/test_openai.py @@ -510,6 +510,7 @@ async def test_openai_backend_converts_anthropic_style_tools() -> None: assert call["tool_choice"] == "required" +@pytest.mark.asyncio async def test_openai_backend_translates_canonical_any_tool_choice_to_required() -> ( None ): @@ -571,6 +572,78 @@ def test_openai_convert_tool_choice(canonical: Any, expected: Any) -> None: assert OpenAIBackend._convert_tool_choice(canonical) == expected # pyright: ignore[reportPrivateUsage] +@pytest.mark.asyncio +async def test_openai_backend_passes_timeout_to_completion_request() -> None: + """OpenAI completion requests receive per-request provider timeout.""" + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="ok", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": 12.5}, + ) + + assert _await_kwargs(client.chat.completions.create)["timeout"] == 12.5 + + +@pytest.mark.asyncio +async def test_openai_backend_passes_timeout_to_structured_parse_request() -> None: + """OpenAI structured parse requests receive per-request provider timeout.""" + client = Mock() + client.chat.completions.parse = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + parsed=_StructuredResponse(answer="ok"), + content='{"answer":"ok"}', + tool_calls=[], + refusal=None, + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=_StructuredResponse, + extra_params={"timeout": "30"}, + ) + + assert _await_kwargs(client.chat.completions.parse)["timeout"] == 30.0 + + @pytest.mark.parametrize( "model", [ diff --git a/tests/llm/test_request_builder.py b/tests/llm/test_request_builder.py index c8ed7dfd..a8355a51 100644 --- a/tests/llm/test_request_builder.py +++ b/tests/llm/test_request_builder.py @@ -1,6 +1,8 @@ +import pytest from pydantic import BaseModel from src.config import ModelConfig +from src.exceptions import ValidationException from src.llm.caching import PromptCachePolicy from src.llm.request_builder import execute_completion from tests.llm.conftest import FakeBackend @@ -95,3 +97,51 @@ async def test_provider_params_are_merged_into_extra_params( call = fake_backend.calls[0] assert call["extra_params"]["top_p"] == 0.9 assert call["extra_params"]["custom_flag"] is True + + +async def test_provider_timeout_is_normalized_into_extra_params( + fake_backend: FakeBackend, +) -> None: + """Numeric-string provider timeout values are normalized before backends.""" + config = ModelConfig( + model="gpt-4.1-mini", + transport="openai", + provider_params={"timeout": "42.5"}, + ) + + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + call = fake_backend.calls[0] + assert call["extra_params"]["timeout"] == 42.5 + + +@pytest.mark.parametrize( + "timeout", + ["slow", "", 0, -1, True, float("nan"), float("inf"), "nan", "inf"], +) +async def test_provider_timeout_rejects_invalid_values( + fake_backend: FakeBackend, + timeout: object, +) -> None: + """Invalid provider timeout values fail before provider SDK calls.""" + config = ModelConfig( + model="gpt-4.1-mini", + transport="openai", + provider_params={"timeout": timeout}, + ) + + with pytest.raises( + ValidationException, + match=r"provider_params\.timeout must be a positive number of seconds", + ): + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) diff --git a/tests/test_config.py b/tests/test_config.py index 7730c751..1a714e55 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -81,3 +81,50 @@ def test_representation_batch_target_input_cannot_exceed_max_input_tokens() -> N MAX_INPUT_TOKENS=1000, REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=2048, ) + + +def _configured_with_timeout(timeout: object) -> ConfiguredModelSettings: + return ConfiguredModelSettings.model_validate( + { + "model": "gpt-5.4-mini", + "transport": "openai", + "overrides": {"provider_params": {"timeout": timeout}}, + } + ) + + +@pytest.mark.parametrize("timeout", [30, 42.5, "42.5", " 60 "]) +def test_provider_timeout_is_normalized_at_config_load(timeout: object) -> None: + settings = _configured_with_timeout(timeout) + + normalized = settings.overrides.provider_params["timeout"] + assert isinstance(normalized, float) + assert normalized == float(str(timeout).strip()) + + +@pytest.mark.parametrize( + "timeout", + ["slow", "", 0, -1, True, float("nan"), float("inf"), "nan", "inf", None, [30]], +) +def test_provider_timeout_is_rejected_at_config_load(timeout: object) -> None: + with pytest.raises( + ValueError, match=r"provider_params\.timeout must be a positive number" + ): + _configured_with_timeout(timeout) + + +def test_provider_timeout_on_fallback_overrides_is_validated_at_config_load() -> None: + with pytest.raises( + ValueError, match=r"provider_params\.timeout must be a positive number" + ): + ConfiguredModelSettings.model_validate( + { + "model": "gpt-5.4-mini", + "transport": "openai", + "fallback": { + "model": "gpt-4.1", + "transport": "openai", + "overrides": {"provider_params": {"timeout": "slow"}}, + }, + } + )