diff --git a/config.toml.example b/config.toml.example index e64d90f0..42146da7 100644 --- a/config.toml.example +++ b/config.toml.example @@ -138,6 +138,7 @@ model = "gpt-5.4-mini" # api_key_env = "DERIVER_CUSTOM_BACKUP_API_KEY" # [deriver.model_config.overrides.provider_params] # verbosity = "low" +# timeout = 3600.0 # Peer card settings [peer_card] diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index 8f4fb443..0c073a56 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -189,8 +189,14 @@ Each model config supports an `overrides.provider_params` dict for passing arbit [deriver.model_config.overrides.provider_params] # These are passed directly to the provider SDK verbosity = "low" +# Per-request timeout in seconds; useful for queued workers that can wait longer +timeout = 3600.0 ``` +Because provider params live on each model config, background workers such as +the Deriver and Dreamer can use longer request timeouts while synchronous +chat paths keep tighter defaults. + #### Transport passthrough keys Three keys inside `provider_params` are recognized as request-level escape hatches and forwarded to the underlying transport. Where a transport actually validates and merges one of these keys, its value must be a mapping — a non-mapping value raises a configuration error (see the per-transport behavior below; a key a transport ignores is not validated): diff --git a/src/llm/backend.py b/src/llm/backend.py index 380911d2..0211f161 100644 --- a/src/llm/backend.py +++ b/src/llm/backend.py @@ -1,11 +1,14 @@ from __future__ import annotations +import math from collections.abc import AsyncIterator from dataclasses import dataclass, field from typing import Any, Protocol, runtime_checkable from pydantic import BaseModel +from src.exceptions import ValidationException + @dataclass(slots=True) class ToolCallResult: @@ -45,6 +48,39 @@ class StreamChunk: output_tokens: int | None = None +def request_timeout_from_extra_params( + extra_params: dict[str, Any] | None, +) -> float | None: + """Return a validated per-request provider timeout from extra params.""" + if not extra_params or "timeout" not in extra_params: + return None + + value = extra_params["timeout"] + if isinstance(value, bool): + raise ValidationException( + "provider_params.timeout must be a positive number of seconds" + ) + if isinstance(value, int | float): + timeout = float(value) + elif isinstance(value, str): + try: + timeout = float(value.strip()) + except ValueError as exc: + raise ValidationException( + "provider_params.timeout must be a positive number of seconds" + ) from exc + else: + raise ValidationException( + "provider_params.timeout must be a positive number of seconds" + ) + + if not math.isfinite(timeout) or timeout <= 0: + raise ValidationException( + "provider_params.timeout must be a positive number of seconds" + ) + return timeout + + @runtime_checkable class ProviderBackend(Protocol): """Transport-agnostic interface for LLM providers. diff --git a/src/llm/backends/anthropic.py b/src/llm/backends/anthropic.py index 614a2e32..c37adf1e 100644 --- a/src/llm/backends/anthropic.py +++ b/src/llm/backends/anthropic.py @@ -8,7 +8,12 @@ from typing import Any from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock from pydantic import BaseModel, ValidationError -from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.backend import ( + CompletionResult, + StreamChunk, + ToolCallResult, + request_timeout_from_extra_params, +) from src.llm.request_builder import apply_sdk_passthroughs from src.llm.structured_output import repair_response_model_json, schema_instruction @@ -74,6 +79,10 @@ class AnthropicBackend: # from ModelConfig.provider_params. Shallow merge with operator-wins. apply_sdk_passthroughs(params, extra_params) + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + params["timeout"] = timeout + # The '{' prefill forces a JSON-first response, which suppresses # tool_use blocks — skip it when tools are available and rely on the # conditional instruction + repair fallback instead. @@ -157,6 +166,11 @@ class AnthropicBackend: # Operator escape hatch: forward Anthropic SDK passthrough kwargs # from ModelConfig.provider_params. Shallow merge with operator-wins. apply_sdk_passthroughs(params, extra_params) + + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + params["timeout"] = timeout + # See complete(): no '{' prefill when tools are available, so # tool_use blocks stay reachable on the streamed path too. use_json_prefill = ( diff --git a/src/llm/backends/gemini.py b/src/llm/backends/gemini.py index c114196e..5194f6a4 100644 --- a/src/llm/backends/gemini.py +++ b/src/llm/backends/gemini.py @@ -4,10 +4,16 @@ from collections.abc import AsyncIterator from datetime import datetime, timedelta, timezone from typing import Any, ClassVar, cast +from google.genai import types as genai_types from pydantic import BaseModel from src.exceptions import LLMError, ValidationException -from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.backend import ( + CompletionResult, + StreamChunk, + ToolCallResult, + request_timeout_from_extra_params, +) from src.llm.caching import ( GeminiCacheHandle, PromptCachePolicy, @@ -289,19 +295,35 @@ class GeminiBackend: # extra_query has no SDK-level equivalent and is ignored. Shallow # merge with operator-wins. Operators are responsible for not setting # unknown fields that google-genai's validation will reject. + http_options: genai_types.HttpOptions | None = None if extra_params: operator_extra_body = extra_params.get("extra_body") if operator_extra_body: config.update( coerce_passthrough_mapping("extra_body", operator_extra_body) ) + raw_http_options = config.get("http_options") + if isinstance(raw_http_options, genai_types.HttpOptions): + http_options = raw_http_options + elif isinstance(raw_http_options, dict): + http_options = genai_types.HttpOptions(**raw_http_options) operator_extra_headers = extra_params.get("extra_headers") if operator_extra_headers: - http_options = config.setdefault("http_options", {}) - existing_headers = http_options.setdefault("headers", {}) + if http_options is None: + http_options = genai_types.HttpOptions() + existing_headers = dict(http_options.headers or {}) existing_headers.update( coerce_passthrough_mapping("extra_headers", operator_extra_headers) ) + http_options.headers = existing_headers + + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + if http_options is None: + http_options = genai_types.HttpOptions() + http_options.timeout = timeout + if http_options is not None: + config["http_options"] = http_options return config def _normalize_response( diff --git a/src/llm/backends/openai.py b/src/llm/backends/openai.py index d5d0ed73..29d91971 100644 --- a/src/llm/backends/openai.py +++ b/src/llm/backends/openai.py @@ -10,7 +10,12 @@ from openai import BadRequestError, LengthFinishReasonError from pydantic import BaseModel, ValidationError from src.exceptions import ValidationException -from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.backend import ( + CompletionResult, + StreamChunk, + ToolCallResult, + request_timeout_from_extra_params, +) from src.llm.request_builder import apply_sdk_passthroughs from src.llm.structured_output import ( StructuredOutputError, @@ -397,6 +402,10 @@ class OpenAIBackend: # if the operator supplies `extra_body.reasoning`, it replaces any # value Honcho auto-injected above. apply_sdk_passthroughs(params, extra_params) + + timeout = request_timeout_from_extra_params(extra_params) + if timeout is not None: + params["timeout"] = timeout return params def _normalize_response( diff --git a/src/llm/request_builder.py b/src/llm/request_builder.py index 3da437b5..20c710aa 100644 --- a/src/llm/request_builder.py +++ b/src/llm/request_builder.py @@ -14,7 +14,12 @@ from pydantic import BaseModel from src.config import ModelConfig, PromptCachePolicy from src.exceptions import ValidationException -from .backend import CompletionResult, ProviderBackend, StreamChunk +from .backend import ( + CompletionResult, + ProviderBackend, + StreamChunk, + request_timeout_from_extra_params, +) # Operator escape-hatch keys recognized inside ModelConfig.provider_params. PASSTHROUGH_KEYS = ("extra_body", "extra_headers", "extra_query") @@ -99,6 +104,14 @@ def build_config_extra_params(config: ModelConfig) -> dict[str, Any]: return extra_params +def _normalize_extra_params(extra_params: dict[str, Any]) -> dict[str, Any]: + """Normalize shared extra params before they reach provider backends.""" + timeout = request_timeout_from_extra_params(extra_params) + if timeout is None: + return extra_params + return {**extra_params, "timeout": timeout} + + async def execute_completion( backend: ProviderBackend, config: ModelConfig, @@ -120,6 +133,7 @@ async def execute_completion( **build_config_extra_params(config), **(extra_params or {}), } + merged_extra_params = _normalize_extra_params(merged_extra_params) if cache_policy is not None: merged_extra_params["cache_policy"] = cache_policy @@ -158,6 +172,7 @@ async def execute_stream( **build_config_extra_params(config), **(extra_params or {}), } + merged_extra_params = _normalize_extra_params(merged_extra_params) if cache_policy is not None: merged_extra_params["cache_policy"] = cache_policy diff --git a/tests/llm/test_backends/test_anthropic.py b/tests/llm/test_backends/test_anthropic.py index 13255ec7..c040df28 100644 --- a/tests/llm/test_backends/test_anthropic.py +++ b/tests/llm/test_backends/test_anthropic.py @@ -430,6 +430,7 @@ async def test_anthropic_backend_stream_no_prefill_when_tools_present() -> None: client = Mock() client.messages.stream = Mock(return_value=_FakeStream()) + backend = AnthropicBackend(client) chunks = [ chunk @@ -449,3 +450,82 @@ async def test_anthropic_backend_stream_no_prefill_when_tools_present() -> None: "If not responding with a tool call, respond with valid JSON" in call["messages"][0]["content"] ) + + +@pytest.mark.asyncio +async def test_anthropic_backend_passes_timeout_to_completion_request() -> None: + """Anthropic completion requests receive per-request provider timeout.""" + client = Mock() + client.messages.create = AsyncMock( + return_value=SimpleNamespace( + content=[TextBlock(type="text", text="ok")], + usage=SimpleNamespace( + input_tokens=10, + output_tokens=5, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ), + stop_reason="end_turn", + ) + ) + + backend = AnthropicBackend(client) + await backend.complete( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": 45}, + ) + + await_args = client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic create call") + assert await_args.kwargs["timeout"] == 45.0 + + +@pytest.mark.asyncio +async def test_anthropic_backend_passes_timeout_to_stream_request() -> None: + """Anthropic stream requests receive per-request provider timeout.""" + + class FakeStream: + """Minimal async stream manager for Anthropic streaming tests.""" + + async def __aenter__(self): + """Return the stream object used by the backend.""" + return self + + async def __aexit__(self, *_args): + """Do not suppress stream errors.""" + return False + + def __aiter__(self): + """Return the async iterator used by the backend.""" + return self + + async def __anext__(self): + """End the fake stream immediately.""" + raise StopAsyncIteration + + async def get_final_message(self): + """Return the final message required by the backend.""" + return SimpleNamespace( + usage=SimpleNamespace(output_tokens=1), + stop_reason="end_turn", + ) + + client = Mock() + client.messages.stream = Mock(return_value=FakeStream()) + + backend = AnthropicBackend(client) + chunks = [ + chunk + async for chunk in backend.stream( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": "60"}, + ) + ] + + assert chunks[-1].is_done is True + assert client.messages.stream.call_args.kwargs["timeout"] == 60.0 diff --git a/tests/llm/test_backends/test_gemini.py b/tests/llm/test_backends/test_gemini.py index 1e3933b4..0cebdb59 100644 --- a/tests/llm/test_backends/test_gemini.py +++ b/tests/llm/test_backends/test_gemini.py @@ -98,6 +98,40 @@ async def test_gemini_backend_maps_thinking_effort_to_thinking_level() -> None: assert call["config"]["thinking_config"] == {"thinking_level": "low"} +@pytest.mark.asyncio +async def test_gemini_backend_maps_timeout_to_http_options() -> None: + """Gemini requests receive provider timeout through config http_options.""" + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=[SimpleNamespace(text="ok")]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": "90"}, + ) + + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + assert await_args.kwargs["config"]["http_options"].timeout == 90.0 + + @pytest.mark.asyncio async def test_gemini_backend_rejects_budget_and_effort_together() -> None: backend = GeminiBackend(Mock()) @@ -385,7 +419,7 @@ async def test_gemini_backend_forwards_provider_params_extra_headers() -> None: if await_args is None: raise AssertionError("Expected Gemini generate_content call") call = await_args.kwargs - assert call["config"]["http_options"]["headers"] == {"X-Trace-Id": "abc123"} + assert call["config"]["http_options"].headers == {"X-Trace-Id": "abc123"} @pytest.mark.asyncio diff --git a/tests/llm/test_backends/test_openai.py b/tests/llm/test_backends/test_openai.py index 8567e034..fbd8e719 100644 --- a/tests/llm/test_backends/test_openai.py +++ b/tests/llm/test_backends/test_openai.py @@ -510,6 +510,7 @@ async def test_openai_backend_converts_anthropic_style_tools() -> None: assert call["tool_choice"] == "required" +@pytest.mark.asyncio async def test_openai_backend_translates_canonical_any_tool_choice_to_required() -> ( None ): @@ -571,6 +572,78 @@ def test_openai_convert_tool_choice(canonical: Any, expected: Any) -> None: assert OpenAIBackend._convert_tool_choice(canonical) == expected # pyright: ignore[reportPrivateUsage] +@pytest.mark.asyncio +async def test_openai_backend_passes_timeout_to_completion_request() -> None: + """OpenAI completion requests receive per-request provider timeout.""" + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="ok", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"timeout": 12.5}, + ) + + assert _await_kwargs(client.chat.completions.create)["timeout"] == 12.5 + + +@pytest.mark.asyncio +async def test_openai_backend_passes_timeout_to_structured_parse_request() -> None: + """OpenAI structured parse requests receive per-request provider timeout.""" + client = Mock() + client.chat.completions.parse = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + parsed=_StructuredResponse(answer="ok"), + content='{"answer":"ok"}', + tool_calls=[], + refusal=None, + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=_StructuredResponse, + extra_params={"timeout": "30"}, + ) + + assert _await_kwargs(client.chat.completions.parse)["timeout"] == 30.0 + + @pytest.mark.parametrize( "model", [ diff --git a/tests/llm/test_request_builder.py b/tests/llm/test_request_builder.py index c8ed7dfd..a8355a51 100644 --- a/tests/llm/test_request_builder.py +++ b/tests/llm/test_request_builder.py @@ -1,6 +1,8 @@ +import pytest from pydantic import BaseModel from src.config import ModelConfig +from src.exceptions import ValidationException from src.llm.caching import PromptCachePolicy from src.llm.request_builder import execute_completion from tests.llm.conftest import FakeBackend @@ -95,3 +97,51 @@ async def test_provider_params_are_merged_into_extra_params( call = fake_backend.calls[0] assert call["extra_params"]["top_p"] == 0.9 assert call["extra_params"]["custom_flag"] is True + + +async def test_provider_timeout_is_normalized_into_extra_params( + fake_backend: FakeBackend, +) -> None: + """Numeric-string provider timeout values are normalized before backends.""" + config = ModelConfig( + model="gpt-4.1-mini", + transport="openai", + provider_params={"timeout": "42.5"}, + ) + + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + call = fake_backend.calls[0] + assert call["extra_params"]["timeout"] == 42.5 + + +@pytest.mark.parametrize( + "timeout", + ["slow", "", 0, -1, True, float("nan"), float("inf"), "nan", "inf"], +) +async def test_provider_timeout_rejects_invalid_values( + fake_backend: FakeBackend, + timeout: object, +) -> None: + """Invalid provider timeout values fail before provider SDK calls.""" + config = ModelConfig( + model="gpt-4.1-mini", + transport="openai", + provider_params={"timeout": timeout}, + ) + + with pytest.raises( + ValidationException, + match=r"provider_params\.timeout must be a positive number of seconds", + ): + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + )