RomanNum3ral
/
honcho
mirror of https://github.com/plastic-labs/honcho.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(llm): support per-request provider timeouts

This commit is contained in:
alvedder 2026-06-22 14:22:58 +05:00
parent 4f9a41360a
commit 6efcad606a
11 changed files with 347 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config.toml.example
View File

@ -138,6 +138,7 @@ model = "gpt-5.4-mini"
# api_key_env = "DERIVER_CUSTOM_BACKUP_API_KEY"
# [deriver.model_config.overrides.provider_params]
# verbosity = "low"
# timeout = 3600.0
# Peer card settings
[peer_card]

6
docs/v3/contributing/configuration.mdx
View File

@ -189,8 +189,14 @@ Each model config supports an `overrides.provider_params` dict for passing arbit
[deriver.model_config.overrides.provider_params]
# These are passed directly to the provider SDK
verbosity = "low"
# Per-request timeout in seconds; useful for queued workers that can wait longer
timeout = 3600.0
```
Because provider params live on each model config, background workers such as
the Deriver and Dreamer can use longer request timeouts while synchronous
chat paths keep tighter defaults.
#### Transport passthrough keys
Three keys inside `provider_params` are recognized as request-level escape hatches and forwarded to the underlying transport. Where a transport actually validates and merges one of these keys, its value must be a mapping — a non-mapping value raises a configuration error (see the per-transport behavior below; a key a transport ignores is not validated):

36
src/llm/backend.py
View File

@ -1,11 +1,14 @@
from __future__ import annotations
import math
from collections.abc import AsyncIterator
from dataclasses import dataclass, field
from typing import Any, Protocol, runtime_checkable
from pydantic import BaseModel
from src.exceptions import ValidationException
@dataclass(slots=True)
class ToolCallResult:
@ -45,6 +48,39 @@ class StreamChunk:
output_tokens: int | None = None
def request_timeout_from_extra_params(
extra_params: dict[str, Any] | None,
) -> float | None:
"""Return a validated per-request provider timeout from extra params."""
if not extra_params or "timeout" not in extra_params:
return None
value = extra_params["timeout"]
if isinstance(value, bool):
raise ValidationException(
"provider_params.timeout must be a positive number of seconds"
)
if isinstance(value, int | float):
timeout = float(value)
elif isinstance(value, str):
try:
timeout = float(value.strip())
except ValueError as exc:
raise ValidationException(
"provider_params.timeout must be a positive number of seconds"
) from exc
else:
raise ValidationException(
"provider_params.timeout must be a positive number of seconds"
)
if not math.isfinite(timeout) or timeout <= 0:
raise ValidationException(
"provider_params.timeout must be a positive number of seconds"
)
return timeout
@runtime_checkable
class ProviderBackend(Protocol):
"""Transport-agnostic interface for LLM providers.

16
src/llm/backends/anthropic.py
View File

@ -8,7 +8,12 @@ from typing import Any
from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock
from pydantic import BaseModel, ValidationError
from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult
from src.llm.backend import (
CompletionResult,
StreamChunk,
ToolCallResult,
request_timeout_from_extra_params,
)
from src.llm.request_builder import apply_sdk_passthroughs
from src.llm.structured_output import repair_response_model_json, schema_instruction
@ -74,6 +79,10 @@ class AnthropicBackend:
# from ModelConfig.provider_params. Shallow merge with operator-wins.
apply_sdk_passthroughs(params, extra_params)
timeout = request_timeout_from_extra_params(extra_params)
if timeout is not None:
params["timeout"] = timeout
# The '{' prefill forces a JSON-first response, which suppresses
# tool_use blocks — skip it when tools are available and rely on the
# conditional instruction + repair fallback instead.
@ -157,6 +166,11 @@ class AnthropicBackend:
# Operator escape hatch: forward Anthropic SDK passthrough kwargs
# from ModelConfig.provider_params. Shallow merge with operator-wins.
apply_sdk_passthroughs(params, extra_params)
timeout = request_timeout_from_extra_params(extra_params)
if timeout is not None:
params["timeout"] = timeout
# See complete(): no '{' prefill when tools are available, so
# tool_use blocks stay reachable on the streamed path too.
use_json_prefill = (

28
src/llm/backends/gemini.py
View File

@ -4,10 +4,16 @@ from collections.abc import AsyncIterator
from datetime import datetime, timedelta, timezone
from typing import Any, ClassVar, cast
from google.genai import types as genai_types
from pydantic import BaseModel
from src.exceptions import LLMError, ValidationException
from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult
from src.llm.backend import (
CompletionResult,
StreamChunk,
ToolCallResult,
request_timeout_from_extra_params,
)
from src.llm.caching import (
GeminiCacheHandle,
PromptCachePolicy,
@ -289,19 +295,35 @@ class GeminiBackend:
# extra_query has no SDK-level equivalent and is ignored. Shallow
# merge with operator-wins. Operators are responsible for not setting
# unknown fields that google-genai's validation will reject.
http_options: genai_types.HttpOptions | None = None
if extra_params:
operator_extra_body = extra_params.get("extra_body")
if operator_extra_body:
config.update(
coerce_passthrough_mapping("extra_body", operator_extra_body)
)
raw_http_options = config.get("http_options")
if isinstance(raw_http_options, genai_types.HttpOptions):
http_options = raw_http_options
elif isinstance(raw_http_options, dict):
http_options = genai_types.HttpOptions(**raw_http_options)
operator_extra_headers = extra_params.get("extra_headers")
if operator_extra_headers:
http_options = config.setdefault("http_options", {})
existing_headers = http_options.setdefault("headers", {})
if http_options is None:
http_options = genai_types.HttpOptions()
existing_headers = dict(http_options.headers or {})
existing_headers.update(
coerce_passthrough_mapping("extra_headers", operator_extra_headers)
)
http_options.headers = existing_headers
timeout = request_timeout_from_extra_params(extra_params)
if timeout is not None:
if http_options is None:
http_options = genai_types.HttpOptions()
http_options.timeout = timeout
if http_options is not None:
config["http_options"] = http_options
return config
def _normalize_response(

11
src/llm/backends/openai.py
View File

@ -10,7 +10,12 @@ from openai import BadRequestError, LengthFinishReasonError
from pydantic import BaseModel, ValidationError
from src.exceptions import ValidationException
from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult
from src.llm.backend import (
CompletionResult,
StreamChunk,
ToolCallResult,
request_timeout_from_extra_params,
)
from src.llm.request_builder import apply_sdk_passthroughs
from src.llm.structured_output import (
StructuredOutputError,
@ -397,6 +402,10 @@ class OpenAIBackend:
# if the operator supplies `extra_body.reasoning`, it replaces any
# value Honcho auto-injected above.
apply_sdk_passthroughs(params, extra_params)
timeout = request_timeout_from_extra_params(extra_params)
if timeout is not None:
params["timeout"] = timeout
return params
def _normalize_response(

17
src/llm/request_builder.py
View File

@ -14,7 +14,12 @@ from pydantic import BaseModel
from src.config import ModelConfig, PromptCachePolicy
from src.exceptions import ValidationException
from .backend import CompletionResult, ProviderBackend, StreamChunk
from .backend import (
CompletionResult,
ProviderBackend,
StreamChunk,
request_timeout_from_extra_params,
)
# Operator escape-hatch keys recognized inside ModelConfig.provider_params.
PASSTHROUGH_KEYS = ("extra_body", "extra_headers", "extra_query")
@ -99,6 +104,14 @@ def build_config_extra_params(config: ModelConfig) -> dict[str, Any]:
return extra_params
def _normalize_extra_params(extra_params: dict[str, Any]) -> dict[str, Any]:
"""Normalize shared extra params before they reach provider backends."""
timeout = request_timeout_from_extra_params(extra_params)
if timeout is None:
return extra_params
return {**extra_params, "timeout": timeout}
async def execute_completion(
backend: ProviderBackend,
config: ModelConfig,
@ -120,6 +133,7 @@ async def execute_completion(
**build_config_extra_params(config),
**(extra_params or {}),
}
merged_extra_params = _normalize_extra_params(merged_extra_params)
if cache_policy is not None:
merged_extra_params["cache_policy"] = cache_policy
@ -158,6 +172,7 @@ async def execute_stream(
**build_config_extra_params(config),
**(extra_params or {}),
}
merged_extra_params = _normalize_extra_params(merged_extra_params)
if cache_policy is not None:
merged_extra_params["cache_policy"] = cache_policy

80
tests/llm/test_backends/test_anthropic.py
View File

@ -430,6 +430,7 @@ async def test_anthropic_backend_stream_no_prefill_when_tools_present() -> None:
client = Mock()
client.messages.stream = Mock(return_value=_FakeStream())
backend = AnthropicBackend(client)
chunks = [
chunk
@ -449,3 +450,82 @@ async def test_anthropic_backend_stream_no_prefill_when_tools_present() -> None:
"If not responding with a tool call, respond with valid JSON"
in call["messages"][0]["content"]
)
@pytest.mark.asyncio
async def test_anthropic_backend_passes_timeout_to_completion_request() -> None:
"""Anthropic completion requests receive per-request provider timeout."""
client = Mock()
client.messages.create = AsyncMock(
return_value=SimpleNamespace(
content=[TextBlock(type="text", text="ok")],
usage=SimpleNamespace(
input_tokens=10,
output_tokens=5,
cache_creation_input_tokens=0,
cache_read_input_tokens=0,
),
stop_reason="end_turn",
)
)
backend = AnthropicBackend(client)
await backend.complete(
model="claude-haiku-4-5",
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
extra_params={"timeout": 45},
)
await_args = client.messages.create.await_args
if await_args is None:
raise AssertionError("Expected Anthropic create call")
assert await_args.kwargs["timeout"] == 45.0
@pytest.mark.asyncio
async def test_anthropic_backend_passes_timeout_to_stream_request() -> None:
"""Anthropic stream requests receive per-request provider timeout."""
class FakeStream:
"""Minimal async stream manager for Anthropic streaming tests."""
async def __aenter__(self):
"""Return the stream object used by the backend."""
return self
async def __aexit__(self, *_args):
"""Do not suppress stream errors."""
return False
def __aiter__(self):
"""Return the async iterator used by the backend."""
return self
async def __anext__(self):
"""End the fake stream immediately."""
raise StopAsyncIteration
async def get_final_message(self):
"""Return the final message required by the backend."""
return SimpleNamespace(
usage=SimpleNamespace(output_tokens=1),
stop_reason="end_turn",
)
client = Mock()
client.messages.stream = Mock(return_value=FakeStream())
backend = AnthropicBackend(client)
chunks = [
chunk
async for chunk in backend.stream(
model="claude-haiku-4-5",
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
extra_params={"timeout": "60"},
)
]
assert chunks[-1].is_done is True
assert client.messages.stream.call_args.kwargs["timeout"] == 60.0

36
tests/llm/test_backends/test_gemini.py
View File

@ -98,6 +98,40 @@ async def test_gemini_backend_maps_thinking_effort_to_thinking_level() -> None:
assert call["config"]["thinking_config"] == {"thinking_level": "low"}
@pytest.mark.asyncio
async def test_gemini_backend_maps_timeout_to_http_options() -> None:
"""Gemini requests receive provider timeout through config http_options."""
client = Mock()
client.aio.models.generate_content = AsyncMock(
return_value=SimpleNamespace(
candidates=[
SimpleNamespace(
finish_reason=SimpleNamespace(name="STOP"),
content=SimpleNamespace(parts=[SimpleNamespace(text="ok")]),
)
],
usage_metadata=SimpleNamespace(
prompt_token_count=12,
candidates_token_count=6,
),
parsed=None,
)
)
backend = GeminiBackend(client)
await backend.complete(
model="gemini-2.5-flash",
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
extra_params={"timeout": "90"},
)
await_args = client.aio.models.generate_content.await_args
if await_args is None:
raise AssertionError("Expected Gemini generate_content call")
assert await_args.kwargs["config"]["http_options"].timeout == 90.0
@pytest.mark.asyncio
async def test_gemini_backend_rejects_budget_and_effort_together() -> None:
backend = GeminiBackend(Mock())
@ -385,7 +419,7 @@ async def test_gemini_backend_forwards_provider_params_extra_headers() -> None:
if await_args is None:
raise AssertionError("Expected Gemini generate_content call")
call = await_args.kwargs
assert call["config"]["http_options"]["headers"] == {"X-Trace-Id": "abc123"}
assert call["config"]["http_options"].headers == {"X-Trace-Id": "abc123"}
@pytest.mark.asyncio

73
tests/llm/test_backends/test_openai.py
View File

@ -510,6 +510,7 @@ async def test_openai_backend_converts_anthropic_style_tools() -> None:
assert call["tool_choice"] == "required"
@pytest.mark.asyncio
async def test_openai_backend_translates_canonical_any_tool_choice_to_required() -> (
None
):
@ -571,6 +572,78 @@ def test_openai_convert_tool_choice(canonical: Any, expected: Any) -> None:
assert OpenAIBackend._convert_tool_choice(canonical) == expected # pyright: ignore[reportPrivateUsage]
@pytest.mark.asyncio
async def test_openai_backend_passes_timeout_to_completion_request() -> None:
"""OpenAI completion requests receive per-request provider timeout."""
client = Mock()
client.chat.completions.create = AsyncMock(
return_value=SimpleNamespace(
choices=[
SimpleNamespace(
finish_reason="stop",
message=SimpleNamespace(
content="ok",
tool_calls=[],
reasoning_details=[],
),
)
],
usage=SimpleNamespace(
prompt_tokens=10,
completion_tokens=5,
prompt_tokens_details=None,
),
)
)
backend = OpenAIBackend(client)
await backend.complete(
model="gpt-4.1",
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
extra_params={"timeout": 12.5},
)
assert _await_kwargs(client.chat.completions.create)["timeout"] == 12.5
@pytest.mark.asyncio
async def test_openai_backend_passes_timeout_to_structured_parse_request() -> None:
"""OpenAI structured parse requests receive per-request provider timeout."""
client = Mock()
client.chat.completions.parse = AsyncMock(
return_value=SimpleNamespace(
choices=[
SimpleNamespace(
finish_reason="stop",
message=SimpleNamespace(
parsed=_StructuredResponse(answer="ok"),
content='{"answer":"ok"}',
tool_calls=[],
refusal=None,
),
)
],
usage=SimpleNamespace(
prompt_tokens=10,
completion_tokens=5,
prompt_tokens_details=None,
),
)
)
backend = OpenAIBackend(client)
await backend.complete(
model="gpt-4.1",
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
response_format=_StructuredResponse,
extra_params={"timeout": "30"},
)
assert _await_kwargs(client.chat.completions.parse)["timeout"] == 30.0
@pytest.mark.parametrize(
"model",
[

50
tests/llm/test_request_builder.py
View File

@ -1,6 +1,8 @@
import pytest
from pydantic import BaseModel
from src.config import ModelConfig
from src.exceptions import ValidationException
from src.llm.caching import PromptCachePolicy
from src.llm.request_builder import execute_completion
from tests.llm.conftest import FakeBackend
@ -95,3 +97,51 @@ async def test_provider_params_are_merged_into_extra_params(
call = fake_backend.calls[0]
assert call["extra_params"]["top_p"] == 0.9
assert call["extra_params"]["custom_flag"] is True
async def test_provider_timeout_is_normalized_into_extra_params(
fake_backend: FakeBackend,
) -> None:
"""Numeric-string provider timeout values are normalized before backends."""
config = ModelConfig(
model="gpt-4.1-mini",
transport="openai",
provider_params={"timeout": "42.5"},
)
await execute_completion(
fake_backend,
config,
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
)
call = fake_backend.calls[0]
assert call["extra_params"]["timeout"] == 42.5
@pytest.mark.parametrize(
"timeout",
["slow", "", 0, -1, True, float("nan"), float("inf"), "nan", "inf"],
)
async def test_provider_timeout_rejects_invalid_values(
fake_backend: FakeBackend,
timeout: object,
) -> None:
"""Invalid provider timeout values fail before provider SDK calls."""
config = ModelConfig(
model="gpt-4.1-mini",
transport="openai",
provider_params={"timeout": timeout},
)
with pytest.raises(
ValidationException,
match=r"provider_params\.timeout must be a positive number of seconds",
):
await execute_completion(
fake_backend,
config,
messages=[{"role": "user", "content": "Hello"}],
max_tokens=100,
)