diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index 267e6814..06add0d0 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -185,6 +185,33 @@ Each model config supports an `overrides.provider_params` dict for passing arbit verbosity = "low" ``` +#### Transport passthrough keys + +Three keys inside `provider_params` are recognized as request-level escape hatches and forwarded to the underlying transport. Where a transport actually validates and merges one of these keys, its value must be a mapping — a non-mapping value raises a configuration error (see the per-transport behavior below; a key a transport ignores is not validated): + +- **`extra_body`** — merged into the request body +- **`extra_headers`** — extra HTTP headers +- **`extra_query`** — extra URL query parameters + +How each transport forwards them differs: + +- **OpenAI and Anthropic** forward all three as identically-named SDK kwargs (`extra_body`, `extra_headers`, `extra_query`). +- **Gemini** has no SDK kwargs for these. It merges `extra_body` into the `GenerateContentConfig` dict and folds `extra_headers` into `http_options.headers`; `extra_query` is **unsupported and silently ignored**. + +The merge is shallow and **operator-wins**: if Honcho and your config both set the same top-level key inside `extra_body`, your value replaces Honcho's. You are responsible for choosing a coherent combination — e.g. unset `thinking_budget_tokens` when supplying an `extra_body.thinking` for Anthropic-via-proxy, since Honcho will not translate between the two shapes. + +Because Gemini merges `extra_body` directly into `GenerateContentConfig` (rather than a nested request body), an `extra_body` written for OpenAI/Anthropic generally will not transfer to Gemini unchanged — and a key collision there can overwrite a field Honcho manages (`thinking_config`, `response_schema`, `tools`, …). + +```toml +# Example: route an OpenAI-compatible proxy and tag requests for tracing +[deriver.model_config.overrides.provider_params.extra_headers] +X-Proxy-Route = "vertex" + +[deriver.model_config.overrides.provider_params.extra_body] +# Provider-native body fields the standard config doesn't expose +anthropic_beta = ["context-1m-2025-01-15"] +``` + ### Changing Transport When changing a feature's `transport`, always specify `model` explicitly. Partial overrides that change transport without model will keep the previous model name, which may not be valid for the new provider. diff --git a/src/config.py b/src/config.py index 6a7869b6..267a8154 100644 --- a/src/config.py +++ b/src/config.py @@ -69,7 +69,23 @@ class ModelOverrideSettings(BaseModel): api_key_env: str | None = None base_url: str | None = None - provider_params: dict[str, Any] = Field(default_factory=dict) + provider_params: dict[str, Any] = Field( + default_factory=dict, + description=( + "Operator escape hatch for provider-specific request fields. " + "Three recognized keys: `extra_body` (merged into the request body), " + "`extra_headers` (HTTP headers), `extra_query` (URL query params). " + "OpenAI and Anthropic transports forward these as identically-named " + "SDK kwargs. The Gemini transport merges `extra_body` into the " + "GenerateContentConfig dict and folds `extra_headers` into " + "`http_options.headers`; `extra_query` is unsupported. Shallow merge " + "with operator-wins — if Honcho and the operator both set the same " + "key inside `extra_body`, the operator's value replaces Honcho's. " + "Operators are responsible for picking a coherent combination of " + "this and other config (e.g. unset `thinking_budget_tokens` when " + "supplying an `extra_body.thinking` for Anthropic-via-proxy)." + ), + ) class PromptCachePolicy(BaseModel): diff --git a/src/llm/backends/anthropic.py b/src/llm/backends/anthropic.py index 17138583..673f4d03 100644 --- a/src/llm/backends/anthropic.py +++ b/src/llm/backends/anthropic.py @@ -9,6 +9,7 @@ from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock from pydantic import BaseModel, ValidationError from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.request_builder import apply_sdk_passthroughs from src.llm.structured_output import repair_response_model_json @@ -69,6 +70,9 @@ class AnthropicBackend: for key in ("top_p", "top_k"): if key in extra_params: params[key] = extra_params[key] + # Operator escape hatch: forward Anthropic SDK passthrough kwargs + # from ModelConfig.provider_params. Shallow merge with operator-wins. + apply_sdk_passthroughs(params, extra_params) use_json_prefill = ( bool(response_format or self._json_mode(extra_params)) @@ -148,6 +152,9 @@ class AnthropicBackend: for key in ("top_p", "top_k"): if key in extra_params: params[key] = extra_params[key] + # Operator escape hatch: forward Anthropic SDK passthrough kwargs + # from ModelConfig.provider_params. Shallow merge with operator-wins. + apply_sdk_passthroughs(params, extra_params) use_json_prefill = ( bool(response_format or is_json_mode) and not thinking_budget_tokens diff --git a/src/llm/backends/gemini.py b/src/llm/backends/gemini.py index b14cefe4..5c70268a 100644 --- a/src/llm/backends/gemini.py +++ b/src/llm/backends/gemini.py @@ -14,6 +14,7 @@ from src.llm.caching import ( build_cache_key, gemini_cache_store, ) +from src.llm.request_builder import coerce_passthrough_mapping from src.llm.structured_output import repair_response_model_json GEMINI_BLOCKED_FINISH_REASONS = { @@ -246,6 +247,26 @@ class GeminiBackend: for key in ("top_p", "top_k", "frequency_penalty", "presence_penalty", "seed"): if extra_params and key in extra_params: config[key] = extra_params[key] + # Operator escape hatch: forward provider_params into the google-genai + # config dict. The Gemini SDK doesn't expose extra_body/extra_headers + # as kwargs (unlike OpenAI/Anthropic) — body-shaped fields live on + # GenerateContentConfig and headers live under config.http_options. + # extra_query has no SDK-level equivalent and is ignored. Shallow + # merge with operator-wins. Operators are responsible for not setting + # unknown fields that google-genai's validation will reject. + if extra_params: + operator_extra_body = extra_params.get("extra_body") + if operator_extra_body: + config.update( + coerce_passthrough_mapping("extra_body", operator_extra_body) + ) + operator_extra_headers = extra_params.get("extra_headers") + if operator_extra_headers: + http_options = config.setdefault("http_options", {}) + existing_headers = http_options.setdefault("headers", {}) + existing_headers.update( + coerce_passthrough_mapping("extra_headers", operator_extra_headers) + ) return config def _normalize_response( diff --git a/src/llm/backends/openai.py b/src/llm/backends/openai.py index b2d82d91..fe8962d5 100644 --- a/src/llm/backends/openai.py +++ b/src/llm/backends/openai.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ValidationError from src.exceptions import ValidationException from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.request_builder import apply_sdk_passthroughs from src.llm.structured_output import ( repair_response_model_json, validate_structured_output, @@ -300,8 +301,10 @@ class OpenAIBackend: # Token-budget style thinking is not part of the native OpenAI API, but # OpenAI-compatible proxies (OpenRouter, etc.) accept a `reasoning` object # on the request body. Pass through via extra_body so it reaches those - # backends; operators on providers that need a different shape (vLLM, - # Fireworks, ...) can override via ModelConfig.provider_params. + # backends. Operators on providers that need a different shape (e.g. + # Anthropic-via-Vertex behind litellm wants `thinking`, not `reasoning`) + # supply that shape via ModelConfig.provider_params.extra_body and unset + # thinking_budget_tokens themselves — Honcho does not try to translate. if thinking_budget_tokens is not None and thinking_budget_tokens > 0: params.setdefault("extra_body", {}).setdefault("reasoning", {})[ "max_tokens" @@ -322,6 +325,11 @@ class OpenAIBackend: ): if key in extra_params: params[key] = extra_params[key] + # Operator escape hatch: forward OpenAI SDK passthrough kwargs from + # ModelConfig.provider_params. Shallow merge with operator-wins — + # if the operator supplies `extra_body.reasoning`, it replaces any + # value Honcho auto-injected above. + apply_sdk_passthroughs(params, extra_params) return params def _normalize_response( diff --git a/src/llm/request_builder.py b/src/llm/request_builder.py index d6be5a22..fd8d69c3 100644 --- a/src/llm/request_builder.py +++ b/src/llm/request_builder.py @@ -7,14 +7,70 @@ src/llm/api.py, src/llm/tool_loop.py, src/llm/runtime.py. from __future__ import annotations from collections.abc import AsyncIterator -from typing import Any +from typing import Any, cast from pydantic import BaseModel from src.config import ModelConfig, PromptCachePolicy +from src.exceptions import ValidationException from .backend import CompletionResult, ProviderBackend, StreamChunk +# Operator escape-hatch keys recognized inside ModelConfig.provider_params. +PASSTHROUGH_KEYS = ("extra_body", "extra_headers", "extra_query") + + +def coerce_passthrough_mapping(key: str, value: Any) -> dict[str, Any]: + """Validate an operator-supplied provider_params passthrough is a mapping. + + ``provider_params`` is typed ``dict[str, Any]`` with no nested schema, so an + operator can supply a non-mapping (e.g. a list or string) for one of the + passthrough keys. Catch that here with a clear error instead of letting a + later ``dict.update()`` raise an opaque ``TypeError`` deep in the transport. + + Args: + key: The passthrough key name, used only for the error message. + value: The operator-supplied value to validate. + + Returns: + The value, narrowed to ``dict[str, Any]``. + + Raises: + ValidationException: If ``value`` is not a mapping. + """ + if not isinstance(value, dict): + raise ValidationException( + f"provider_params.{key} must be a mapping, got {type(value).__name__}" + ) + return cast(dict[str, Any], value) + + +def apply_sdk_passthroughs( + params: dict[str, Any], extra_params: dict[str, Any] +) -> None: + """Forward operator provider_params passthroughs onto an SDK call dict. + + OpenAI and Anthropic both accept ``extra_body`` / ``extra_headers`` / + ``extra_query`` as identically-named SDK kwargs, so they share this merge. + Operator values shallow-merge onto ``params`` in place, winning over any + value Honcho already set under the same top-level key (e.g. an auto-injected + ``extra_body.reasoning``). Gemini handles passthroughs separately because the + google-genai SDK does not expose these as kwargs. + + Args: + params: The SDK call kwargs being assembled; mutated in place. + extra_params: Flattened per-call params (see build_config_extra_params). + + Raises: + ValidationException: If a passthrough value is not a mapping. + """ + for passthrough_key in PASSTHROUGH_KEYS: + operator_value = extra_params.get(passthrough_key) + if not operator_value: + continue + existing = params.setdefault(passthrough_key, {}) + existing.update(coerce_passthrough_mapping(passthrough_key, operator_value)) + def build_config_extra_params(config: ModelConfig) -> dict[str, Any]: """Flatten ModelConfig's optional knobs and provider_params into extra_params. diff --git a/tests/llm/test_backends/test_anthropic.py b/tests/llm/test_backends/test_anthropic.py index 52de0fa2..c7a4bb75 100644 --- a/tests/llm/test_backends/test_anthropic.py +++ b/tests/llm/test_backends/test_anthropic.py @@ -122,6 +122,99 @@ async def test_anthropic_backend_skips_assistant_prefill_for_claude_4_models() - assert call["messages"][0]["content"].startswith("Hello\n\nRespond with valid JSON") +@pytest.mark.asyncio +async def test_anthropic_backend_forwards_provider_params_passthroughs() -> None: + """provider_params.extra_body/extra_headers/extra_query reach the Anthropic + SDK call as kwargs of the same name (the SDK's documented passthrough). + """ + client = Mock() + client.messages.create = AsyncMock( + return_value=SimpleNamespace( + content=[TextBlock(type="text", text="ok")], + usage=SimpleNamespace( + input_tokens=10, + output_tokens=5, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ), + stop_reason="end_turn", + ) + ) + + backend = AnthropicBackend(client) + await backend.complete( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={ + "extra_body": {"anthropic_beta": ["context-1m-2025-01-15"]}, + "extra_headers": {"X-Proxy-Route": "vertex"}, + "extra_query": {"trace_id": "abc123"}, + }, + ) + + await_args = client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic client call") + call = await_args.kwargs + assert call["extra_body"] == {"anthropic_beta": ["context-1m-2025-01-15"]} + assert call["extra_headers"] == {"X-Proxy-Route": "vertex"} + assert call["extra_query"] == {"trace_id": "abc123"} + + +@pytest.mark.asyncio +async def test_anthropic_backend_stream_forwards_provider_params_passthroughs() -> None: + """The stream() path forwards provider_params passthroughs to the SDK the + same way complete() does — it has its own merge block, so cover it too. + """ + + class _FakeStream: + async def __aenter__(self) -> "_FakeStream": + return self + + async def __aexit__(self, *_: object) -> bool: + return False + + def __aiter__(self) -> "_FakeStream": + return self + + async def __anext__(self) -> object: + raise StopAsyncIteration + + async def get_final_message(self) -> SimpleNamespace: + return SimpleNamespace( + usage=SimpleNamespace(output_tokens=5), + stop_reason="end_turn", + ) + + client = Mock() + client.messages.stream = Mock(return_value=_FakeStream()) + + backend = AnthropicBackend(client) + chunks = [ + chunk + async for chunk in backend.stream( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={ + "extra_body": {"anthropic_beta": ["context-1m-2025-01-15"]}, + "extra_headers": {"X-Proxy-Route": "vertex"}, + "extra_query": {"trace_id": "abc123"}, + }, + ) + ] + + assert chunks # the terminal is_done chunk is always emitted + call = client.messages.stream.call_args + if call is None: + raise AssertionError("Expected Anthropic stream call") + kwargs = call.kwargs + assert kwargs["extra_body"] == {"anthropic_beta": ["context-1m-2025-01-15"]} + assert kwargs["extra_headers"] == {"X-Proxy-Route": "vertex"} + assert kwargs["extra_query"] == {"trace_id": "abc123"} + + @pytest.mark.asyncio async def test_anthropic_backend_ignores_thinking_effort() -> None: client = Mock() diff --git a/tests/llm/test_backends/test_gemini.py b/tests/llm/test_backends/test_gemini.py index b327c8e4..ca4fce97 100644 --- a/tests/llm/test_backends/test_gemini.py +++ b/tests/llm/test_backends/test_gemini.py @@ -315,6 +315,119 @@ async def test_gemini_backend_strips_system_and_tools_when_using_cached_content( assert "tool_config" not in call["config"] +@pytest.mark.asyncio +async def test_gemini_backend_forwards_provider_params_extra_body() -> None: + """provider_params.extra_body merges into the GenerateContentConfig dict + (Gemini's body-shaped fields live there, not as an SDK kwarg). + """ + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=[SimpleNamespace(text="ok")]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"extra_body": {"candidate_count": 3, "seed": 42}}, + ) + + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + call = await_args.kwargs + assert call["config"]["candidate_count"] == 3 + assert call["config"]["seed"] == 42 + + +@pytest.mark.asyncio +async def test_gemini_backend_forwards_provider_params_extra_headers() -> None: + """provider_params.extra_headers folds into config.http_options.headers.""" + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=[SimpleNamespace(text="ok")]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"extra_headers": {"X-Trace-Id": "abc123"}}, + ) + + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + call = await_args.kwargs + assert call["config"]["http_options"]["headers"] == {"X-Trace-Id": "abc123"} + + +@pytest.mark.asyncio +async def test_gemini_backend_silently_ignores_extra_query() -> None: + """extra_query has no google-genai SDK equivalent. The backend drops it + rather than crashing or surfacing it somewhere unexpected. + """ + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=[SimpleNamespace(text="ok")]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"extra_query": {"trace_id": "abc123"}}, + ) + + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + call = await_args.kwargs + # extra_query is not surfaced anywhere in the request — neither at the top + # level of the SDK call nor inside config. + assert "extra_query" not in call + assert "extra_query" not in call["config"] + + def test_gemini_sanitize_schema_strips_unsupported_keywords() -> None: """Gemini's function-declarations validator rejects JSON-Schema keywords outside its narrow allowlist (additionalProperties, allOf, if/then, $ref, diff --git a/tests/llm/test_backends/test_openai.py b/tests/llm/test_backends/test_openai.py index 695b12cd..dd0c7808 100644 --- a/tests/llm/test_backends/test_openai.py +++ b/tests/llm/test_backends/test_openai.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, Mock import pytest +from src.exceptions import ValidationException from src.llm.backends.openai import OpenAIBackend @@ -227,6 +228,164 @@ async def test_openai_backend_skips_extra_body_when_thinking_budget_zero() -> No assert "extra_body" not in call +@pytest.mark.asyncio +async def test_openai_backend_forwards_provider_params_extra_body() -> None: + """Operator-supplied extra_body in provider_params reaches the OpenAI SDK call. + + This is the escape hatch for OpenAI-compatible proxies that translate to + other providers (litellm → Vertex AI Anthropic) and need provider-native + body fields (e.g. Anthropic's `thinking`). + """ + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="ok", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={ + "extra_body": {"thinking": {"type": "enabled", "budget_tokens": 4096}} + }, + ) + + await_args = client.chat.completions.create.await_args + if await_args is None: + raise AssertionError("Expected OpenAI create call") + call = await_args.kwargs + assert call["extra_body"] == { + "thinking": {"type": "enabled", "budget_tokens": 4096} + } + + +@pytest.mark.asyncio +async def test_openai_backend_forwards_provider_params_extra_headers_and_query() -> ( + None +): + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="ok", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={ + "extra_headers": {"X-Proxy-Route": "vertex"}, + "extra_query": {"trace_id": "abc123"}, + }, + ) + + await_args = client.chat.completions.create.await_args + if await_args is None: + raise AssertionError("Expected OpenAI create call") + call = await_args.kwargs + assert call["extra_headers"] == {"X-Proxy-Route": "vertex"} + assert call["extra_query"] == {"trace_id": "abc123"} + + +@pytest.mark.asyncio +async def test_openai_backend_operator_extra_body_wins_over_auto_injection() -> None: + """When the operator supplies extra_body.reasoning, it must replace the + value that thinking_budget_tokens would otherwise auto-inject (operator-wins + shallow merge). + """ + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="ok", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="x-ai/grok-4.1-fast", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + thinking_budget_tokens=256, + extra_params={ + "extra_body": {"reasoning": {"effort": "high", "max_tokens": 9999}} + }, + ) + + await_args = client.chat.completions.create.await_args + if await_args is None: + raise AssertionError("Expected OpenAI create call") + call = await_args.kwargs + # Operator's whole `reasoning` dict replaces Honcho's auto-injected one. + assert call["extra_body"] == {"reasoning": {"effort": "high", "max_tokens": 9999}} + + +@pytest.mark.asyncio +async def test_openai_backend_rejects_non_mapping_passthrough() -> None: + """A non-mapping passthrough (operator misconfiguration) raises a clear + ValidationException instead of an opaque TypeError deep in the transport. + """ + client = Mock() + client.chat.completions.create = AsyncMock() + + backend = OpenAIBackend(client) + with pytest.raises(ValidationException, match=r"provider_params\.extra_headers"): + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + extra_params={"extra_headers": [["X-Foo", "bar"]]}, + ) + + client.chat.completions.create.assert_not_awaited() + + @pytest.mark.asyncio async def test_openai_backend_converts_anthropic_style_tools() -> None: client = Mock()