From 3186d2ce39fb7b7d0774a01beaed1d4799e32103 Mon Sep 17 00:00:00 2001 From: Benjamin McCormick Date: Wed, 29 Oct 2025 16:53:28 -0400 Subject: [PATCH] feat: add optional backup providers that kick in on retry --- src/config.py | 66 ++++++++++++++++++++- src/deriver/deriver.py | 6 +- src/dialectic/chat.py | 6 +- src/dreamer/dreamer.py | 3 +- src/utils/clients.py | 102 +++++++++++++++++++++++++++++---- src/utils/summarizer.py | 6 +- tests/bench/peer_card_bench.py | 6 +- tests/utils/test_clients.py | 11 ++-- 8 files changed, 175 insertions(+), 31 deletions(-) diff --git a/src/config.py b/src/config.py index 6458717e..681c9225 100644 --- a/src/config.py +++ b/src/config.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Annotated, Any, ClassVar, Literal +from typing import Annotated, Any, ClassVar, Literal, Protocol import tomllib from dotenv import load_dotenv @@ -40,6 +40,15 @@ def load_toml_config(config_path: str = "config.toml") -> dict[str, Any]: TOML_CONFIG = load_toml_config() +class LLMComponentSettings(Protocol): + """Protocol for settings classes that use LLM providers with backup support.""" + + PROVIDER: SupportedProviders + MODEL: str + BACKUP_PROVIDER: SupportedProviders | None + BACKUP_MODEL: str | None + + class TomlConfigSettingsSource(PydanticBaseSettingsSource): """Custom settings source for loading from TOML file.""" @@ -198,6 +207,8 @@ class DeriverSettings(HonchoSettings): PROVIDER: SupportedProviders = "google" MODEL: str = "gemini-2.5-flash-lite" + BACKUP_PROVIDER: SupportedProviders | None = None + BACKUP_MODEL: str | None = None MAX_OUTPUT_TOKENS: Annotated[int, Field(default=10_000, gt=0, le=100_000)] = 10_000 # Thinking budget tokens are only applied when using Anthropic as provider @@ -219,6 +230,15 @@ class DeriverSettings(HonchoSettings): MAX_INPUT_TOKENS: Annotated[int, Field(default=23000, gt=0, le=23000)] = 23000 + @model_validator(mode="after") + def validate_backup_configuration(self): + """Ensure both backup fields are set together or both are None.""" + if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None): + raise ValueError( + "BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None" + ) + return self + @model_validator(mode="after") def validate_batch_tokens_vs_context_limit(self): if self.REPRESENTATION_BATCH_MAX_TOKENS > self.MAX_INPUT_TOKENS: @@ -235,15 +255,28 @@ class PeerCardSettings(HonchoSettings): PROVIDER: SupportedProviders = "openai" MODEL: str = "gpt-5-nano-2025-08-07" + BACKUP_PROVIDER: SupportedProviders | None = None + BACKUP_MODEL: str | None = None # Note: peer cards should be very short, but GPT-5 models need output tokens for thinking which cannot be turned off... MAX_OUTPUT_TOKENS: Annotated[int, Field(default=4000, gt=1000, le=10_000)] = 4000 + @model_validator(mode="after") + def validate_backup_configuration(self): + """Ensure both backup fields are set together or both are None.""" + if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None): + raise ValueError( + "BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None" + ) + return self + class DialecticSettings(HonchoSettings): model_config = SettingsConfigDict(env_prefix="DIALECTIC_", extra="ignore") # pyright: ignore PROVIDER: SupportedProviders = "anthropic" MODEL: str = "claude-sonnet-4-20250514" + BACKUP_PROVIDER: SupportedProviders | None = None + BACKUP_MODEL: str | None = None PERFORM_QUERY_GENERATION: bool = False QUERY_GENERATION_PROVIDER: SupportedProviders = "groq" @@ -262,6 +295,15 @@ class DialecticSettings(HonchoSettings): int, Field(default=100_000, gt=10_000, le=200_000) ] = 100_000 + @model_validator(mode="after") + def validate_backup_configuration(self): + """Ensure both backup fields are set together or both are None.""" + if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None): + raise ValueError( + "BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None" + ) + return self + class SummarySettings(HonchoSettings): model_config = SettingsConfigDict(env_prefix="SUMMARY_", extra="ignore") # pyright: ignore @@ -273,11 +315,22 @@ class SummarySettings(HonchoSettings): PROVIDER: SupportedProviders = "openai" MODEL: str = "gpt-4o-mini-2024-07-18" + BACKUP_PROVIDER: SupportedProviders | None = None + BACKUP_MODEL: str | None = None MAX_TOKENS_SHORT: Annotated[int, Field(default=1000, gt=0, le=10_000)] = 1000 MAX_TOKENS_LONG: Annotated[int, Field(default=4000, gt=0, le=20_000)] = 4000 THINKING_BUDGET_TOKENS: Annotated[int, Field(default=512, gt=0, le=2000)] = 512 + @model_validator(mode="after") + def validate_backup_configuration(self): + """Ensure both backup fields are set together or both are None.""" + if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None): + raise ValueError( + "BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None" + ) + return self + class WebhookSettings(HonchoSettings): model_config = SettingsConfigDict(env_prefix="WEBHOOK_", extra="ignore") # pyright: ignore @@ -305,8 +358,19 @@ class DreamSettings(HonchoSettings): # LLM settings for dream processing PROVIDER: SupportedProviders = "openai" MODEL: str = "gpt-4o-mini-2024-07-18" + BACKUP_PROVIDER: SupportedProviders | None = None + BACKUP_MODEL: str | None = None MAX_OUTPUT_TOKENS: Annotated[int, Field(default=2000, gt=0, le=10_000)] = 2000 + @model_validator(mode="after") + def validate_backup_configuration(self): + """Ensure both backup fields are set together or both are None.""" + if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None): + raise ValueError( + "BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None" + ) + return self + class AppSettings(HonchoSettings): # No env_prefix for app-level settings diff --git a/src/deriver/deriver.py b/src/deriver/deriver.py index 5990a29b..701bbc50 100644 --- a/src/deriver/deriver.py +++ b/src/deriver/deriver.py @@ -53,8 +53,7 @@ async def critical_analysis_call( ) response = await honcho_llm_call( - provider=settings.DERIVER.PROVIDER, - model=settings.DERIVER.MODEL, + llm_settings=settings.DERIVER, prompt=prompt, max_tokens=settings.DERIVER.MAX_OUTPUT_TOKENS or settings.LLM.DEFAULT_MAX_TOKENS, @@ -90,8 +89,7 @@ async def peer_card_call( ) response = await honcho_llm_call( - provider=settings.PEER_CARD.PROVIDER, - model=settings.PEER_CARD.MODEL, + llm_settings=settings.PEER_CARD, prompt=prompt, max_tokens=settings.PEER_CARD.MAX_OUTPUT_TOKENS or settings.LLM.DEFAULT_MAX_TOKENS, diff --git a/src/dialectic/chat.py b/src/dialectic/chat.py index d90977e9..db76201c 100644 --- a/src/dialectic/chat.py +++ b/src/dialectic/chat.py @@ -72,8 +72,7 @@ async def dialectic_call( ) response = await honcho_llm_call( - provider=settings.DIALECTIC.PROVIDER, - model=settings.DIALECTIC.MODEL, + llm_settings=settings.DIALECTIC, prompt=prompt, max_tokens=settings.DIALECTIC.MAX_OUTPUT_TOKENS, track_name="Dialectic Call", @@ -128,8 +127,7 @@ async def dialectic_stream( ) response = await honcho_llm_call( - provider=settings.DIALECTIC.PROVIDER, - model=settings.DIALECTIC.MODEL, + llm_settings=settings.DIALECTIC, prompt=prompt, max_tokens=settings.DIALECTIC.MAX_OUTPUT_TOKENS, track_name="Dialectic Stream", diff --git a/src/dreamer/dreamer.py b/src/dreamer/dreamer.py index 9ee43bb8..19a5751e 100644 --- a/src/dreamer/dreamer.py +++ b/src/dreamer/dreamer.py @@ -184,8 +184,7 @@ async def consolidate_call( prompt = consolidation_prompt(representation) response = await honcho_llm_call( - provider=settings.DREAM.PROVIDER, - model=settings.DREAM.MODEL, + llm_settings=settings.DREAM, prompt=prompt, max_tokens=settings.DREAM.MAX_OUTPUT_TOKENS, track_name="Dream Call", diff --git a/src/utils/clients.py b/src/utils/clients.py index 9a0628ef..64588f2b 100644 --- a/src/utils/clients.py +++ b/src/utils/clients.py @@ -1,6 +1,7 @@ import json import logging from collections.abc import AsyncIterator +from contextvars import ContextVar from typing import Any, Generic, Literal, TypeVar, cast, overload from anthropic import AsyncAnthropic @@ -15,7 +16,7 @@ from pydantic import BaseModel, Field, ValidationError from sentry_sdk.ai.monitoring import ai_track from tenacity import retry, stop_after_attempt, wait_exponential -from src.config import settings +from src.config import LLMComponentSettings, settings from src.utils.json_parser import validate_and_repair_json from src.utils.logging import conditional_observe from src.utils.representation import PromptRepresentation @@ -26,6 +27,9 @@ logger = logging.getLogger(__name__) T = TypeVar("T") M = TypeVar("M", bound=BaseModel) +# Context variable to track retry attempts for provider switching +_current_attempt: ContextVar[int] = ContextVar("current_attempt", default=0) + CLIENTS: dict[ SupportedProviders, AsyncAnthropic | AsyncOpenAI | genai.Client | AsyncGroq, @@ -106,8 +110,7 @@ class HonchoLLMCallStreamChunk(BaseModel): @overload async def honcho_llm_call( - provider: SupportedProviders, - model: str, + llm_settings: LLMComponentSettings, prompt: str, max_tokens: int, track_name: str | None = None, @@ -127,8 +130,7 @@ async def honcho_llm_call( @overload async def honcho_llm_call( - provider: SupportedProviders, - model: str, + llm_settings: LLMComponentSettings, prompt: str, max_tokens: int, track_name: str | None = None, @@ -147,8 +149,7 @@ async def honcho_llm_call( @overload async def honcho_llm_call( - provider: SupportedProviders, - model: str, + llm_settings: LLMComponentSettings, prompt: str, max_tokens: int, track_name: str | None = None, @@ -167,8 +168,7 @@ async def honcho_llm_call( @conditional_observe(name="LLM Call") async def honcho_llm_call( - provider: SupportedProviders, - model: str, + llm_settings: LLMComponentSettings, prompt: str, max_tokens: int, track_name: str | None = None, @@ -183,6 +183,73 @@ async def honcho_llm_call( retry_attempts: int = 3, stream: bool = False, ) -> HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk]: + """ + Make an LLM call with automatic backup provider failover. + + Args: + llm_settings: Settings object containing PROVIDER, MODEL, + BACKUP_PROVIDER, and BACKUP_MODEL + prompt: The prompt to send to the LLM + max_tokens: Maximum tokens to generate + track_name: Optional name for AI tracking + response_model: Optional Pydantic model for structured output + json_mode: Whether to use JSON mode + stop_seqs: Stop sequences + reasoning_effort: OpenAI reasoning effort (GPT-5 only) + verbosity: OpenAI verbosity (GPT-5 only) + thinking_budget_tokens: Anthropic thinking budget + enable_retry: Whether to enable retry with exponential backoff + retry_attempts: Number of retry attempts + stream: Whether to stream the response + + Returns: + HonchoLLMCallResponse or AsyncIterator depending on stream parameter + + Raises: + ValueError: If provider is not configured + """ + # Extract provider/model from settings, with backup support + attempt = _current_attempt.get() + + # Use backup if: (1) we're retrying, (2) backup is configured, (3) backup client exists + use_backup = ( + attempt > 0 + and llm_settings.BACKUP_PROVIDER is not None + and llm_settings.BACKUP_MODEL is not None + and llm_settings.BACKUP_PROVIDER in CLIENTS + ) + + if ( + use_backup + and llm_settings.BACKUP_PROVIDER is not None + and llm_settings.BACKUP_MODEL is not None + ): + provider = llm_settings.BACKUP_PROVIDER + model = llm_settings.BACKUP_MODEL + logger.info( + f"Retry attempt {attempt}: switching from " + + f"{llm_settings.PROVIDER}/{llm_settings.MODEL} to " + + f"backup {provider}/{model}" + ) + + # Filter out incompatible parameters when using backup + if provider != "anthropic" and thinking_budget_tokens: + logger.warning( + f"thinking_budget_tokens not supported by {provider}, ignoring" + ) + thinking_budget_tokens = None + + if model and "gpt-5" not in model and (reasoning_effort or verbosity): + logger.warning( + "reasoning_effort/verbosity only supported by GPT-5 models, ignoring" + ) + reasoning_effort = None + verbosity = None + else: + provider = llm_settings.PROVIDER + model = llm_settings.MODEL + + # Validate client exists client = CLIENTS.get(provider) if not client: raise ValueError(f"Missing client for {provider}") @@ -193,13 +260,28 @@ async def honcho_llm_call( if track_name: decorated = ai_track(track_name)(decorated) - # apply retry logic + # apply retry logic - retries on ANY exception if enable_retry: + + def before_retry_callback(retry_state: Any) -> None: + """Update attempt counter before each retry.""" + _current_attempt.set(retry_state.attempt_number) + exc = retry_state.outcome.exception() if retry_state.outcome else None + if exc: + logger.warning( + f"Error on attempt {retry_state.attempt_number} with " + + f"{llm_settings.PROVIDER}/{llm_settings.MODEL}: {exc}" + ) + decorated = retry( stop=stop_after_attempt(retry_attempts), wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=before_retry_callback, )(decorated) + # Reset attempt counter at start + _current_attempt.set(0) + if stream: return await decorated( provider, diff --git a/src/utils/summarizer.py b/src/utils/summarizer.py index 065e3f99..569a5e8f 100644 --- a/src/utils/summarizer.py +++ b/src/utils/summarizer.py @@ -123,8 +123,7 @@ Produce as thorough a summary as possible in {output_words} words or less. """) return await honcho_llm_call( - provider=settings.SUMMARY.PROVIDER, - model=settings.SUMMARY.MODEL, + llm_settings=settings.SUMMARY, prompt=prompt, max_tokens=settings.SUMMARY.MAX_TOKENS_SHORT, ) @@ -172,8 +171,7 @@ Produce as thorough a summary as possible in {output_words} words or less. """) return await honcho_llm_call( - provider=settings.SUMMARY.PROVIDER, - model=settings.SUMMARY.MODEL, + llm_settings=settings.SUMMARY, prompt=prompt, max_tokens=settings.SUMMARY.MAX_TOKENS_LONG, ) diff --git a/tests/bench/peer_card_bench.py b/tests/bench/peer_card_bench.py index 8119a795..8878ebb0 100644 --- a/tests/bench/peer_card_bench.py +++ b/tests/bench/peer_card_bench.py @@ -143,6 +143,9 @@ def build_peer_card_caller( "openai" if candidate.provider == "custom" else candidate.provider ) + settings.PEER_CARD.PROVIDER = cast(Any, resolved_provider) + settings.PEER_CARD.MODEL = candidate.model + async def call( old_peer_card: list[str] | None, new_observations: Representation ) -> PeerCardQuery: @@ -152,8 +155,7 @@ def build_peer_card_caller( ) response = await honcho_llm_call( - provider=cast(Any, resolved_provider), - model=candidate.model, + llm_settings=settings.PEER_CARD, prompt=prompt, max_tokens=settings.PEER_CARD.MAX_OUTPUT_TOKENS, response_model=PeerCardQuery, diff --git a/tests/utils/test_clients.py b/tests/utils/test_clients.py index 54909d8d..f4cb9073 100644 --- a/tests/utils/test_clients.py +++ b/tests/utils/test_clients.py @@ -25,6 +25,7 @@ from openai.types.chat.chat_completion_message import ChatCompletionMessage from openai.types.completion_usage import CompletionUsage from pydantic import BaseModel, Field +from src.config import settings from src.utils.clients import ( CLIENTS, HonchoLLMCallResponse, @@ -962,10 +963,11 @@ class TestMainLLMCallFunction: mock_client.messages.stream.return_value = mock_stream with patch.dict(CLIENTS, {"anthropic": mock_client}): + settings.DIALECTIC.PROVIDER = "anthropic" + settings.DIALECTIC.MODEL = "claude-4-sonnet" chunks: list[HonchoLLMCallStreamChunk] = [] async for chunk in await honcho_llm_call( - provider="anthropic", - model="claude-3-sonnet", + llm_settings=settings.DIALECTIC, prompt="Hello", max_tokens=100, stream=True, @@ -990,9 +992,10 @@ class TestMainLLMCallFunction: mock_client.messages.create = AsyncMock(return_value=mock_response) with patch.dict(CLIENTS, {"anthropic": mock_client}): + settings.DIALECTIC.PROVIDER = "anthropic" + settings.DIALECTIC.MODEL = "claude-4-sonnet" response = await honcho_llm_call( - provider="anthropic", - model="claude-3-sonnet", + llm_settings=settings.DIALECTIC, prompt="Hello", max_tokens=100, enable_retry=False,