RomanNum3ral
/
honcho
mirror of https://github.com/plastic-labs/honcho.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add optional backup providers that kick in on retry

This commit is contained in:
Benjamin McCormick 2025-10-29 16:53:28 -04:00
parent a90156e113
commit 3186d2ce39
8 changed files with 175 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
src/config.py
View File

@ -1,6 +1,6 @@
import logging
from pathlib import Path
from typing import Annotated, Any, ClassVar, Literal
from typing import Annotated, Any, ClassVar, Literal, Protocol
import tomllib
from dotenv import load_dotenv
@ -40,6 +40,15 @@ def load_toml_config(config_path: str = "config.toml") -> dict[str, Any]:
TOML_CONFIG = load_toml_config()
class LLMComponentSettings(Protocol):
"""Protocol for settings classes that use LLM providers with backup support."""
PROVIDER: SupportedProviders
MODEL: str
BACKUP_PROVIDER: SupportedProviders | None
BACKUP_MODEL: str | None
class TomlConfigSettingsSource(PydanticBaseSettingsSource):
"""Custom settings source for loading from TOML file."""
@ -198,6 +207,8 @@ class DeriverSettings(HonchoSettings):
PROVIDER: SupportedProviders = "google"
MODEL: str = "gemini-2.5-flash-lite"
BACKUP_PROVIDER: SupportedProviders | None = None
BACKUP_MODEL: str | None = None
MAX_OUTPUT_TOKENS: Annotated[int, Field(default=10_000, gt=0, le=100_000)] = 10_000
# Thinking budget tokens are only applied when using Anthropic as provider
@ -219,6 +230,15 @@ class DeriverSettings(HonchoSettings):
MAX_INPUT_TOKENS: Annotated[int, Field(default=23000, gt=0, le=23000)] = 23000
@model_validator(mode="after")
def validate_backup_configuration(self):
"""Ensure both backup fields are set together or both are None."""
if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None):
raise ValueError(
"BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None"
)
return self
@model_validator(mode="after")
def validate_batch_tokens_vs_context_limit(self):
if self.REPRESENTATION_BATCH_MAX_TOKENS > self.MAX_INPUT_TOKENS:
@ -235,15 +255,28 @@ class PeerCardSettings(HonchoSettings):
PROVIDER: SupportedProviders = "openai"
MODEL: str = "gpt-5-nano-2025-08-07"
BACKUP_PROVIDER: SupportedProviders | None = None
BACKUP_MODEL: str | None = None
# Note: peer cards should be very short, but GPT-5 models need output tokens for thinking which cannot be turned off...
MAX_OUTPUT_TOKENS: Annotated[int, Field(default=4000, gt=1000, le=10_000)] = 4000
@model_validator(mode="after")
def validate_backup_configuration(self):
"""Ensure both backup fields are set together or both are None."""
if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None):
raise ValueError(
"BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None"
)
return self
class DialecticSettings(HonchoSettings):
model_config = SettingsConfigDict(env_prefix="DIALECTIC_", extra="ignore") # pyright: ignore
PROVIDER: SupportedProviders = "anthropic"
MODEL: str = "claude-sonnet-4-20250514"
BACKUP_PROVIDER: SupportedProviders | None = None
BACKUP_MODEL: str | None = None
PERFORM_QUERY_GENERATION: bool = False
QUERY_GENERATION_PROVIDER: SupportedProviders = "groq"
@ -262,6 +295,15 @@ class DialecticSettings(HonchoSettings):
int, Field(default=100_000, gt=10_000, le=200_000)
] = 100_000
@model_validator(mode="after")
def validate_backup_configuration(self):
"""Ensure both backup fields are set together or both are None."""
if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None):
raise ValueError(
"BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None"
)
return self
class SummarySettings(HonchoSettings):
model_config = SettingsConfigDict(env_prefix="SUMMARY_", extra="ignore") # pyright: ignore
@ -273,11 +315,22 @@ class SummarySettings(HonchoSettings):
PROVIDER: SupportedProviders = "openai"
MODEL: str = "gpt-4o-mini-2024-07-18"
BACKUP_PROVIDER: SupportedProviders | None = None
BACKUP_MODEL: str | None = None
MAX_TOKENS_SHORT: Annotated[int, Field(default=1000, gt=0, le=10_000)] = 1000
MAX_TOKENS_LONG: Annotated[int, Field(default=4000, gt=0, le=20_000)] = 4000
THINKING_BUDGET_TOKENS: Annotated[int, Field(default=512, gt=0, le=2000)] = 512
@model_validator(mode="after")
def validate_backup_configuration(self):
"""Ensure both backup fields are set together or both are None."""
if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None):
raise ValueError(
"BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None"
)
return self
class WebhookSettings(HonchoSettings):
model_config = SettingsConfigDict(env_prefix="WEBHOOK_", extra="ignore") # pyright: ignore
@ -305,8 +358,19 @@ class DreamSettings(HonchoSettings):
# LLM settings for dream processing
PROVIDER: SupportedProviders = "openai"
MODEL: str = "gpt-4o-mini-2024-07-18"
BACKUP_PROVIDER: SupportedProviders | None = None
BACKUP_MODEL: str | None = None
MAX_OUTPUT_TOKENS: Annotated[int, Field(default=2000, gt=0, le=10_000)] = 2000
@model_validator(mode="after")
def validate_backup_configuration(self):
"""Ensure both backup fields are set together or both are None."""
if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None):
raise ValueError(
"BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None"
)
return self
class AppSettings(HonchoSettings):
# No env_prefix for app-level settings

6
src/deriver/deriver.py
View File

@ -53,8 +53,7 @@ async def critical_analysis_call(
)
response = await honcho_llm_call(
provider=settings.DERIVER.PROVIDER,
model=settings.DERIVER.MODEL,
llm_settings=settings.DERIVER,
prompt=prompt,
max_tokens=settings.DERIVER.MAX_OUTPUT_TOKENS
or settings.LLM.DEFAULT_MAX_TOKENS,
@ -90,8 +89,7 @@ async def peer_card_call(
)
response = await honcho_llm_call(
provider=settings.PEER_CARD.PROVIDER,
model=settings.PEER_CARD.MODEL,
llm_settings=settings.PEER_CARD,
prompt=prompt,
max_tokens=settings.PEER_CARD.MAX_OUTPUT_TOKENS
or settings.LLM.DEFAULT_MAX_TOKENS,

6
src/dialectic/chat.py
View File

@ -72,8 +72,7 @@ async def dialectic_call(
)
response = await honcho_llm_call(
provider=settings.DIALECTIC.PROVIDER,
model=settings.DIALECTIC.MODEL,
llm_settings=settings.DIALECTIC,
prompt=prompt,
max_tokens=settings.DIALECTIC.MAX_OUTPUT_TOKENS,
track_name="Dialectic Call",
@ -128,8 +127,7 @@ async def dialectic_stream(
)
response = await honcho_llm_call(
provider=settings.DIALECTIC.PROVIDER,
model=settings.DIALECTIC.MODEL,
llm_settings=settings.DIALECTIC,
prompt=prompt,
max_tokens=settings.DIALECTIC.MAX_OUTPUT_TOKENS,
track_name="Dialectic Stream",

3
src/dreamer/dreamer.py
View File

@ -184,8 +184,7 @@ async def consolidate_call(
prompt = consolidation_prompt(representation)
response = await honcho_llm_call(
provider=settings.DREAM.PROVIDER,
model=settings.DREAM.MODEL,
llm_settings=settings.DREAM,
prompt=prompt,
max_tokens=settings.DREAM.MAX_OUTPUT_TOKENS,
track_name="Dream Call",

102
src/utils/clients.py
View File

@ -1,6 +1,7 @@
import json
import logging
from collections.abc import AsyncIterator
from contextvars import ContextVar
from typing import Any, Generic, Literal, TypeVar, cast, overload
from anthropic import AsyncAnthropic
@ -15,7 +16,7 @@ from pydantic import BaseModel, Field, ValidationError
from sentry_sdk.ai.monitoring import ai_track
from tenacity import retry, stop_after_attempt, wait_exponential
from src.config import settings
from src.config import LLMComponentSettings, settings
from src.utils.json_parser import validate_and_repair_json
from src.utils.logging import conditional_observe
from src.utils.representation import PromptRepresentation
@ -26,6 +27,9 @@ logger = logging.getLogger(__name__)
T = TypeVar("T")
M = TypeVar("M", bound=BaseModel)
# Context variable to track retry attempts for provider switching
_current_attempt: ContextVar[int] = ContextVar("current_attempt", default=0)
CLIENTS: dict[
SupportedProviders,
AsyncAnthropic | AsyncOpenAI | genai.Client | AsyncGroq,
@ -106,8 +110,7 @@ class HonchoLLMCallStreamChunk(BaseModel):
@overload
async def honcho_llm_call(
provider: SupportedProviders,
model: str,
llm_settings: LLMComponentSettings,
prompt: str,
max_tokens: int,
track_name: str | None = None,
@ -127,8 +130,7 @@ async def honcho_llm_call(
@overload
async def honcho_llm_call(
provider: SupportedProviders,
model: str,
llm_settings: LLMComponentSettings,
prompt: str,
max_tokens: int,
track_name: str | None = None,
@ -147,8 +149,7 @@ async def honcho_llm_call(
@overload
async def honcho_llm_call(
provider: SupportedProviders,
model: str,
llm_settings: LLMComponentSettings,
prompt: str,
max_tokens: int,
track_name: str | None = None,
@ -167,8 +168,7 @@ async def honcho_llm_call(
@conditional_observe(name="LLM Call")
async def honcho_llm_call(
provider: SupportedProviders,
model: str,
llm_settings: LLMComponentSettings,
prompt: str,
max_tokens: int,
track_name: str | None = None,
@ -183,6 +183,73 @@ async def honcho_llm_call(
retry_attempts: int = 3,
stream: bool = False,
) -> HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk]:
"""
Make an LLM call with automatic backup provider failover.
Args:
llm_settings: Settings object containing PROVIDER, MODEL,
BACKUP_PROVIDER, and BACKUP_MODEL
prompt: The prompt to send to the LLM
max_tokens: Maximum tokens to generate
track_name: Optional name for AI tracking
response_model: Optional Pydantic model for structured output
json_mode: Whether to use JSON mode
stop_seqs: Stop sequences
reasoning_effort: OpenAI reasoning effort (GPT-5 only)
verbosity: OpenAI verbosity (GPT-5 only)
thinking_budget_tokens: Anthropic thinking budget
enable_retry: Whether to enable retry with exponential backoff
retry_attempts: Number of retry attempts
stream: Whether to stream the response
Returns:
HonchoLLMCallResponse or AsyncIterator depending on stream parameter
Raises:
ValueError: If provider is not configured
"""
# Extract provider/model from settings, with backup support
attempt = _current_attempt.get()
# Use backup if: (1) we're retrying, (2) backup is configured, (3) backup client exists
use_backup = (
attempt > 0
and llm_settings.BACKUP_PROVIDER is not None
and llm_settings.BACKUP_MODEL is not None
and llm_settings.BACKUP_PROVIDER in CLIENTS
)
if (
use_backup
and llm_settings.BACKUP_PROVIDER is not None
and llm_settings.BACKUP_MODEL is not None
):
provider = llm_settings.BACKUP_PROVIDER
model = llm_settings.BACKUP_MODEL
logger.info(
f"Retry attempt {attempt}: switching from "
+ f"{llm_settings.PROVIDER}/{llm_settings.MODEL} to "
+ f"backup {provider}/{model}"
)
# Filter out incompatible parameters when using backup
if provider != "anthropic" and thinking_budget_tokens:
logger.warning(
f"thinking_budget_tokens not supported by {provider}, ignoring"
)
thinking_budget_tokens = None
if model and "gpt-5" not in model and (reasoning_effort or verbosity):
logger.warning(
"reasoning_effort/verbosity only supported by GPT-5 models, ignoring"
)
reasoning_effort = None
verbosity = None
else:
provider = llm_settings.PROVIDER
model = llm_settings.MODEL
# Validate client exists
client = CLIENTS.get(provider)
if not client:
raise ValueError(f"Missing client for {provider}")
@ -193,13 +260,28 @@ async def honcho_llm_call(
if track_name:
decorated = ai_track(track_name)(decorated)
# apply retry logic
# apply retry logic - retries on ANY exception
if enable_retry:
def before_retry_callback(retry_state: Any) -> None:
"""Update attempt counter before each retry."""
_current_attempt.set(retry_state.attempt_number)
exc = retry_state.outcome.exception() if retry_state.outcome else None
if exc:
logger.warning(
f"Error on attempt {retry_state.attempt_number} with "
+ f"{llm_settings.PROVIDER}/{llm_settings.MODEL}: {exc}"
)
decorated = retry(
stop=stop_after_attempt(retry_attempts),
wait=wait_exponential(multiplier=1, min=4, max=10),
before_sleep=before_retry_callback,
)(decorated)
# Reset attempt counter at start
_current_attempt.set(0)
if stream:
return await decorated(
provider,

6
src/utils/summarizer.py
View File

@ -123,8 +123,7 @@ Produce as thorough a summary as possible in {output_words} words or less.
""")
return await honcho_llm_call(
provider=settings.SUMMARY.PROVIDER,
model=settings.SUMMARY.MODEL,
llm_settings=settings.SUMMARY,
prompt=prompt,
max_tokens=settings.SUMMARY.MAX_TOKENS_SHORT,
)
@ -172,8 +171,7 @@ Produce as thorough a summary as possible in {output_words} words or less.
""")
return await honcho_llm_call(
provider=settings.SUMMARY.PROVIDER,
model=settings.SUMMARY.MODEL,
llm_settings=settings.SUMMARY,
prompt=prompt,
max_tokens=settings.SUMMARY.MAX_TOKENS_LONG,
)

6
tests/bench/peer_card_bench.py
View File

@ -143,6 +143,9 @@ def build_peer_card_caller(
"openai" if candidate.provider == "custom" else candidate.provider
)
settings.PEER_CARD.PROVIDER = cast(Any, resolved_provider)
settings.PEER_CARD.MODEL = candidate.model
async def call(
old_peer_card: list[str] | None, new_observations: Representation
) -> PeerCardQuery:
@ -152,8 +155,7 @@ def build_peer_card_caller(
)
response = await honcho_llm_call(
provider=cast(Any, resolved_provider),
model=candidate.model,
llm_settings=settings.PEER_CARD,
prompt=prompt,
max_tokens=settings.PEER_CARD.MAX_OUTPUT_TOKENS,
response_model=PeerCardQuery,

11
tests/utils/test_clients.py
View File

@ -25,6 +25,7 @@ from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.completion_usage import CompletionUsage
from pydantic import BaseModel, Field
from src.config import settings
from src.utils.clients import (
CLIENTS,
HonchoLLMCallResponse,
@ -962,10 +963,11 @@ class TestMainLLMCallFunction:
mock_client.messages.stream.return_value = mock_stream
with patch.dict(CLIENTS, {"anthropic": mock_client}):
settings.DIALECTIC.PROVIDER = "anthropic"
settings.DIALECTIC.MODEL = "claude-4-sonnet"
chunks: list[HonchoLLMCallStreamChunk] = []
async for chunk in await honcho_llm_call(
provider="anthropic",
model="claude-3-sonnet",
llm_settings=settings.DIALECTIC,
prompt="Hello",
max_tokens=100,
stream=True,
@ -990,9 +992,10 @@ class TestMainLLMCallFunction:
mock_client.messages.create = AsyncMock(return_value=mock_response)
with patch.dict(CLIENTS, {"anthropic": mock_client}):
settings.DIALECTIC.PROVIDER = "anthropic"
settings.DIALECTIC.MODEL = "claude-4-sonnet"
response = await honcho_llm_call(
provider="anthropic",
model="claude-3-sonnet",
llm_settings=settings.DIALECTIC,
prompt="Hello",
max_tokens=100,
enable_retry=False,