diff --git a/.env.template b/.env.template index 56a000ab..123af642 100644 --- a/.env.template +++ b/.env.template @@ -15,8 +15,13 @@ LOG_LEVEL=INFO # Embedding settings # EMBED_MESSAGES=true -# MAX_EMBEDDING_TOKENS=8192 -# MAX_EMBEDDING_TOKENS_PER_REQUEST=300000 +# EMBEDDING_VECTOR_DIMENSIONS=1536 +# EMBEDDING_MAX_INPUT_TOKENS=8192 +# EMBEDDING_MAX_TOKENS_PER_REQUEST=300000 +# EMBEDDING_MODEL_CONFIG__TRANSPORT=openai +# EMBEDDING_MODEL_CONFIG__MODEL=text-embedding-3-small +# EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL= +# EMBEDDING_MODEL_CONFIG__OVERRIDES__API_KEY_ENV= # LANGFUSE_HOST= # LANGFUSE_PUBLIC_KEY= @@ -62,55 +67,59 @@ AUTH_USE_AUTH=false # Honcho uses LLMs for memory extraction, summarization, dialectic chat, and # dream consolidation. The server will fail to start without a provider configured. # -# Quick start: uncomment the two lines below, set your endpoint and API key, -# then uncomment the provider/model lines in each feature section below. -# Any OpenAI-compatible endpoint works (OpenRouter, Together, Fireworks, etc.). +# Quick start: set LLM_OPENAI_API_KEY below to use the built-in defaults. +# Text-generation features default to transport = "openai" and +# model = "gpt-5.4-mini". Embeddings default to transport = "openai" and +# model = "text-embedding-3-small". For OpenAI-compatible proxies +# (OpenRouter, Together, Fireworks, vLLM, Ollama, LiteLLM), override +# MODEL_CONFIG__MODEL and MODEL_CONFIG__OVERRIDES__BASE_URL on each feature +# section you want to route through that endpoint. # Models must support tool calling (function calling). # -LLM_OPENAI_COMPATIBLE_BASE_URL=https://openrouter.ai/api/v1 -LLM_OPENAI_COMPATIBLE_API_KEY=your-api-key-here +# Supported transports: openai, anthropic, gemini +# Each transport picks up its API key from the corresponding LLM_*_API_KEY. +# Base URLs are set per-module via MODEL_CONFIG__OVERRIDES__BASE_URL. # -# Provider options for each feature: custom, vllm, google, anthropic, openai, groq -# "custom" routes through the OpenAI-compatible endpoint above. -# Model name format depends on your provider (e.g., OpenRouter: vendor/model-name). -# -# ---- Alternative: vLLM self-hosted ------------------------------------------ -# LLM_VLLM_BASE_URL=http://localhost:8000/v1 -# LLM_VLLM_API_KEY=not-needed -# -# ---- Alternative: direct vendor keys (no endpoint needed) ------------------- -# LLM_GEMINI_API_KEY= +LLM_OPENAI_API_KEY=your-api-key-here # LLM_ANTHROPIC_API_KEY= -# LLM_OPENAI_API_KEY= -# LLM_GROQ_API_KEY= -# -# ---- General LLM settings --------------------------------------------------- -# Embedding provider — defaults to openai (requires LLM_OPENAI_API_KEY). -# Set to openrouter to route embeddings through your custom endpoint instead. -LLM_EMBEDDING_PROVIDER=openrouter +# LLM_GEMINI_API_KEY= + +# ============================================================================= +# LLM Configuration +# ============================================================================= +# Global LLM settings # LLM_DEFAULT_MAX_TOKENS=2500 -# LLM_MAX_TOOL_OUTPUT_CHARS=10000 -# LLM_MAX_MESSAGE_CONTENT_CHARS=2000 +# LLM_MAX_TOOL_OUTPUT_CHARS=10000 # Max chars for tool output (~2500 tokens) +# LLM_MAX_MESSAGE_CONTENT_CHARS=2000 # Max chars per message in tool results # ============================================================================= # Deriver (Background Worker) # ============================================================================= # DERIVER_ENABLED=true -DERIVER_PROVIDER=custom -DERIVER_MODEL=your-model-here # e.g. google/gemini-2.5-flash -# DERIVER_THINKING_BUDGET_TOKENS=1024 # gt=0 required; omit for non-thinking models +# Defaults: +# DERIVER_MODEL_CONFIG__TRANSPORT=openai +# DERIVER_MODEL_CONFIG__MODEL=gpt-5.4-mini +# Optional overrides: +# DERIVER_MODEL_CONFIG__MODEL=your-model-here +# DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 # DERIVER_WORKERS=1 # DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0 # DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 -# DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000 -# DERIVER_TEMPERATURE= +# DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000 # 30 days +# DERIVER_MODEL_CONFIG__TEMPERATURE= +# DERIVER_MODEL_CONFIG__THINKING_EFFORT=minimal +# DERIVER_MODEL_CONFIG__THINKING_BUDGET_TOKENS=1024 # Gemini/Anthropic only # DERIVER_DEDUPLICATE=true -# DERIVER_MAX_OUTPUT_TOKENS=4096 +# DERIVER_MODEL_CONFIG__MAX_OUTPUT_TOKENS=4096 # DERIVER_LOG_OBSERVATIONS=false # DERIVER_MAX_INPUT_TOKENS=23000 # DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100 # DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=1024 -# DERIVER_FLUSH_ENABLED=false +# DERIVER_FLUSH_ENABLED=false # Bypass batch token threshold, process work immediately +# DERIVER_MODEL_CONFIG__FALLBACK__MODEL= +# DERIVER_MODEL_CONFIG__FALLBACK__TRANSPORT= +# DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL= +# DERIVER_MODEL_CONFIG__OVERRIDES__API_KEY_ENV= # ============================================================================= # Peer Card @@ -125,58 +134,79 @@ DERIVER_MODEL=your-model-here # e.g. google/gemini-2.5-flash # DIALECTIC_HISTORY_TOKEN_LIMIT=8192 # DIALECTIC_SESSION_HISTORY_MAX_TOKENS=4096 # -# Per-level provider, model, and tuning: -DIALECTIC_LEVELS__minimal__PROVIDER=custom -DIALECTIC_LEVELS__minimal__MODEL=your-model-here # e.g. google/gemini-2.5-flash -# DIALECTIC_LEVELS__minimal__THINKING_BUDGET_TOKENS=0 +# Per-level settings (reasoning_level parameter in API) +# Each level has its own nested MODEL_CONFIG, tool iterations, and max output tokens. +# MAX_OUTPUT_TOKENS is optional per level; if not set, uses global DIALECTIC_MAX_OUTPUT_TOKENS. +# Defaults: +# DIALECTIC_LEVELS__minimal__MODEL_CONFIG__TRANSPORT=openai +# DIALECTIC_LEVELS__minimal__MODEL_CONFIG__MODEL=gpt-5.4-mini # DIALECTIC_LEVELS__minimal__MAX_TOOL_ITERATIONS=1 # DIALECTIC_LEVELS__minimal__MAX_OUTPUT_TOKENS=250 -DIALECTIC_LEVELS__low__PROVIDER=custom -DIALECTIC_LEVELS__low__MODEL=your-model-here -# DIALECTIC_LEVELS__low__THINKING_BUDGET_TOKENS=0 +# DIALECTIC_LEVELS__minimal__TOOL_CHOICE=any +# DIALECTIC_LEVELS__low__MODEL_CONFIG__TRANSPORT=openai +# DIALECTIC_LEVELS__low__MODEL_CONFIG__MODEL=gpt-5.4-mini # DIALECTIC_LEVELS__low__MAX_TOOL_ITERATIONS=5 -DIALECTIC_LEVELS__medium__PROVIDER=custom -DIALECTIC_LEVELS__medium__MODEL=your-model-here -# DIALECTIC_LEVELS__medium__THINKING_BUDGET_TOKENS=0 +# DIALECTIC_LEVELS__low__TOOL_CHOICE=any +# DIALECTIC_LEVELS__medium__MODEL_CONFIG__TRANSPORT=openai +# DIALECTIC_LEVELS__medium__MODEL_CONFIG__MODEL=gpt-5.4-mini # DIALECTIC_LEVELS__medium__MAX_TOOL_ITERATIONS=2 -DIALECTIC_LEVELS__high__PROVIDER=custom -DIALECTIC_LEVELS__high__MODEL=your-model-here -# DIALECTIC_LEVELS__high__THINKING_BUDGET_TOKENS=0 +# DIALECTIC_LEVELS__high__MODEL_CONFIG__TRANSPORT=openai +# DIALECTIC_LEVELS__high__MODEL_CONFIG__MODEL=gpt-5.4-mini # DIALECTIC_LEVELS__high__MAX_TOOL_ITERATIONS=4 -DIALECTIC_LEVELS__max__PROVIDER=custom -DIALECTIC_LEVELS__max__MODEL=your-model-here -# DIALECTIC_LEVELS__max__THINKING_BUDGET_TOKENS=0 +# DIALECTIC_LEVELS__max__MODEL_CONFIG__TRANSPORT=openai +# DIALECTIC_LEVELS__max__MODEL_CONFIG__MODEL=gpt-5.4-mini # DIALECTIC_LEVELS__max__MAX_TOOL_ITERATIONS=10 +# Optional overrides: +# DIALECTIC_LEVELS__minimal__MODEL_CONFIG__MODEL=your-model-here +# DIALECTIC_LEVELS__low__MODEL_CONFIG__MODEL=your-model-here +# DIALECTIC_LEVELS__medium__MODEL_CONFIG__MODEL=your-model-here +# DIALECTIC_LEVELS__high__MODEL_CONFIG__MODEL=your-model-here +# DIALECTIC_LEVELS__max__MODEL_CONFIG__MODEL=your-model-here +# DIALECTIC_LEVELS__max__MODEL_CONFIG__THINKING_EFFORT=medium +# DIALECTIC_LEVELS__max__MODEL_CONFIG__THINKING_BUDGET_TOKENS=1024 +# Optional backup per level (must set both or neither): +# DIALECTIC_LEVELS__max__MODEL_CONFIG__FALLBACK__MODEL=gemini-2.5-pro +# DIALECTIC_LEVELS__max__MODEL_CONFIG__FALLBACK__TRANSPORT=gemini # ============================================================================= # Summary # ============================================================================= # SUMMARY_ENABLED=true -SUMMARY_PROVIDER=custom -SUMMARY_MODEL=your-model-here # e.g. google/gemini-2.5-flash -# SUMMARY_THINKING_BUDGET_TOKENS=512 # gt=0 required; omit for non-thinking models +# Defaults: +# SUMMARY_MODEL_CONFIG__TRANSPORT=openai +# SUMMARY_MODEL_CONFIG__MODEL=gpt-5.4-mini +# Optional overrides: +# SUMMARY_MODEL_CONFIG__MODEL=your-model-here +# SUMMARY_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 +# SUMMARY_MODEL_CONFIG__THINKING_EFFORT=minimal +# SUMMARY_MODEL_CONFIG__THINKING_BUDGET_TOKENS=1024 # Gemini/Anthropic only # SUMMARY_MESSAGES_PER_SHORT_SUMMARY=20 # SUMMARY_MESSAGES_PER_LONG_SUMMARY=60 # SUMMARY_MAX_TOKENS_SHORT=1000 # SUMMARY_MAX_TOKENS_LONG=4000 +# SUMMARY_MODEL_CONFIG__FALLBACK__MODEL= # ============================================================================= # Dream # ============================================================================= # DREAM_ENABLED=true -DREAM_PROVIDER=custom -DREAM_MODEL=your-model-here # e.g. google/gemini-2.5-flash -DREAM_DEDUCTION_MODEL=your-model-here -DREAM_INDUCTION_MODEL=your-model-here -# DREAM_THINKING_BUDGET_TOKENS=8192 # gt=0 required; omit for non-thinking models +# Defaults: +# DREAM_DEDUCTION_MODEL_CONFIG__TRANSPORT=openai +# DREAM_DEDUCTION_MODEL_CONFIG__MODEL=gpt-5.4-mini +# DREAM_INDUCTION_MODEL_CONFIG__TRANSPORT=openai +# DREAM_INDUCTION_MODEL_CONFIG__MODEL=gpt-5.4-mini +# Optional overrides: +# DREAM_DEDUCTION_MODEL_CONFIG__MODEL=your-model-here +# DREAM_DEDUCTION_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 +# DREAM_INDUCTION_MODEL_CONFIG__MODEL=your-model-here +# DREAM_INDUCTION_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 # DREAM_DOCUMENT_THRESHOLD=50 # DREAM_IDLE_TIMEOUT_MINUTES=60 # DREAM_MIN_HOURS_BETWEEN_DREAMS=8 # DREAM_ENABLED_TYPES=["omni"] -# DREAM_MAX_OUTPUT_TOKENS=16384 # DREAM_MAX_TOOL_ITERATIONS=20 # DREAM_HISTORY_TOKEN_LIMIT=16384 -# + # Surprisal sampling (advanced): # DREAM_SURPRISAL__ENABLED=false # DREAM_SURPRISAL__TREE_TYPE=kdtree diff --git a/CHANGELOG.md b/CHANGELOG.md index e5208a95..ab9302c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] + +### Added + +- New `src/llm/` package as the single owner of provider runtime: clients, backends, history adapters, tool loop, request builder, credentials, and caching policy +- `AttemptPlan` dataclass captures per-retry provider selection (client, model, reasoning_effort, thinking_budget_tokens, selected_config) and pins it across stream-final retries so streaming doesn't bounce back to primary after the tool loop has settled on fallback +- Gemini JSON-schema sanitizer for `function_declarations` — strips keywords Gemini's validator rejects (`additionalProperties`, `allOf`, etc.) while preserving semantics for all other backends +- Dreamer specialists derive `effective_max_tokens` from `model_config.max_output_tokens` with a per-specialist default fallback +- Regression tests covering fallback-config thinking-param reach, provider_params → extra_params boundary, OpenAI reasoning-model parameter routing, Gemini blocked finish_reason handling, and fail-fast `max_tool_iterations` validation + +### Changed + +- All LLM orchestration moved out of `src/utils/clients.py` into `src/llm/` with modules split by responsibility (api, executor, tool_loop, runtime, registry, conversation, request_builder, credentials, caching, backends, history_adapters) +- Default `ModelConfig` factories (deriver, summary, dreamer specialists, dialectic levels) normalized to `openai/gpt-5.4-mini` with no extra parameters set by default; operators add transport/thinking overrides explicitly +- OpenAI reasoning-model routing widened via `_uses_max_completion_tokens` heuristic covering `gpt-5.x` and `o1/o3/o4` — these models receive `max_completion_tokens` instead of `max_tokens` +- Override client factories switched from unbounded `@cache` to `@lru_cache(maxsize=128)` for predictable memory growth on long-running processes +- `get_backend` now delegates to `client_for_model_config`, so the live-test path and production path share one missing-API-key validation +- Blocked Gemini responses (`SAFETY`, `RECITATION`, `PROHIBITED_CONTENT`, `BLOCKLIST`) raise `LLMError` in the streaming path too (previously only the non-streaming path), ensuring retry/fallback logic fires uniformly +- Transport-change env overrides now strip transport-specific thinking params (thinking_budget_tokens vs. reasoning_effort) during config merge, including at the dialectic-level merge, so switching from Anthropic → OpenAI doesn't leave orphaned Anthropic-only params that the OpenAI backend would reject +- `max_tool_iterations` out-of-range inputs now raise `ValidationException` instead of being silently clamped +- Troubleshooting docs updated to reflect nested-env-var form for per-component thinking-budget overrides + +### Fixed + +- Fallback `ModelConfig` temperature and `thinking_budget_tokens` reach the backend on the final retry — previously the primary's values were pre-populated into caller kwargs early and clobbered fallback values via `effective_config_for_call(update=...)` +- Stream-final retries pin to the `AttemptPlan` that succeeded rather than re-running provider selection through the outer `current_attempt` ContextVar (which could roll streaming back to primary after the tool loop had already switched to fallback) +- OpenAI structured-output calls continue to use `chat.completions.parse()` with strict schema enforcement, while tool-calling paths use `chat.completions.create()` without `strict:True` for broader proxy compatibility (OpenRouter, vLLM, Ollama) +- Gemini `cached_content` reuse keys now include `system_instruction` and `tool_config` so cache hits don't cross configurations that differ only in those fields + +### Removed + +- `src/utils/clients.py` deleted; its responsibilities are split across `src/llm/registry.py`, `src/llm/credentials.py`, and the backend-specific modules + ## [3.0.6] - 2026-04-10 ### Changed diff --git a/README.md b/README.md index 5ee20e58..9a7cf541 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,6 @@ DB_CONNECTION_URI= # Connection uri for a postgres database (with postgresql+psy LLM_GEMINI_API_KEY= # API Key for Google Gemini (used for deriver, summary, and dialectic minimal/low by default) LLM_ANTHROPIC_API_KEY= # API Key for Anthropic (used for dialectic medium/high/max and dream by default) LLM_OPENAI_API_KEY= # API Key for OpenAI (used for embeddings when EMBED_MESSAGES=true) -LLM_GROQ_API_KEY= # API Key for Groq (optional) ``` > Note that the `DB_CONNECTION_URI` must have the prefix `postgresql+psycopg` to @@ -420,16 +419,17 @@ Then modify the values as needed. The TOML file is organized into sections: All configuration values can be overridden using environment variables. The environment variable names follow this pattern: -- `{SECTION}_{KEY}` for nested settings +- `{SECTION}_{KEY}` for top-level section settings +- Use `__` inside `{KEY}` for nested settings - Just `{KEY}` for app-level settings Examples: - `DB_CONNECTION_URI` - Database connection string - `AUTH_JWT_SECRET` - JWT secret key -- `DIALECTIC_LEVELS__low__MODEL` - Model for low reasoning level -- `DERIVER_PROVIDER` - Provider for background deriver -- `SUMMARY_PROVIDER` - Summary generation provider +- `DERIVER_MODEL_CONFIG__TRANSPORT` - Transport for the background deriver +- `SUMMARY_MODEL_CONFIG__MODEL` - Summary model override +- `DIALECTIC_LEVELS__low__MODEL_CONFIG__MODEL` - Model for low reasoning level - `LOG_LEVEL` - Application log level - `METRICS_ENABLED` - Enable Prometheus metrics - `TELEMETRY_ENABLED` - Enable CloudEvents telemetry diff --git a/config.toml.example b/config.toml.example index b9cf84c0..236f3402 100644 --- a/config.toml.example +++ b/config.toml.example @@ -11,8 +11,6 @@ GET_CONTEXT_MAX_TOKENS = 100000 MAX_FILE_SIZE = 5242880 # 5MB MAX_MESSAGE_SIZE = 25000 # Characters EMBED_MESSAGES = true -MAX_EMBEDDING_TOKENS = 8192 -MAX_EMBEDDING_TOKENS_PER_REQUEST = 300000 # LANGFUSE_HOST = "https://api.langfuse.com" # LANGFUSE_PUBLIC_KEY = "your-public-key-here" # COLLECT_METRICS_LOCAL = false @@ -51,25 +49,32 @@ PROFILES_SAMPLE_RATE = 0.1 # LLM settings [llm] DEFAULT_MAX_TOKENS = 2500 -EMBEDDING_PROVIDER = "openai" MAX_TOOL_OUTPUT_CHARS = 10000 # Max chars for tool output (~2500 tokens) MAX_MESSAGE_CONTENT_CHARS = 2000 # Max chars per message in tool results # API Keys for LLM providers (set the ones you need) -# GEMINI_API_KEY = "your-api-key" # Default: deriver, summary, dialectic minimal/low -# ANTHROPIC_API_KEY = "your-api-key" # Default: dialectic medium/high/max, dream -# OPENAI_API_KEY = "your-api-key" # Default: embeddings -# GROQ_API_KEY = "your-api-key" # Not used by default +# Supported transports: openai, anthropic, gemini +# Base URLs are set per-module via model_config.overrides.base_url +# Built-in text-generation defaults use openai / gpt-5.4-mini. +# Embeddings default to openai / text-embedding-3-small. +OPENAI_API_KEY = "your-api-key-here" +# ANTHROPIC_API_KEY = "your-api-key" +# GEMINI_API_KEY = "your-api-key" -# OpenAI-compatible endpoint (OpenRouter, Together, Fireworks, LiteLLM, etc.) -# Set provider to "custom" in feature config to route calls through this endpoint. -# OPENAI_COMPATIBLE_BASE_URL = "https://openrouter.ai/api/v1" -# OPENAI_COMPATIBLE_API_KEY = "your-api-key" +# Embedding settings +[embedding] +VECTOR_DIMENSIONS = 1536 +MAX_INPUT_TOKENS = 8192 +MAX_TOKENS_PER_REQUEST = 300000 -# vLLM endpoint (for self-hosted models) -# Set provider to "vllm" in feature config to route calls through this endpoint. -# VLLM_BASE_URL = "http://localhost:8000/v1" -# VLLM_API_KEY = "not-needed" +[embedding.model_config] +transport = "openai" +model = "text-embedding-3-small" + +# Optional module-level endpoint overrides +# [embedding.model_config.overrides] +# base_url = "https://embedding-proxy.internal.example/v1" +# api_key_env = "EMBEDDING_CUSTOM_API_KEY" # Deriver settings [deriver] @@ -78,20 +83,38 @@ WORKERS = 1 POLLING_SLEEP_INTERVAL_SECONDS = 1.0 STALE_SESSION_TIMEOUT_MINUTES = 5 # QUEUE_ERROR_RETENTION_SECONDS = 2592000 # 30 days -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" -# TEMPERATURE = 0.0 -# BACKUP_PROVIDER = "anthropic" -# BACKUP_MODEL = "claude-haiku-4-5" DEDUPLICATE = true -MAX_OUTPUT_TOKENS = 4096 -THINKING_BUDGET_TOKENS = 1024 LOG_OBSERVATIONS = false MAX_INPUT_TOKENS = 23000 WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100 REPRESENTATION_BATCH_MAX_TOKENS = 1024 FLUSH_ENABLED = false # Bypass batch token threshold, process work immediately +[deriver.model_config] +transport = "openai" +model = "gpt-5.4-mini" +# temperature = 0.0 +# thinking_effort = "minimal" +# thinking_budget_tokens = 1024 +# max_output_tokens = 4096 + +# Optional module-level endpoint overrides +# transport = "openai" +# model = "my-local-model" +# [deriver.model_config.overrides] +# base_url = "https://llm.internal.example/v1" +# api_key_env = "DERIVER_CUSTOM_API_KEY" + +# Optional fallback model +# [deriver.model_config.fallback] +# transport = "anthropic" +# model = "claude-haiku-4-5" +# [deriver.model_config.fallback.overrides] +# base_url = "https://llm-backup.internal.example/v1" +# api_key_env = "DERIVER_CUSTOM_BACKUP_API_KEY" +# [deriver.model_config.overrides.provider_params] +# verbosity = "low" + # Peer card settings [peer_card] ENABLED = true @@ -106,55 +129,64 @@ SESSION_HISTORY_MAX_TOKENS = 4096 # Per-level settings for reasoning levels # MAX_OUTPUT_TOKENS is optional per level; if not set, uses global MAX_OUTPUT_TOKENS [dialectic.levels.minimal] -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" -THINKING_BUDGET_TOKENS = 0 MAX_TOOL_ITERATIONS = 1 MAX_OUTPUT_TOKENS = 250 +TOOL_CHOICE = "any" + +[dialectic.levels.minimal.model_config] +transport = "openai" +model = "gpt-5.4-mini" [dialectic.levels.low] -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" -THINKING_BUDGET_TOKENS = 0 MAX_TOOL_ITERATIONS = 5 -# MAX_OUTPUT_TOKENS = 8192 # Optional: override global default +TOOL_CHOICE = "any" + +[dialectic.levels.low.model_config] +transport = "openai" +model = "gpt-5.4-mini" [dialectic.levels.medium] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 1024 MAX_TOOL_ITERATIONS = 2 -# MAX_OUTPUT_TOKENS = 8192 # Optional: override global default + +[dialectic.levels.medium.model_config] +transport = "openai" +model = "gpt-5.4-mini" [dialectic.levels.high] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 1024 MAX_TOOL_ITERATIONS = 4 -# MAX_OUTPUT_TOKENS = 8192 # Optional: override global default + +[dialectic.levels.high.model_config] +transport = "openai" +model = "gpt-5.4-mini" [dialectic.levels.max] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 2048 MAX_TOOL_ITERATIONS = 10 -# MAX_OUTPUT_TOKENS = 8192 # Optional: override global default -# Backup provider example (optional, must set both or neither): -# BACKUP_PROVIDER = "google" -# BACKUP_MODEL = "gemini-2.5-pro" + +[dialectic.levels.max.model_config] +transport = "openai" +model = "gpt-5.4-mini" + +# [dialectic.levels.max.model_config.fallback] +# transport = "gemini" +# model = "gemini-2.5-pro" # Summary settings [summary] ENABLED = true MESSAGES_PER_SHORT_SUMMARY = 20 MESSAGES_PER_LONG_SUMMARY = 60 -PROVIDER = "google" -MODEL = "gemini-2.5-flash" MAX_TOKENS_SHORT = 1000 MAX_TOKENS_LONG = 4000 -THINKING_BUDGET_TOKENS = 512 -# BACKUP_PROVIDER = "google" -# BACKUP_MODEL = "gemini-2.5-flash" + +[summary.model_config] +transport = "openai" +model = "gpt-5.4-mini" +# thinking_effort = "minimal" +# thinking_budget_tokens = 1024 + +# [summary.model_config.fallback] +# transport = "anthropic" +# model = "claude-haiku-4-5" # Dream settings [dream] @@ -163,18 +195,16 @@ DOCUMENT_THRESHOLD = 50 IDLE_TIMEOUT_MINUTES = 60 MIN_HOURS_BETWEEN_DREAMS = 8 ENABLED_TYPES = ["omni"] -PROVIDER = "anthropic" -MODEL = "claude-sonnet-4-20250514" -MAX_OUTPUT_TOKENS = 16384 -THINKING_BUDGET_TOKENS = 8192 MAX_TOOL_ITERATIONS = 20 HISTORY_TOKEN_LIMIT = 16384 -# BACKUP_PROVIDER = "google" -# BACKUP_MODEL = "gemini-2.5-flash" -# Specialist models (use same provider as main model) -DEDUCTION_MODEL = "claude-haiku-4-5" -INDUCTION_MODEL = "claude-haiku-4-5" +[dream.deduction_model_config] +transport = "openai" +model = "gpt-5.4-mini" + +[dream.induction_model_config] +transport = "openai" +model = "gpt-5.4-mini" # Surprisal-based sampling subsystem [dream.surprisal] @@ -224,6 +254,8 @@ TYPE = "pgvector" # Migration flag: set to true when migration from pgvector is complete MIGRATED = false NAMESPACE = "honcho" +# This should match embedding.vector_dimensions. pgvector and dual-write mode +# currently still require 1536 until a schema migration lands. DIMENSIONS = 1536 # TURBOPUFFER_API_KEY = "your-turbopuffer-api-key" # TURBOPUFFER_REGION = "us-east-1" diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index 5ae99cc9..02916ab6 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -26,13 +26,13 @@ cp config.toml.example config.toml All config values map to environment variables: -- `{SECTION}_{KEY}` for section settings (e.g., `DB_CONNECTION_URI` → `[db].CONNECTION_URI`) +- `{SECTION}_{KEY}` for top-level section settings (e.g., `DB_CONNECTION_URI` → `[db].CONNECTION_URI`) - `{KEY}` for app-level settings (e.g., `LOG_LEVEL` → `[app].LOG_LEVEL`) -- `{SECTION}__{NESTED}__{KEY}` for deeply nested settings (double underscore, e.g., `DIALECTIC_LEVELS__minimal__PROVIDER`) +- Use `__` inside `{KEY}` for nested settings (e.g., `DIALECTIC_LEVELS__minimal__MODEL_CONFIG__TRANSPORT`, `DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL`) ## LLM Configuration -The [Self-Hosting Guide](./self-hosting#llm-setup) covers the basic setup: one OpenAI-compatible endpoint, one model for all features. This section covers recommended model tiers, using multiple providers, and per-feature tuning. +The [Self-Hosting Guide](./self-hosting#llm-setup) covers the basic setup: either the built-in OpenAI defaults or one OpenAI-compatible endpoint/model for all features. This section covers recommended model tiers, using multiple providers, and per-feature tuning. All Honcho agents (deriver, dialectic, dream) require tool calling. Your models must support the OpenAI tool calling format. @@ -52,104 +52,175 @@ You can mix providers freely — for example, use Gemini for the deriver and Cla ### Provider Types -| Provider value | What it connects to | Key env var | +| Transport value | What it connects to | API key env var | |---|---|---| -| `custom` | Any OpenAI-compatible endpoint (OpenRouter, Together, Fireworks, LiteLLM, Ollama) | `LLM_OPENAI_COMPATIBLE_API_KEY` + `LLM_OPENAI_COMPATIBLE_BASE_URL` | -| `vllm` | vLLM self-hosted models | `LLM_VLLM_API_KEY` + `LLM_VLLM_BASE_URL` | -| `google` | Google Gemini (direct) | `LLM_GEMINI_API_KEY` | +| `openai` | OpenAI or any OpenAI-compatible endpoint (OpenRouter, Together, Fireworks, LiteLLM, vLLM, Ollama) | `LLM_OPENAI_API_KEY` | | `anthropic` | Anthropic Claude (direct) | `LLM_ANTHROPIC_API_KEY` | -| `openai` | OpenAI (direct) | `LLM_OPENAI_API_KEY` | -| `groq` | Groq (direct) | `LLM_GROQ_API_KEY` | +| `gemini` | Google Gemini (direct) | `LLM_GEMINI_API_KEY` | + +For OpenAI-compatible proxies (OpenRouter, vLLM, Ollama, etc.), use `transport = "openai"` and set `MODEL_CONFIG__OVERRIDES__BASE_URL` on each feature to point at your endpoint. ### Tiered Model Setup Once you're past initial setup, you can assign different models per feature for better cost/quality tradeoffs. This example uses OpenRouter with light/medium/heavy tiers: ```bash -LLM_OPENAI_COMPATIBLE_BASE_URL=https://openrouter.ai/api/v1 -LLM_OPENAI_COMPATIBLE_API_KEY=sk-or-v1-... +LLM_OPENAI_API_KEY=sk-or-v1-... + +# All features route through OpenRouter via overrides.base_url +# (You can set this on each feature's MODEL_CONFIG) # Light tier — high throughput, cheap -DERIVER_PROVIDER=custom -DERIVER_MODEL=google/gemini-2.5-flash-lite -SUMMARY_PROVIDER=custom -SUMMARY_MODEL=google/gemini-2.5-flash -DIALECTIC_LEVELS__minimal__PROVIDER=custom -DIALECTIC_LEVELS__minimal__MODEL=google/gemini-2.5-flash-lite -DIALECTIC_LEVELS__low__PROVIDER=custom -DIALECTIC_LEVELS__low__MODEL=google/gemini-2.5-flash-lite +DERIVER_MODEL_CONFIG__TRANSPORT=openai +DERIVER_MODEL_CONFIG__MODEL=google/gemini-2.5-flash-lite +DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 +SUMMARY_MODEL_CONFIG__TRANSPORT=openai +SUMMARY_MODEL_CONFIG__MODEL=google/gemini-2.5-flash +DIALECTIC_LEVELS__minimal__MODEL_CONFIG__TRANSPORT=openai +DIALECTIC_LEVELS__minimal__MODEL_CONFIG__MODEL=google/gemini-2.5-flash-lite +DIALECTIC_LEVELS__low__MODEL_CONFIG__TRANSPORT=openai +DIALECTIC_LEVELS__low__MODEL_CONFIG__MODEL=google/gemini-2.5-flash-lite # Medium tier — better reasoning -DIALECTIC_LEVELS__medium__PROVIDER=custom -DIALECTIC_LEVELS__medium__MODEL=anthropic/claude-haiku-4-5 -DIALECTIC_LEVELS__high__PROVIDER=custom -DIALECTIC_LEVELS__high__MODEL=anthropic/claude-haiku-4-5 -DIALECTIC_LEVELS__max__PROVIDER=custom -DIALECTIC_LEVELS__max__MODEL=anthropic/claude-haiku-4-5 +DIALECTIC_LEVELS__medium__MODEL_CONFIG__TRANSPORT=openai +DIALECTIC_LEVELS__medium__MODEL_CONFIG__MODEL=anthropic/claude-haiku-4-5 +DIALECTIC_LEVELS__high__MODEL_CONFIG__TRANSPORT=openai +DIALECTIC_LEVELS__high__MODEL_CONFIG__MODEL=anthropic/claude-haiku-4-5 +DIALECTIC_LEVELS__max__MODEL_CONFIG__TRANSPORT=openai +DIALECTIC_LEVELS__max__MODEL_CONFIG__MODEL=anthropic/claude-haiku-4-5 # Heavy tier — best quality for complex tasks -DREAM_PROVIDER=custom -DREAM_MODEL=anthropic/claude-sonnet-4-20250514 -DREAM_DEDUCTION_MODEL=anthropic/claude-haiku-4-5 -DREAM_INDUCTION_MODEL=anthropic/claude-haiku-4-5 +DREAM_DEDUCTION_MODEL_CONFIG__TRANSPORT=openai +DREAM_DEDUCTION_MODEL_CONFIG__MODEL=anthropic/claude-haiku-4-5 +DREAM_INDUCTION_MODEL_CONFIG__TRANSPORT=openai +DREAM_INDUCTION_MODEL_CONFIG__MODEL=anthropic/claude-haiku-4-5 ``` ### Direct Vendor Keys -Instead of an OpenAI-compatible proxy, you can use vendor APIs directly. Leave `PROVIDER` overrides unset and the code defaults route per feature: +Instead of an OpenAI-compatible proxy, you can use vendor APIs directly. Each transport picks up its own `LLM_{TRANSPORT}_API_KEY`. + +If you keep the built-in defaults, only `LLM_OPENAI_API_KEY` is required: ```bash -LLM_GEMINI_API_KEY=... # deriver, summary, dialectic minimal/low -LLM_ANTHROPIC_API_KEY=... # dialectic medium/high/max, dream -LLM_OPENAI_API_KEY=... # embeddings +LLM_OPENAI_API_KEY=... + +# Built-in model defaults +# - deriver: openai / gpt-5.4-mini +# - dialectic (all levels): openai / gpt-5.4-mini +# - summary: openai / gpt-5.4-mini +# - dream specialists: openai / gpt-5.4-mini +# - embeddings: openai / text-embedding-3-small +``` + +To use Gemini or Anthropic directly, override the features you want to move: + +```bash +LLM_GEMINI_API_KEY=... +DERIVER_MODEL_CONFIG__TRANSPORT=gemini +DERIVER_MODEL_CONFIG__MODEL=gemini-2.5-flash + +LLM_ANTHROPIC_API_KEY=... +DREAM_DEDUCTION_MODEL_CONFIG__TRANSPORT=anthropic +DREAM_DEDUCTION_MODEL_CONFIG__MODEL=claude-haiku-4-5 ``` ### Self-Hosted (vLLM / Ollama) +Use `transport = "openai"` and set `MODEL_CONFIG__OVERRIDES__BASE_URL` on each feature: + ```bash # vLLM -LLM_VLLM_BASE_URL=http://localhost:8000/v1 -LLM_VLLM_API_KEY=not-needed -DERIVER_PROVIDER=vllm -DERIVER_MODEL=your-model-name +LLM_OPENAI_API_KEY=not-needed +DERIVER_MODEL_CONFIG__TRANSPORT=openai +DERIVER_MODEL_CONFIG__MODEL=your-model-name +DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=http://localhost:8000/v1 -# Ollama (uses custom provider) -LLM_OPENAI_COMPATIBLE_BASE_URL=http://localhost:11434/v1 -LLM_OPENAI_COMPATIBLE_API_KEY=ollama -DERIVER_PROVIDER=custom -DERIVER_MODEL=llama3.3:70b +# Ollama +LLM_OPENAI_API_KEY=ollama +DERIVER_MODEL_CONFIG__TRANSPORT=openai +DERIVER_MODEL_CONFIG__MODEL=llama3.3:70b +DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=http://localhost:11434/v1 ``` -Set `PROVIDER` and `MODEL` for each feature the same way. +Set `MODEL_CONFIG__TRANSPORT`, `MODEL_CONFIG__MODEL`, and `MODEL_CONFIG__OVERRIDES__BASE_URL` for each feature the same way. + +The same overrides are available in `config.toml`: + +```toml +[deriver.model_config] +transport = "openai" +model = "my-local-model" + +[deriver.model_config.overrides] +base_url = "http://localhost:8000/v1" +api_key_env = "DERIVER_LOCAL_API_KEY" +``` ### Thinking Budget -Default configs use `THINKING_BUDGET_TOKENS` tuned for Anthropic models. Non-Anthropic providers don't support extended thinking and will error or silently fail. The [Self-Hosting Guide](./self-hosting#llm-setup) sets these to `0` by default. If you switch to Anthropic models, you can re-enable them: +Built-in defaults do not set `MODEL_CONFIG__THINKING_BUDGET_TOKENS` or `MODEL_CONFIG__THINKING_EFFORT`. Add one only when your chosen model supports it. + +Use `MODEL_CONFIG__THINKING_EFFORT` for OpenAI reasoning models: ```bash -# Anthropic models — enable thinking -DERIVER_THINKING_BUDGET_TOKENS=1024 -SUMMARY_THINKING_BUDGET_TOKENS=512 -DREAM_THINKING_BUDGET_TOKENS=8192 -DIALECTIC_LEVELS__medium__THINKING_BUDGET_TOKENS=1024 -DIALECTIC_LEVELS__high__THINKING_BUDGET_TOKENS=1024 -DIALECTIC_LEVELS__max__THINKING_BUDGET_TOKENS=2048 -# minimal and low stay at 0 +DERIVER_MODEL_CONFIG__THINKING_EFFORT=minimal +DIALECTIC_LEVELS__max__MODEL_CONFIG__THINKING_EFFORT=medium ``` +Use `MODEL_CONFIG__THINKING_BUDGET_TOKENS` for Anthropic and Gemini models. Set it to `0` or omit it for providers that don't support extended thinking: + +```bash +SUMMARY_MODEL_CONFIG__THINKING_BUDGET_TOKENS=1024 +DREAM_DEDUCTION_MODEL_CONFIG__THINKING_BUDGET_TOKENS=1024 +``` + +### Provider-Specific Parameters + +Each model config supports an `overrides.provider_params` dict for passing arbitrary parameters to the underlying provider SDK. Use this for vendor-specific features that aren't part of the standard config: + +```toml +[deriver.model_config.overrides.provider_params] +# These are passed directly to the provider SDK +verbosity = "low" +``` + +### Changing Transport + +When changing a feature's `transport`, always specify `model` explicitly. Partial overrides that change transport without model will keep the previous model name, which may not be valid for the new provider. + ### General LLM Settings ```bash LLM_DEFAULT_MAX_TOKENS=2500 -# Embedding provider (used when EMBED_MESSAGES=true) -LLM_EMBEDDING_PROVIDER=openai # Options: openai, gemini, openrouter - # Tool output limits (to prevent token explosion) LLM_MAX_TOOL_OUTPUT_CHARS=10000 # ~2500 tokens at 4 chars/token LLM_MAX_MESSAGE_CONTENT_CHARS=2000 # Max chars per message in tool results ``` +### Embedding Configuration + +Embeddings use their own nested model config, separate from the main text-generation LLM settings. + +```bash +# Embedding vector settings +EMBEDDING_VECTOR_DIMENSIONS=1536 +EMBEDDING_MAX_INPUT_TOKENS=8192 +EMBEDDING_MAX_TOKENS_PER_REQUEST=300000 + +# Embedding transport/model selection +EMBEDDING_MODEL_CONFIG__TRANSPORT=openai # openai, gemini +EMBEDDING_MODEL_CONFIG__MODEL=text-embedding-3-small + +# Optional endpoint overrides +EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=http://localhost:8000/v1 +EMBEDDING_MODEL_CONFIG__OVERRIDES__API_KEY_ENV=EMBEDDING_CUSTOM_API_KEY +``` + +Current constraint: +- `EMBEDDING_VECTOR_DIMENSIONS` can be changed for fully migrated external vector stores, but pgvector and dual-write mode still require `1536` until the schema migration lands. + ### Feature-Specific Model Configuration Each feature can use a different provider and model. Below are all the tuning knobs. @@ -173,45 +244,51 @@ Each reasoning level has its own provider, model, and settings: ```toml # config.toml example [dialectic.levels.minimal] -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" -THINKING_BUDGET_TOKENS = 0 MAX_TOOL_ITERATIONS = 1 MAX_OUTPUT_TOKENS = 250 TOOL_CHOICE = "any" +[dialectic.levels.minimal.model_config] +transport = "openai" +model = "gpt-5.4-mini" + [dialectic.levels.low] -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" -THINKING_BUDGET_TOKENS = 0 MAX_TOOL_ITERATIONS = 5 TOOL_CHOICE = "any" +[dialectic.levels.low.model_config] +transport = "openai" +model = "gpt-5.4-mini" + [dialectic.levels.medium] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 1024 MAX_TOOL_ITERATIONS = 2 +[dialectic.levels.medium.model_config] +transport = "openai" +model = "gpt-5.4-mini" + [dialectic.levels.high] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 1024 MAX_TOOL_ITERATIONS = 4 +[dialectic.levels.high.model_config] +transport = "openai" +model = "gpt-5.4-mini" + [dialectic.levels.max] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 2048 MAX_TOOL_ITERATIONS = 10 + +[dialectic.levels.max.model_config] +transport = "openai" +model = "gpt-5.4-mini" ``` Environment variables for nested levels use double underscores: ```bash -DIALECTIC_LEVELS__minimal__PROVIDER=google -DIALECTIC_LEVELS__minimal__MODEL=gemini-2.5-flash-lite -DIALECTIC_LEVELS__minimal__THINKING_BUDGET_TOKENS=0 +DIALECTIC_LEVELS__minimal__MODEL_CONFIG__TRANSPORT=openai +DIALECTIC_LEVELS__minimal__MODEL_CONFIG__MODEL=gpt-5.4-mini DIALECTIC_LEVELS__minimal__MAX_TOOL_ITERATIONS=1 +DIALECTIC_LEVELS__minimal__MAX_OUTPUT_TOKENS=250 +DIALECTIC_LEVELS__minimal__TOOL_CHOICE=any ``` **Deriver (Theory of Mind):** @@ -222,12 +299,16 @@ The Deriver extracts facts from messages and builds theory-of-mind representatio DERIVER_ENABLED=true # LLM settings -DERIVER_PROVIDER=google -DERIVER_MODEL=gemini-2.5-flash-lite -DERIVER_MAX_OUTPUT_TOKENS=4096 -DERIVER_THINKING_BUDGET_TOKENS=1024 +DERIVER_MODEL_CONFIG__TRANSPORT=openai +DERIVER_MODEL_CONFIG__MODEL=gpt-5.4-mini DERIVER_MAX_INPUT_TOKENS=23000 -DERIVER_TEMPERATURE= # Optional override (unset by default) +# DERIVER_MODEL_CONFIG__THINKING_EFFORT=minimal +# DERIVER_MODEL_CONFIG__THINKING_BUDGET_TOKENS=1024 +# DERIVER_MODEL_CONFIG__TEMPERATURE=0.7 # Optional temperature override + +# Backup model (optional) +# DERIVER_MODEL_CONFIG__FALLBACK__MODEL=claude-haiku-4-5 +# DERIVER_MODEL_CONFIG__FALLBACK__TRANSPORT=anthropic # Worker settings DERIVER_WORKERS=1 # Increase for higher throughput @@ -256,11 +337,12 @@ Session summaries provide compressed context for long conversations — short su ```bash SUMMARY_ENABLED=true -SUMMARY_PROVIDER=google -SUMMARY_MODEL=gemini-2.5-flash +SUMMARY_MODEL_CONFIG__TRANSPORT=openai +SUMMARY_MODEL_CONFIG__MODEL=gpt-5.4-mini SUMMARY_MAX_TOKENS_SHORT=1000 SUMMARY_MAX_TOKENS_LONG=4000 -SUMMARY_THINKING_BUDGET_TOKENS=512 +# SUMMARY_MODEL_CONFIG__THINKING_EFFORT=minimal +# SUMMARY_MODEL_CONFIG__THINKING_BUDGET_TOKENS=1024 SUMMARY_MESSAGES_PER_SHORT_SUMMARY=20 SUMMARY_MESSAGES_PER_LONG_SUMMARY=60 ``` @@ -275,18 +357,14 @@ DREAM_DOCUMENT_THRESHOLD=50 DREAM_IDLE_TIMEOUT_MINUTES=60 DREAM_MIN_HOURS_BETWEEN_DREAMS=8 DREAM_ENABLED_TYPES=["omni"] - -# LLM settings -DREAM_PROVIDER=anthropic -DREAM_MODEL=claude-sonnet-4-20250514 -DREAM_MAX_OUTPUT_TOKENS=16384 -DREAM_THINKING_BUDGET_TOKENS=8192 DREAM_MAX_TOOL_ITERATIONS=20 DREAM_HISTORY_TOKEN_LIMIT=16384 -# Specialist models (use same provider as main model) -DREAM_DEDUCTION_MODEL=claude-haiku-4-5 -DREAM_INDUCTION_MODEL=claude-haiku-4-5 +# Specialist model configs (each is independent) +DREAM_DEDUCTION_MODEL_CONFIG__TRANSPORT=openai +DREAM_DEDUCTION_MODEL_CONFIG__MODEL=gpt-5.4-mini +DREAM_INDUCTION_MODEL_CONFIG__TRANSPORT=openai +DREAM_INDUCTION_MODEL_CONFIG__MODEL=gpt-5.4-mini ``` **Surprisal-Based Sampling (Advanced):** @@ -315,8 +393,8 @@ GET_CONTEXT_MAX_TOKENS=100000 MAX_MESSAGE_SIZE=25000 MAX_FILE_SIZE=5242880 # 5MB EMBED_MESSAGES=true -MAX_EMBEDDING_TOKENS=8192 -MAX_EMBEDDING_TOKENS_PER_REQUEST=300000 +EMBEDDING_MAX_INPUT_TOKENS=8192 +EMBEDDING_MAX_TOKENS_PER_REQUEST=300000 NAMESPACE=honcho ``` @@ -452,8 +530,10 @@ DEFAULT_TTL_SECONDS = 300 [deriver] ENABLED = true WORKERS = 1 -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" + +[deriver.model_config] +transport = "openai" +model = "gpt-5.4-mini" [peer_card] ENABLED = true @@ -462,44 +542,62 @@ ENABLED = true MAX_OUTPUT_TOKENS = 8192 [dialectic.levels.minimal] -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" -THINKING_BUDGET_TOKENS = 0 MAX_TOOL_ITERATIONS = 1 +MAX_OUTPUT_TOKENS = 250 +TOOL_CHOICE = "any" + +[dialectic.levels.minimal.model_config] +transport = "openai" +model = "gpt-5.4-mini" [dialectic.levels.low] -PROVIDER = "google" -MODEL = "gemini-2.5-flash-lite" -THINKING_BUDGET_TOKENS = 0 MAX_TOOL_ITERATIONS = 5 +TOOL_CHOICE = "any" + +[dialectic.levels.low.model_config] +transport = "openai" +model = "gpt-5.4-mini" [dialectic.levels.medium] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 1024 MAX_TOOL_ITERATIONS = 2 +[dialectic.levels.medium.model_config] +transport = "openai" +model = "gpt-5.4-mini" + [dialectic.levels.high] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 1024 MAX_TOOL_ITERATIONS = 4 +[dialectic.levels.high.model_config] +transport = "openai" +model = "gpt-5.4-mini" + [dialectic.levels.max] -PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 2048 MAX_TOOL_ITERATIONS = 10 +[dialectic.levels.max.model_config] +transport = "openai" +model = "gpt-5.4-mini" + [summary] ENABLED = true -PROVIDER = "google" -MODEL = "gemini-2.5-flash" +MAX_TOKENS_SHORT = 1000 +MAX_TOKENS_LONG = 4000 + +[summary.model_config] +transport = "openai" +model = "gpt-5.4-mini" [dream] ENABLED = true -PROVIDER = "anthropic" -MODEL = "claude-sonnet-4-20250514" + +[dream.deduction_model_config] +transport = "openai" +model = "gpt-5.4-mini" + +[dream.induction_model_config] +transport = "openai" +model = "gpt-5.4-mini" [webhook] MAX_WORKSPACE_LIMIT = 10 @@ -536,6 +634,6 @@ uv run alembic revision --autogenerate -m "Description" # Create new migration 4. **Deriver not processing** — Check logs. Increase `DERIVER_WORKERS` for throughput. Verify database and LLM connectivity. -5. **Dialectic level issues** — All five levels must be configured. For Anthropic, `THINKING_BUDGET_TOKENS` must be >= 1024. For non-Anthropic providers, set to `0`. `MAX_OUTPUT_TOKENS` must exceed `THINKING_BUDGET_TOKENS`. +5. **Dialectic level issues** — Unset level fields inherit from the built-in defaults. For Anthropic, `THINKING_BUDGET_TOKENS` must be >= 1024 when enabled. For providers without budgeted thinking, omit it or set it to `0`. `MAX_OUTPUT_TOKENS` must exceed `THINKING_BUDGET_TOKENS`. 6. **Vector store issues** — For Turbopuffer, set the API key. Check `VECTOR_STORE_DIMENSIONS` matches your embedding model. diff --git a/docs/v3/contributing/self-hosting.mdx b/docs/v3/contributing/self-hosting.mdx index fc298bd8..975f9d76 100644 --- a/docs/v3/contributing/self-hosting.mdx +++ b/docs/v3/contributing/self-hosting.mdx @@ -36,30 +36,24 @@ You'll need a PostgreSQL database with the pgvector extension. Choose one: Honcho uses LLMs for memory extraction, summarization, dialectic chat, and dreaming. The server will **fail to start** without a provider configured. -You need one API key and one model. Any OpenAI-compatible endpoint works — OpenRouter, Together, Fireworks, Ollama, vLLM, or a direct vendor API. Models must support tool calling (function calling). +If you keep the built-in defaults, you only need one API key: all text-generation features default to `openai / gpt-5.4-mini`, and embeddings default to `openai / text-embedding-3-small`. Any OpenAI-compatible endpoint works too — OpenRouter, Together, Fireworks, Ollama, vLLM, or LiteLLM. Models must support tool calling (function calling). -The `.env.template` has provider and model lines ready for each feature. After copying it to `.env`, you need to set three things: +After copying `.env.template` to `.env`, the default setup is: ```bash -# 1. Your endpoint and API key (already uncommented in the template) -LLM_OPENAI_COMPATIBLE_BASE_URL=https://openrouter.ai/api/v1 -LLM_OPENAI_COMPATIBLE_API_KEY=sk-or-v1-... - -# 2. Replace "your-model-here" everywhere with your model -# (these are spread across the Deriver, Dialectic, Summary, and Dream sections) -DERIVER_MODEL=google/gemini-2.5-flash # e.g. google/gemini-2.5-flash -SUMMARY_MODEL=google/gemini-2.5-flash -DREAM_MODEL=google/gemini-2.5-flash -DIALECTIC_LEVELS__minimal__MODEL=google/gemini-2.5-flash -# ... same for low, medium, high, max - -# 3. Everything else is already configured: -# - PROVIDER=custom for all features (routes through your endpoint) -# - THINKING_BUDGET_TOKENS=0 (correct for non-Anthropic models) -# - LLM_EMBEDDING_PROVIDER=openrouter (uses same endpoint for embeddings) +# Required for the built-in defaults +LLM_OPENAI_API_KEY=sk-... ``` -Use find-and-replace to swap all `your-model-here` with your chosen model in one step. +If you want a different model or an OpenAI-compatible proxy, uncomment and edit the relevant `*_MODEL_CONFIG__TRANSPORT`, `*_MODEL_CONFIG__MODEL`, and `*_MODEL_CONFIG__OVERRIDES__BASE_URL` lines in the Deriver, Dialectic, Summary, and Dream sections. For example: + +```bash +LLM_OPENAI_API_KEY=sk-or-v1-... + +DERIVER_MODEL_CONFIG__TRANSPORT=openai +DERIVER_MODEL_CONFIG__MODEL=google/gemini-2.5-flash +DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 +``` For recommended model tiers per feature, using multiple providers, or direct vendor API keys, see the [Configuration Guide](./configuration#llm-configuration). diff --git a/docs/v3/contributing/troubleshooting.mdx b/docs/v3/contributing/troubleshooting.mdx index f041e2db..bb71a475 100644 --- a/docs/v3/contributing/troubleshooting.mdx +++ b/docs/v3/contributing/troubleshooting.mdx @@ -115,12 +115,12 @@ Messages are stored but no observations, summaries, or representations are being ### OpenRouter / custom provider not working -If you set `PROVIDER=custom` but calls fail: +If calls to an OpenAI-compatible proxy fail: -1. **Verify the endpoint and key are set:** +1. **Verify the endpoint and key are set.** Use `transport = "openai"` with a base URL override: ```bash - LLM_OPENAI_COMPATIBLE_BASE_URL=https://openrouter.ai/api/v1 - LLM_OPENAI_COMPATIBLE_API_KEY=sk-or-v1-... + LLM_OPENAI_API_KEY=sk-or-v1-... + DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 ``` 2. **Check model names match the provider's format.** OpenRouter uses `vendor/model` format (e.g., `anthropic/claude-haiku-4-5`), not the raw model ID. @@ -139,29 +139,30 @@ If you set `PROVIDER=custom` but calls fail: 2. **In Docker**, `localhost` inside a container doesn't reach the host. Use `host.docker.internal` (macOS/Windows) or the host's network IP: ```bash - LLM_VLLM_BASE_URL=http://host.docker.internal:8000/v1 + DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=http://host.docker.internal:8000/v1 ``` 3. **Structured output failures** — vLLM's structured output support is limited to certain response formats. If you see JSON parsing errors, check the deriver/dream logs for the raw response. ### Thinking budget errors with non-Anthropic providers -If you see errors like `thinking budget not supported`, `invalid parameter`, or silent failures where agents produce no output, your `THINKING_BUDGET_TOKENS` is likely set to a value > 0 with a provider that doesn't support Anthropic-style extended thinking. +If you see errors like `thinking budget not supported`, `invalid parameter`, or silent failures where agents produce no output, one of your per-component `*_MODEL_CONFIG__THINKING_BUDGET_TOKENS` overrides is likely set to a value > 0 with a provider that doesn't support Anthropic-style extended thinking. The built-in defaults do not set thinking budgets, so this only applies if you added those overrides yourself. -**Fix:** Set `THINKING_BUDGET_TOKENS=0` for every component when using non-Anthropic providers: +**Fix:** Set `*_MODEL_CONFIG__THINKING_BUDGET_TOKENS=0` for every component when using models that don't support thinking: ```bash -DERIVER_THINKING_BUDGET_TOKENS=0 -SUMMARY_THINKING_BUDGET_TOKENS=0 -DREAM_THINKING_BUDGET_TOKENS=0 -DIALECTIC_LEVELS__minimal__THINKING_BUDGET_TOKENS=0 -DIALECTIC_LEVELS__low__THINKING_BUDGET_TOKENS=0 -DIALECTIC_LEVELS__medium__THINKING_BUDGET_TOKENS=0 -DIALECTIC_LEVELS__high__THINKING_BUDGET_TOKENS=0 -DIALECTIC_LEVELS__max__THINKING_BUDGET_TOKENS=0 +DERIVER_MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +SUMMARY_MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +DREAM_DEDUCTION_MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +DREAM_INDUCTION_MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +DIALECTIC_LEVELS__minimal__MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +DIALECTIC_LEVELS__low__MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +DIALECTIC_LEVELS__medium__MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +DIALECTIC_LEVELS__high__MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 +DIALECTIC_LEVELS__max__MODEL_CONFIG__THINKING_BUDGET_TOKENS=0 ``` -This applies to OpenRouter (with non-Anthropic models), vLLM, Ollama, Groq, Google, and OpenAI providers. Only Anthropic models support the thinking budget parameter. +For OpenAI reasoning models, use `*_MODEL_CONFIG__THINKING_EFFORT` instead of `*_MODEL_CONFIG__THINKING_BUDGET_TOKENS`. ## Database Issues diff --git a/docs/v3/guides/integrations/paperclip.mdx b/docs/v3/guides/integrations/paperclip.mdx index de579246..3b2aa156 100644 --- a/docs/v3/guides/integrations/paperclip.mdx +++ b/docs/v3/guides/integrations/paperclip.mdx @@ -57,7 +57,7 @@ The current plugin gives agent peers explicit observation settings: - `observe_me` defaults to `true` - `observe_others` defaults to `true` -In practice, that means agent peers can both be observed by Honcho and form representations of other peers they interact with. +In practice, that means agent peers can both be observed by Honcho and form representations of other peers they interact with. ## How It Works diff --git a/pyproject.toml b/pyproject.toml index 7b0fca89..228ca719 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "fastapi[standard]>=0.131.0", - "groq>=0.31.0", "python-dotenv>=1.0.0", "sqlalchemy>=2.0.30", "fastapi-pagination>=0.14.2", @@ -95,6 +94,12 @@ asyncio_default_fixture_loop_scope = "session" addopts = "--strict-markers -n auto --ignore=tests/alembic" testpaths = ["tests"] pythonpath = ["src"] +markers = [ + "live_llm: calls live LLM provider APIs and requires --live-llm", + "requires_anthropic: requires LLM_ANTHROPIC_API_KEY", + "requires_openai: requires LLM_OPENAI_API_KEY", + "requires_gemini: requires LLM_GEMINI_API_KEY", +] filterwarnings = [ "ignore:Call to deprecated close\\. \\(Use aclose\\(\\) instead\\).*:DeprecationWarning", "ignore:websockets\\.legacy is deprecated; see .* for upgrade instructions:DeprecationWarning", diff --git a/src/config.py b/src/config.py index f01c451b..cae0c5de 100644 --- a/src/config.py +++ b/src/config.py @@ -1,10 +1,11 @@ import logging +import os from pathlib import Path -from typing import Annotated, Any, ClassVar, Literal, Protocol +from typing import Annotated, Any, ClassVar, Literal, cast import tomllib from dotenv import load_dotenv -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator from pydantic.fields import FieldInfo from pydantic_settings import ( BaseSettings, @@ -14,17 +15,27 @@ from pydantic_settings import ( SettingsConfigDict, ) -from src.utils.types import SupportedProviders - # Load .env file for local development. # Make sure this is called before AppSettings is instantiated if you rely on .env for AppSettings construction. -load_dotenv(override=True) +if not os.getenv("PYTHON_DOTENV_DISABLED"): + load_dotenv(override=True) logger = logging.getLogger(__name__) +ModelTransport = Literal["anthropic", "openai", "gemini"] +EmbeddingTransport = Literal["openai", "gemini"] + + +def _default_embedding_model_for_transport(transport: EmbeddingTransport) -> str: + if transport == "gemini": + return "gemini-embedding-001" + return "text-embedding-3-small" + def load_toml_config(config_path: str = "config.toml") -> dict[str, Any]: """Load configuration from TOML file if it exists.""" + if config_path == "config.toml" and os.getenv("HONCHO_CONFIG_TOML_DISABLED"): + return {} config_file = Path(config_path) if config_file.exists(): try: @@ -40,13 +51,463 @@ def load_toml_config(config_path: str = "config.toml") -> dict[str, Any]: TOML_CONFIG = load_toml_config() -class LLMComponentSettings(Protocol): - """Protocol for settings classes that use LLM providers with backup support.""" +ThinkingEffortLevel = Literal[ + "none", "minimal", "low", "medium", "high", "xhigh", "max" +] - PROVIDER: SupportedProviders - MODEL: str - BACKUP_PROVIDER: SupportedProviders | None - BACKUP_MODEL: str | None + +class ModelOverrideSettings(BaseModel): + """Advanced module-level transport overrides.""" + + api_key: str | None = None + api_key_env: str | None = None + base_url: str | None = None + + provider_params: dict[str, Any] = Field(default_factory=dict) + + +class PromptCachePolicy(BaseModel): + """Per-call prompt-caching configuration. + + Lives in config.py (not src/llm/caching.py) so ModelConfig can reference + it as a field without a circular import. src/llm/caching.py re-exports + this class for existing import paths. + """ + + mode: Literal["none", "prefix", "gemini_cached_content"] = "none" + ttl_seconds: int | None = None + key_version: str = "v1" + + +def _normalize_model_transport(data: Any) -> Any: + """Normalize 'provider/model' shorthand into separate transport + model fields.""" + if not isinstance(data, dict): + return data + raw_data = cast(dict[Any, Any], data) + update: dict[str, Any] = {str(key): value for key, value in raw_data.items()} + model_value = update.get("model") + transport_value = update.get("transport") + if isinstance(model_value, str) and "/" in model_value and transport_value is None: + prefix, bare_model = model_value.split("/", 1) + if prefix in {"anthropic", "openai", "gemini"}: + update["transport"] = prefix + update["model"] = bare_model + return update + + +def _validate_thinking_constraints( + transport: ModelTransport, thinking_budget_tokens: int | None +) -> None: + """Enforce transport-specific thinking_budget_tokens rules. + + Anthropic requires a minimum of 1024 tokens when thinking is enabled. + Gemini/OpenAI accept any non-negative value (including 0 to disable). + """ + if ( + transport == "anthropic" + and thinking_budget_tokens is not None + and 0 < thinking_budget_tokens < 1024 + ): + raise ValueError("thinking_budget_tokens must be >= 1024 for Anthropic models") + + +class FallbackModelSettings(BaseModel): + """Independent fallback model configuration. No inheritance from primary.""" + + model: str + transport: ModelTransport + + temperature: float | None = None + top_p: float | None = None + top_k: int | None = None + frequency_penalty: float | None = None + presence_penalty: float | None = None + seed: int | None = None + + thinking_effort: ThinkingEffortLevel | None = Field( + default=None, + validation_alias=AliasChoices("thinking_effort", "reasoning_effort"), + ) + thinking_budget_tokens: int | None = None + + max_output_tokens: int | None = None + stop_sequences: list[str] | None = None + + cache_policy: PromptCachePolicy | None = None + + overrides: ModelOverrideSettings = Field(default_factory=ModelOverrideSettings) + + @model_validator(mode="before") + @classmethod + def _normalize_legacy_model_format(cls, data: Any) -> Any: + return _normalize_model_transport(data) + + @property + def reasoning_effort(self) -> ThinkingEffortLevel | None: + return self.thinking_effort + + @model_validator(mode="after") + def _validate_runtime_shape(self) -> "FallbackModelSettings": + _validate_thinking_constraints(self.transport, self.thinking_budget_tokens) + return self + + +class ConfiguredModelSettings(BaseModel): + """Operator-configurable persisted model settings.""" + + model: str + transport: ModelTransport + + fallback: FallbackModelSettings | None = None + + temperature: float | None = None + top_p: float | None = None + top_k: int | None = None + frequency_penalty: float | None = None + presence_penalty: float | None = None + seed: int | None = None + + thinking_effort: ThinkingEffortLevel | None = Field( + default=None, + validation_alias=AliasChoices("thinking_effort", "reasoning_effort"), + ) + thinking_budget_tokens: int | None = None + + max_output_tokens: int | None = None + stop_sequences: list[str] | None = None + + cache_policy: PromptCachePolicy | None = None + + overrides: ModelOverrideSettings = Field(default_factory=ModelOverrideSettings) + + @model_validator(mode="before") + @classmethod + def _normalize_legacy_model_format(cls, data: Any) -> Any: + return _normalize_model_transport(data) + + @property + def reasoning_effort(self) -> ThinkingEffortLevel | None: + """Backward-compatible alias for the generic thinking effort field.""" + return self.thinking_effort + + @model_validator(mode="after") + def _validate_runtime_shape(self) -> "ConfiguredModelSettings": + _validate_thinking_constraints(self.transport, self.thinking_budget_tokens) + return self + + +class ResolvedFallbackConfig(BaseModel): + """Runtime-resolved fallback config with credentials already resolved.""" + + model: str + transport: ModelTransport + + api_key: str | None = None + base_url: str | None = None + + temperature: float | None = None + top_p: float | None = None + top_k: int | None = None + frequency_penalty: float | None = None + presence_penalty: float | None = None + seed: int | None = None + + thinking_effort: ThinkingEffortLevel | None = Field( + default=None, + validation_alias=AliasChoices("thinking_effort", "reasoning_effort"), + ) + thinking_budget_tokens: int | None = None + provider_params: dict[str, Any] = Field(default_factory=dict) + + max_output_tokens: int | None = None + stop_sequences: list[str] | None = None + + cache_policy: PromptCachePolicy | None = None + + @property + def reasoning_effort(self) -> ThinkingEffortLevel | None: + return self.thinking_effort + + +class ModelConfig(BaseModel): + """Reusable model configuration for any non-embedding LLM caller.""" + + model: str + transport: ModelTransport + + fallback: ResolvedFallbackConfig | None = None + + api_key: str | None = None + base_url: str | None = None + + temperature: float | None = None + top_p: float | None = None + top_k: int | None = None + frequency_penalty: float | None = None + presence_penalty: float | None = None + seed: int | None = None + + thinking_effort: ThinkingEffortLevel | None = Field( + default=None, + validation_alias=AliasChoices("thinking_effort", "reasoning_effort"), + ) + thinking_budget_tokens: int | None = None + provider_params: dict[str, Any] = Field(default_factory=dict) + + max_output_tokens: int | None = None + stop_sequences: list[str] | None = None + + cache_policy: PromptCachePolicy | None = None + + @model_validator(mode="before") + @classmethod + def _normalize_legacy_model_format(cls, data: Any) -> Any: + return _normalize_model_transport(data) + + @property + def reasoning_effort(self) -> ThinkingEffortLevel | None: + """Backward-compatible alias for the generic thinking effort field.""" + return self.thinking_effort + + @model_validator(mode="after") + def _validate_thinking_constraints_on_self(self) -> "ModelConfig": + _validate_thinking_constraints(self.transport, self.thinking_budget_tokens) + return self + + def for_model( + self, + model_override: str, + *, + transport_override: ModelTransport | None = None, + ) -> "ModelConfig": + return self.model_copy( + update={ + "model": model_override, + "transport": transport_override or self.transport, + } + ) + + +class ConfiguredEmbeddingModelSettings(BaseModel): + """Operator-configurable persisted embedding settings.""" + + model: str = "text-embedding-3-small" + transport: EmbeddingTransport = "openai" + overrides: ModelOverrideSettings = Field(default_factory=ModelOverrideSettings) + + @model_validator(mode="before") + @classmethod + def _normalize_legacy_model_format(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + + raw_data = cast(dict[Any, Any], data) + update: dict[str, Any] = {str(key): value for key, value in raw_data.items()} + model_value = update.get("model") + transport_value = update.get("transport") + if ( + isinstance(model_value, str) + and "/" in model_value + and transport_value is None + ): + prefix, bare_model = model_value.split("/", 1) + if prefix in {"openai", "gemini"}: + update["transport"] = prefix + update["model"] = bare_model + return update + + @model_validator(mode="after") + def _default_model_for_transport(self) -> "ConfiguredEmbeddingModelSettings": + if "model" not in self.model_fields_set: + self.model = _default_embedding_model_for_transport(self.transport) + return self + + +class EmbeddingModelConfig(BaseModel): + """Runtime embedding configuration with resolved credentials.""" + + model: str = "text-embedding-3-small" + transport: EmbeddingTransport = "openai" + api_key: str | None = None + base_url: str | None = None + + @model_validator(mode="before") + @classmethod + def _normalize_legacy_model_format(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + + raw_data = cast(dict[Any, Any], data) + update: dict[str, Any] = {str(key): value for key, value in raw_data.items()} + model_value = update.get("model") + transport_value = update.get("transport") + if ( + isinstance(model_value, str) + and "/" in model_value + and transport_value is None + ): + prefix, bare_model = model_value.split("/", 1) + if prefix in {"openai", "gemini"}: + update["transport"] = prefix + update["model"] = bare_model + return update + + @model_validator(mode="after") + def _default_model_for_transport(self) -> "EmbeddingModelConfig": + if "model" not in self.model_fields_set: + self.model = _default_embedding_model_for_transport(self.transport) + return self + + +def _resolve_secret(value: str | None, env_name: str | None) -> str | None: + if value is not None: + return value + if env_name is None: + return None + return os.getenv(env_name) + + +def _resolve_fallback_config( + fallback: FallbackModelSettings, +) -> ResolvedFallbackConfig: + """Resolve a FallbackModelSettings into a runtime ResolvedFallbackConfig.""" + return ResolvedFallbackConfig( + model=fallback.model, + transport=fallback.transport, + api_key=_resolve_secret( + fallback.overrides.api_key, + fallback.overrides.api_key_env, + ), + base_url=fallback.overrides.base_url, + temperature=fallback.temperature, + top_p=fallback.top_p, + top_k=fallback.top_k, + frequency_penalty=fallback.frequency_penalty, + presence_penalty=fallback.presence_penalty, + seed=fallback.seed, + thinking_effort=fallback.thinking_effort, + thinking_budget_tokens=fallback.thinking_budget_tokens, + provider_params=fallback.overrides.provider_params, + max_output_tokens=fallback.max_output_tokens, + stop_sequences=fallback.stop_sequences, + cache_policy=fallback.cache_policy, + ) + + +def resolve_model_config(configured: ConfiguredModelSettings) -> ModelConfig: + """Resolve persisted model settings into the runtime ModelConfig.""" + + resolved_fallback = ( + _resolve_fallback_config(configured.fallback) + if configured.fallback is not None + else None + ) + + return ModelConfig( + model=configured.model, + transport=configured.transport, + fallback=resolved_fallback, + api_key=_resolve_secret( + configured.overrides.api_key, + configured.overrides.api_key_env, + ), + base_url=configured.overrides.base_url, + temperature=configured.temperature, + top_p=configured.top_p, + top_k=configured.top_k, + frequency_penalty=configured.frequency_penalty, + presence_penalty=configured.presence_penalty, + seed=configured.seed, + thinking_effort=configured.thinking_effort, + thinking_budget_tokens=configured.thinking_budget_tokens, + provider_params=configured.overrides.provider_params, + max_output_tokens=configured.max_output_tokens, + stop_sequences=configured.stop_sequences, + cache_policy=configured.cache_policy, + ) + + +def _default_embedding_api_key(transport: EmbeddingTransport) -> str | None: + """Fall back to the global LLM API key for the matching transport.""" + if transport == "openai": + return settings.LLM.OPENAI_API_KEY + if transport == "gemini": + return settings.LLM.GEMINI_API_KEY + + +def resolve_embedding_model_config( + configured: ConfiguredEmbeddingModelSettings, +) -> EmbeddingModelConfig: + """Resolve persisted embedding settings into the runtime config.""" + + api_key = _resolve_secret( + configured.overrides.api_key, + configured.overrides.api_key_env, + ) + if api_key is None: + api_key = _default_embedding_api_key(configured.transport) + + return EmbeddingModelConfig( + model=configured.model, + transport=configured.transport, + api_key=api_key, + base_url=configured.overrides.base_url, + ) + + +_TRANSPORT_SPECIFIC_THINKING_KEYS: frozenset[str] = frozenset( + {"thinking_budget_tokens", "thinking_effort"} +) + + +def _fill_defaults_for_nested_field( + data: dict[str, Any], + field_name: str, + default_factory: Any, +) -> dict[str, Any]: + """Fill missing keys in a partial nested dict from the field's defaults. + + When Pydantic's env_nested_delimiter splits an env var like + ``DERIVER_MODEL_CONFIG__THINKING_BUDGET_TOKENS=2048`` it produces + ``{"MODEL_CONFIG": {"THINKING_BUDGET_TOKENS": 2048}}``. Without merging + that partial dict would fail validation because required keys like + ``model`` and ``transport`` are missing. This helper fills them from + the field's ``default_factory`` so partial overrides work. + + If the env override switches ``transport`` to a value that differs from + the default's, transport-specific thinking params + (``thinking_budget_tokens``, ``thinking_effort``) are dropped from the + default before merging. This prevents e.g. a Gemini default's + ``thinking_budget_tokens=1024`` from leaking into an OpenAI override, + which would then be rejected by the OpenAI backend (OpenAI uses + ``reasoning.effort``, not a token budget). Explicit thinking params in + the env override are preserved. + """ + raw: Any = data.get(field_name) or data.get(field_name.lower()) + if not isinstance(raw, dict): + return data + + default_obj = default_factory() + if isinstance(default_obj, BaseModel): + default_dict: dict[str, Any] = default_obj.model_dump(by_alias=True) + else: + default_dict = dict(default_obj) + + raw_dict = cast(dict[str, Any], raw) + raw_lower = {k.lower(): v for k, v in raw_dict.items()} + default_lower = {k.lower(): v for k, v in default_dict.items()} + override_transport = raw_lower.get("transport") + default_transport = default_lower.get("transport") + if override_transport is not None and override_transport != default_transport: + for k in list(default_dict.keys()): + if k.lower() in _TRANSPORT_SPECIFIC_THINKING_KEYS: + del default_dict[k] + + merged: dict[str, Any] = {**default_dict, **raw_dict} + # Preserve the key casing used in data + key = field_name if field_name in data else field_name.lower() + data[key] = merged + return data class TomlConfigSettingsSource(PydanticBaseSettingsSource): @@ -61,6 +522,7 @@ class TomlConfigSettingsSource(PydanticBaseSettingsSource): "SENTRY": "sentry", "CACHE": "cache", "LLM": "llm", + "EMBEDDING": "embedding", "DERIVER": "deriver", "PEER_CARD": "peer_card", "DIALECTIC": "dialectic", @@ -132,26 +594,6 @@ class HonchoSettings(BaseSettings): ) -class BackupLLMSettingsMixin: - """Mixin class for settings that support backup LLM provider configuration. - - Provides backup provider and model fields along with validation to ensure - both fields are set together or both are None. - """ - - BACKUP_PROVIDER: SupportedProviders | None = None - BACKUP_MODEL: str | None = None - - @model_validator(mode="after") - def _validate_backup_configuration(self): - """Ensure both backup fields are set together or both are None.""" - if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None): - raise ValueError( - "BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None" - ) - return self - - class DBSettings(HonchoSettings): model_config = SettingsConfigDict(env_prefix="DB_", extra="ignore") # pyright: ignore @@ -204,16 +646,7 @@ class LLMSettings(HonchoSettings): # API Keys for LLM providers ANTHROPIC_API_KEY: str | None = None OPENAI_API_KEY: str | None = None - OPENAI_COMPATIBLE_API_KEY: str | None = None GEMINI_API_KEY: str | None = None - GROQ_API_KEY: str | None = None - OPENAI_COMPATIBLE_BASE_URL: str | None = None - - # Separate vLLM endpoint (for local models) - VLLM_API_KEY: str | None = None - VLLM_BASE_URL: str | None = None - - EMBEDDING_PROVIDER: Literal["openai", "gemini", "openrouter"] = "openai" # General LLM settings DEFAULT_MAX_TOKENS: Annotated[int, Field(default=1000, gt=0, le=100_000)] = 2500 @@ -232,8 +665,41 @@ class LLMSettings(HonchoSettings): ) -class DeriverSettings(BackupLLMSettingsMixin, HonchoSettings): - model_config = SettingsConfigDict(env_prefix="DERIVER_", extra="ignore") # pyright: ignore +class EmbeddingSettings(HonchoSettings): + model_config = SettingsConfigDict( # pyright: ignore + env_prefix="EMBEDDING_", env_nested_delimiter="__", extra="ignore" + ) + + @staticmethod + def _MODEL_CONFIG_DEFAULT() -> ConfiguredEmbeddingModelSettings: + return ConfiguredEmbeddingModelSettings( + transport="openai", + model="text-embedding-3-small", + ) + + MODEL_CONFIG: ConfiguredEmbeddingModelSettings = Field( + default_factory=_MODEL_CONFIG_DEFAULT + ) + VECTOR_DIMENSIONS: Annotated[int, Field(default=1536, gt=0)] = 1536 + MAX_INPUT_TOKENS: Annotated[int, Field(default=8192, gt=0)] = 8192 + MAX_TOKENS_PER_REQUEST: Annotated[int, Field(default=300_000, gt=0)] = 300_000 + + @model_validator(mode="before") + @classmethod + def _merge_model_config_defaults(cls, data: Any) -> Any: + if isinstance(data, dict): + _fill_defaults_for_nested_field( + cast(dict[str, Any], data), + "MODEL_CONFIG", + cls._MODEL_CONFIG_DEFAULT, + ) + return data # pyright: ignore[reportUnknownVariableType] + + +class DeriverSettings(HonchoSettings): + model_config = SettingsConfigDict( # pyright: ignore + env_prefix="DERIVER_", env_nested_delimiter="__", extra="ignore" + ) ENABLED: bool = True @@ -248,16 +714,21 @@ class DeriverSettings(BackupLLMSettingsMixin, HonchoSettings): int, Field(default=30 * 24 * 3600, gt=0) ] = 30 * 24 * 3600 # 30 days default - PROVIDER: SupportedProviders = "google" - MODEL: str = "gemini-2.5-flash-lite" - TEMPERATURE: float | None = None + @staticmethod + def _MODEL_CONFIG_DEFAULT() -> ConfiguredModelSettings: + # Minimal default: transport + model only. Any other knobs would merge + # into operator-supplied env / config.toml overrides via + # _fill_defaults_for_nested_field and clobber intent. + return ConfiguredModelSettings( + transport="openai", + model="gpt-5.4-mini", + ) + + MODEL_CONFIG: ConfiguredModelSettings = Field(default_factory=_MODEL_CONFIG_DEFAULT) # Whether to deduplicate documents when creating them DEDUPLICATE: bool = True - MAX_OUTPUT_TOKENS: Annotated[int, Field(default=4096, gt=0, le=100_000)] = 4096 - THINKING_BUDGET_TOKENS: Annotated[int, Field(default=1024, gt=0, le=5000)] = 1024 - LOG_OBSERVATIONS: bool = False MAX_INPUT_TOKENS: Annotated[int, Field(default=23000, gt=0, le=23000)] = 23000 @@ -276,6 +747,17 @@ class DeriverSettings(BackupLLMSettingsMixin, HonchoSettings): # When enabled, bypasses the batch token threshold and processes work immediately FLUSH_ENABLED: bool = False + @model_validator(mode="before") + @classmethod + def _merge_model_config_defaults(cls, data: Any) -> Any: + if isinstance(data, dict): + _fill_defaults_for_nested_field( + cast(dict[str, Any], data), + "MODEL_CONFIG", + cls._MODEL_CONFIG_DEFAULT, + ) + return data # pyright: ignore[reportUnknownVariableType] + @model_validator(mode="after") def validate_batch_tokens_vs_context_limit(self): if self.REPRESENTATION_BATCH_MAX_TOKENS > self.MAX_INPUT_TOKENS: @@ -307,14 +789,9 @@ class DialecticLevelSettings(BaseModel): model_config = SettingsConfigDict(populate_by_name=True) # pyright: ignore - PROVIDER: Annotated[SupportedProviders, Field(validation_alias="provider")] - MODEL: Annotated[str, Field(validation_alias="model")] - BACKUP_PROVIDER: Annotated[ - SupportedProviders | None, Field(validation_alias="backup_provider") - ] = None - BACKUP_MODEL: Annotated[str | None, Field(validation_alias="backup_model")] = None - THINKING_BUDGET_TOKENS: Annotated[ - int, Field(ge=0, le=100_000, validation_alias="thinking_budget_tokens") + MODEL_CONFIG: Annotated[ + ConfiguredModelSettings, + Field(validation_alias="model_config"), ] MAX_TOOL_ITERATIONS: Annotated[ int, Field(ge=0, le=50, validation_alias="max_tool_iterations") @@ -326,72 +803,69 @@ class DialecticLevelSettings(BaseModel): None # None/auto lets model decide, "any"/"required" forces tool use ) - @model_validator(mode="after") - def _validate_backup_configuration(self) -> "DialecticLevelSettings": - """Ensure both backup fields are set together or both are None.""" - if (self.BACKUP_PROVIDER is None) != (self.BACKUP_MODEL is None): - raise ValueError( - "BACKUP_PROVIDER and BACKUP_MODEL must both be set or both be None" - ) - return self - @model_validator(mode="after") def _validate_anthropic_thinking_budget(self) -> "DialecticLevelSettings": """Ensure Anthropic thinking budget is >= 1024 when enabled.""" if ( - self.PROVIDER == "anthropic" - and self.THINKING_BUDGET_TOKENS > 0 - and self.THINKING_BUDGET_TOKENS < 1024 + self.MODEL_CONFIG.transport == "anthropic" + and self.MODEL_CONFIG.thinking_budget_tokens is not None + and self.MODEL_CONFIG.thinking_budget_tokens > 0 + and self.MODEL_CONFIG.thinking_budget_tokens < 1024 ): raise ValueError( - f"THINKING_BUDGET_TOKENS must be >= 1024 for Anthropic provider when enabled (got {self.THINKING_BUDGET_TOKENS})" + "MODEL_CONFIG.thinking_budget_tokens must be >= 1024 for " + + "Anthropic models when enabled " + + f"(got {self.MODEL_CONFIG.thinking_budget_tokens})" ) return self +def _default_dialectic_levels() -> dict[ReasoningLevel, DialecticLevelSettings]: + # Minimal defaults per level: transport + model only. Non-MODEL_CONFIG + # level tuning (MAX_TOOL_ITERATIONS, MAX_OUTPUT_TOKENS, TOOL_CHOICE) + # stays here because it's the per-level behavior, not a model knob — + # operators still override any of it via + # DIALECTIC_LEVELS____MODEL_CONFIG__* without conflict. + def _default_model_config() -> ConfiguredModelSettings: + return ConfiguredModelSettings( + transport="openai", + model="gpt-5.4-mini", + ) + + return { + "minimal": DialecticLevelSettings( + MODEL_CONFIG=_default_model_config(), + MAX_TOOL_ITERATIONS=1, + MAX_OUTPUT_TOKENS=250, + TOOL_CHOICE="any", + ), + "low": DialecticLevelSettings( + MODEL_CONFIG=_default_model_config(), + MAX_TOOL_ITERATIONS=5, + TOOL_CHOICE="any", + ), + "medium": DialecticLevelSettings( + MODEL_CONFIG=_default_model_config(), + MAX_TOOL_ITERATIONS=2, + ), + "high": DialecticLevelSettings( + MODEL_CONFIG=_default_model_config(), + MAX_TOOL_ITERATIONS=4, + ), + "max": DialecticLevelSettings( + MODEL_CONFIG=_default_model_config(), + MAX_TOOL_ITERATIONS=10, + ), + } + + class DialecticSettings(HonchoSettings): model_config = SettingsConfigDict( # pyright: ignore env_prefix="DIALECTIC_", env_nested_delimiter="__", extra="ignore" ) - # Per-level settings for provider, model, thinking budget, and tool iterations - # TODO: Fill in appropriate values for each reasoning level LEVELS: dict[ReasoningLevel, DialecticLevelSettings] = Field( - default_factory=lambda: { - "minimal": DialecticLevelSettings( - PROVIDER="google", - MODEL="gemini-2.5-flash-lite", - THINKING_BUDGET_TOKENS=0, - MAX_TOOL_ITERATIONS=1, - MAX_OUTPUT_TOKENS=250, - TOOL_CHOICE="any", - ), - "low": DialecticLevelSettings( - PROVIDER="google", - MODEL="gemini-2.5-flash-lite", - THINKING_BUDGET_TOKENS=0, - MAX_TOOL_ITERATIONS=5, - TOOL_CHOICE="any", - ), - "medium": DialecticLevelSettings( - PROVIDER="anthropic", - MODEL="claude-haiku-4-5", - THINKING_BUDGET_TOKENS=1024, - MAX_TOOL_ITERATIONS=2, - ), - "high": DialecticLevelSettings( - PROVIDER="anthropic", - MODEL="claude-haiku-4-5", - THINKING_BUDGET_TOKENS=1024, - MAX_TOOL_ITERATIONS=4, - ), - "max": DialecticLevelSettings( - PROVIDER="anthropic", - MODEL="claude-haiku-4-5", - THINKING_BUDGET_TOKENS=2048, - MAX_TOOL_ITERATIONS=10, - ), - } + default_factory=_default_dialectic_levels ) MAX_OUTPUT_TOKENS: Annotated[int, Field(default=8192, gt=0, le=100_000)] = 8192 @@ -406,13 +880,68 @@ class DialecticSettings(HonchoSettings): int, Field(default=4_096, ge=0, le=16_384) ] = 4_096 + @model_validator(mode="before") + @classmethod + def _merge_level_defaults(cls, data: Any) -> Any: + """Merge partial level overrides with built-in defaults.""" + if not isinstance(data, dict): + return data + typed_data = cast(dict[str, Any], data) + levels_raw: dict[str, Any] | None = typed_data.get("LEVELS") or typed_data.get( + "levels" + ) + if not isinstance(levels_raw, dict): + return data # pyright: ignore[reportUnknownVariableType] + defaults = _default_dialectic_levels() + for level_name_key, level_override_val in levels_raw.items(): + level_name = str(level_name_key) + if not isinstance(level_override_val, dict): + continue + level_override = cast(dict[str, Any], level_override_val) + if level_name in defaults: + base: dict[str, Any] = defaults[level_name].model_dump(by_alias=True) + # Recursively merge nested MODEL_CONFIG / model_config too. + # model_dump() always produces the Python field name + # ("MODEL_CONFIG"), but TOML overrides arrive as lowercase + # ("model_config"). Check both casings in the override and + # resolve the base value from whichever casing is present. + for mc_key in ("MODEL_CONFIG", "model_config"): + if mc_key in level_override and isinstance( + level_override[mc_key], dict + ): + base_mc: dict[str, Any] = dict( + base.get("MODEL_CONFIG") or base.get("model_config") or {} + ) + override_mc = cast(dict[str, Any], level_override[mc_key]) + override_lower = {k.lower(): v for k, v in override_mc.items()} + base_lower = {k.lower(): v for k, v in base_mc.items()} + override_transport = override_lower.get("transport") + base_transport = base_lower.get("transport") + if ( + override_transport is not None + and override_transport != base_transport + ): + for k in list(base_mc.keys()): + if k.lower() in _TRANSPORT_SPECIFIC_THINKING_KEYS: + del base_mc[k] + level_override[mc_key] = {**base_mc, **override_mc} + levels_raw[level_name] = {**base, **level_override} + return data # pyright: ignore[reportUnknownVariableType] + @model_validator(mode="after") def _validate_token_budgets(self) -> "DialecticSettings": """Ensure the output token limit exceeds all thinking budgets.""" for level, level_settings in self.LEVELS.items(): - if self.MAX_OUTPUT_TOKENS <= level_settings.THINKING_BUDGET_TOKENS: + thinking_budget = level_settings.MODEL_CONFIG.thinking_budget_tokens or 0 + effective_max = ( + level_settings.MAX_OUTPUT_TOKENS + if level_settings.MAX_OUTPUT_TOKENS is not None + else self.MAX_OUTPUT_TOKENS + ) + if thinking_budget > 0 and thinking_budget >= effective_max: raise ValueError( - f"MAX_OUTPUT_TOKENS must be greater than THINKING_BUDGET_TOKENS for level '{level}'" + "MAX_OUTPUT_TOKENS must be greater than MODEL_CONFIG." + + f"thinking_budget_tokens for level '{level}'" ) return self @@ -425,21 +954,40 @@ class DialecticSettings(HonchoSettings): return self -class SummarySettings(BackupLLMSettingsMixin, HonchoSettings): - model_config = SettingsConfigDict(env_prefix="SUMMARY_", extra="ignore") # pyright: ignore +class SummarySettings(HonchoSettings): + model_config = SettingsConfigDict( # pyright: ignore + env_prefix="SUMMARY_", env_nested_delimiter="__", extra="ignore" + ) ENABLED: bool = True MESSAGES_PER_SHORT_SUMMARY: Annotated[int, Field(default=20, gt=0, le=100)] = 20 MESSAGES_PER_LONG_SUMMARY: Annotated[int, Field(default=60, gt=0, le=500)] = 60 - PROVIDER: SupportedProviders = "google" - MODEL: str = "gemini-2.5-flash" + @staticmethod + def _MODEL_CONFIG_DEFAULT() -> ConfiguredModelSettings: + # Minimal default; extra knobs would merge into env/TOML overrides. + return ConfiguredModelSettings( + transport="openai", + model="gpt-5.4-mini", + ) + + MODEL_CONFIG: ConfiguredModelSettings = Field(default_factory=_MODEL_CONFIG_DEFAULT) + + @model_validator(mode="before") + @classmethod + def _merge_model_config_defaults(cls, data: Any) -> Any: + if isinstance(data, dict): + _fill_defaults_for_nested_field( + cast(dict[str, Any], data), + "MODEL_CONFIG", + cls._MODEL_CONFIG_DEFAULT, + ) + return data # pyright: ignore[reportUnknownVariableType] + MAX_TOKENS_SHORT: Annotated[int, Field(default=1000, gt=0, le=10_000)] = 1000 MAX_TOKENS_LONG: Annotated[int, Field(default=4000, gt=0, le=20_000)] = 4000 - THINKING_BUDGET_TOKENS: Annotated[int, Field(default=512, gt=0, le=2000)] = 512 - class WebhookSettings(HonchoSettings): model_config = SettingsConfigDict(env_prefix="WEBHOOK_", extra="ignore") # pyright: ignore @@ -528,7 +1076,7 @@ class SurprisalSettings(BaseModel): INCLUDE_LEVELS: list[str] = ["explicit", "deductive"] -class DreamSettings(BackupLLMSettingsMixin, HonchoSettings): +class DreamSettings(HonchoSettings): model_config = SettingsConfigDict( # pyright: ignore env_prefix="DREAM_", env_nested_delimiter="__", extra="ignore" ) @@ -539,11 +1087,6 @@ class DreamSettings(BackupLLMSettingsMixin, HonchoSettings): MIN_HOURS_BETWEEN_DREAMS: Annotated[int, Field(default=8, gt=0, le=72)] = 8 ENABLED_TYPES: list[str] = ["omni"] - PROVIDER: SupportedProviders = "anthropic" - MODEL: str = "claude-sonnet-4-20250514" - MAX_OUTPUT_TOKENS: Annotated[int, Field(default=16_384, gt=0, le=64_000)] = 16_384 - THINKING_BUDGET_TOKENS: Annotated[int, Field(default=8192, gt=0, le=32_000)] = 8192 - # Agent iteration limit - increased for extended reasoning workflow MAX_TOOL_ITERATIONS: Annotated[int, Field(default=20, gt=0, le=50)] = 20 @@ -552,23 +1095,66 @@ class DreamSettings(BackupLLMSettingsMixin, HonchoSettings): 16_384 ) - ## NOTE: specialist models use the same provider as the main model + @staticmethod + def _DEDUCTION_MODEL_CONFIG_DEFAULT() -> ConfiguredModelSettings: + # Minimal default; extra knobs would merge into env/TOML overrides. + return ConfiguredModelSettings( + transport="openai", + model="gpt-5.4-mini", + ) - # Deduction Specialist: handles logical inference - DEDUCTION_MODEL: str = "claude-haiku-4-5" - # Induction Specialist: identifies patterns across observations - INDUCTION_MODEL: str = "claude-haiku-4-5" + DEDUCTION_MODEL_CONFIG: ConfiguredModelSettings = Field( + default_factory=_DEDUCTION_MODEL_CONFIG_DEFAULT + ) + + @staticmethod + def _INDUCTION_MODEL_CONFIG_DEFAULT() -> ConfiguredModelSettings: + # Minimal default; extra knobs would merge into env/TOML overrides. + return ConfiguredModelSettings( + transport="openai", + model="gpt-5.4-mini", + ) + + INDUCTION_MODEL_CONFIG: ConfiguredModelSettings = Field( + default_factory=_INDUCTION_MODEL_CONFIG_DEFAULT + ) # Surprisal-based sampling subsystem SURPRISAL: SurprisalSettings = Field(default_factory=SurprisalSettings) - @model_validator(mode="after") - def _validate_token_budgets(self) -> "DreamSettings": - """Ensure the output token limit exceeds the thinking budget.""" - if self.MAX_OUTPUT_TOKENS <= self.THINKING_BUDGET_TOKENS: - raise ValueError( - "MAX_OUTPUT_TOKENS must be greater than THINKING_BUDGET_TOKENS" + @model_validator(mode="before") + @classmethod + def _merge_model_config_defaults(cls, data: Any) -> Any: + if isinstance(data, dict): + typed_data = cast(dict[str, Any], data) + _fill_defaults_for_nested_field( + typed_data, + "DEDUCTION_MODEL_CONFIG", + cls._DEDUCTION_MODEL_CONFIG_DEFAULT, ) + _fill_defaults_for_nested_field( + typed_data, + "INDUCTION_MODEL_CONFIG", + cls._INDUCTION_MODEL_CONFIG_DEFAULT, + ) + return data # pyright: ignore[reportUnknownVariableType] + + @model_validator(mode="after") + def _validate_specialist_token_budgets(self) -> "DreamSettings": + """Ensure thinking_budget_tokens < max_output_tokens for each specialist.""" + for name, cfg in ( + ("DEDUCTION_MODEL_CONFIG", self.DEDUCTION_MODEL_CONFIG), + ("INDUCTION_MODEL_CONFIG", self.INDUCTION_MODEL_CONFIG), + ): + if ( + cfg.max_output_tokens is not None + and cfg.thinking_budget_tokens is not None + and cfg.max_output_tokens <= cfg.thinking_budget_tokens + ): + raise ValueError( + f"dream.{name}.max_output_tokens must be greater than " + + f"dream.{name}.thinking_budget_tokens" + ) return self @@ -633,10 +1219,6 @@ class AppSettings(HonchoSettings): MAX_MESSAGE_SIZE: Annotated[int, Field(default=25_000, gt=0)] = 25_000 EMBED_MESSAGES: bool = True - MAX_EMBEDDING_TOKENS: Annotated[int, Field(default=8192, gt=0)] = 8192 - MAX_EMBEDDING_TOKENS_PER_REQUEST: Annotated[int, Field(default=300_000, gt=0)] = ( - 300_000 - ) LANGFUSE_HOST: str | None = None LANGFUSE_PUBLIC_KEY: str | None = None @@ -651,6 +1233,7 @@ class AppSettings(HonchoSettings): AUTH: AuthSettings = Field(default_factory=AuthSettings) SENTRY: SentrySettings = Field(default_factory=SentrySettings) LLM: LLMSettings = Field(default_factory=LLMSettings) + EMBEDDING: EmbeddingSettings = Field(default_factory=EmbeddingSettings) DERIVER: DeriverSettings = Field(default_factory=DeriverSettings) DIALECTIC: DialecticSettings = Field(default_factory=DialecticSettings) PEER_CARD: PeerCardSettings = Field(default_factory=PeerCardSettings) @@ -676,11 +1259,25 @@ class AppSettings(HonchoSettings): self.CACHE.NAMESPACE = self.NAMESPACE if "NAMESPACE" not in self.VECTOR_STORE.model_fields_set: self.VECTOR_STORE.NAMESPACE = self.NAMESPACE + if "DIMENSIONS" not in self.VECTOR_STORE.model_fields_set: + self.VECTOR_STORE.DIMENSIONS = self.EMBEDDING.VECTOR_DIMENSIONS + elif self.VECTOR_STORE.DIMENSIONS != self.EMBEDDING.VECTOR_DIMENSIONS: + raise ValueError( + "VECTOR_STORE.DIMENSIONS must match EMBEDDING.VECTOR_DIMENSIONS" + ) if "NAMESPACE" not in self.TELEMETRY.model_fields_set: self.TELEMETRY.NAMESPACE = self.NAMESPACE if "NAMESPACE" not in self.METRICS.model_fields_set: self.METRICS.NAMESPACE = self.NAMESPACE + if self.EMBEDDING.VECTOR_DIMENSIONS != 1536 and ( + self.VECTOR_STORE.TYPE == "pgvector" or not self.VECTOR_STORE.MIGRATED + ): + raise ValueError( + "EMBEDDING.VECTOR_DIMENSIONS must remain 1536 while pgvector is " + + "active or vector-store migration is incomplete" + ) + return self diff --git a/src/crud/document.py b/src/crud/document.py index 688ed2f0..7de9dfd6 100644 --- a/src/crud/document.py +++ b/src/crud/document.py @@ -348,7 +348,8 @@ async def query_documents( embedding = await embedding_client.embed(query) except ValueError as e: raise ValidationException( - f"Query exceeds maximum token limit of {settings.MAX_EMBEDDING_TOKENS}." + "Query exceeds maximum token limit of " + + f"{settings.EMBEDDING.MAX_INPUT_TOKENS}." ) from e if _uses_pgvector(): diff --git a/src/crud/representation.py b/src/crud/representation.py index 616a17d1..de97d0d3 100644 --- a/src/crud/representation.py +++ b/src/crud/representation.py @@ -80,7 +80,8 @@ class RepresentationManager: embeddings = await embedding_client.simple_batch_embed(observation_texts) except ValueError as e: raise exceptions.ValidationException( - f"Observation content exceeds maximum token limit of {settings.MAX_EMBEDDING_TOKENS}." + "Observation content exceeds maximum token limit of " + + f"{settings.EMBEDDING.MAX_INPUT_TOKENS}." ) from e batch_embed_duration = (time.perf_counter() - batch_embed_start) * 1000 diff --git a/src/deriver/__main__.py b/src/deriver/__main__.py index e7934678..c3d498b2 100644 --- a/src/deriver/__main__.py +++ b/src/deriver/__main__.py @@ -50,7 +50,6 @@ def setup_logging(): logging.getLogger("httpcore").setLevel(logging.WARNING) logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("openai._base_client").setLevel(logging.WARNING) - logging.getLogger("groq._base_client").setLevel(logging.WARNING) async def run_deriver(): diff --git a/src/deriver/deriver.py b/src/deriver/deriver.py index b8735e3c..1fcc5ad2 100644 --- a/src/deriver/deriver.py +++ b/src/deriver/deriver.py @@ -2,9 +2,10 @@ import logging import time from src import crud -from src.config import settings +from src.config import ConfiguredModelSettings, settings from src.crud.representation import RepresentationManager from src.dependencies import tracked_db +from src.llm import honcho_llm_call from src.models import Message from src.schemas import ResolvedConfiguration from src.telemetry import prometheus_metrics @@ -16,7 +17,6 @@ from src.telemetry.prometheus.metrics import ( TokenTypes, ) from src.telemetry.sentry import with_sentry_transaction -from src.utils.clients import honcho_llm_call from src.utils.config_helpers import get_configuration from src.utils.formatting import format_new_turn_with_timestamp from src.utils.representation import PromptRepresentation, Representation @@ -27,6 +27,10 @@ from .prompts import estimate_minimal_deriver_prompt_tokens, minimal_deriver_pro logger = logging.getLogger(__name__) +def _get_deriver_model_config() -> ConfiguredModelSettings: + return settings.DERIVER.MODEL_CONFIG + + @with_sentry_transaction("minimal_deriver_batch", op="deriver") async def process_representation_tasks_batch( messages: list[Message], @@ -119,22 +123,24 @@ async def process_representation_tasks_batch( ) # validation on settings means max_tokens will always be > 0 - max_tokens = settings.DERIVER.MAX_OUTPUT_TOKENS or settings.LLM.DEFAULT_MAX_TOKENS + base_model_config = _get_deriver_model_config() + max_tokens = base_model_config.max_output_tokens or settings.LLM.DEFAULT_MAX_TOKENS + model_config = base_model_config.model_copy( + update={ + "stop_sequences": [" \n", "\n\n\n\n"], + } + ) # Single LLM call llm_start = time.perf_counter() response = await honcho_llm_call( - llm_settings=settings.DERIVER, + model_config=model_config, prompt=prompt, max_tokens=max_tokens, track_name="Minimal Deriver", response_model=PromptRepresentation, json_mode=True, - temperature=settings.DERIVER.TEMPERATURE, - stop_seqs=[" \n", "\n\n\n\n"], - thinking_budget_tokens=settings.DERIVER.THINKING_BUDGET_TOKENS, max_input_tokens=settings.DERIVER.MAX_INPUT_TOKENS, - reasoning_effort="minimal", enable_retry=True, retry_attempts=3, trace_name="minimal_deriver", diff --git a/src/dialectic/core.py b/src/dialectic/core.py index f2dee426..f8f3b841 100644 --- a/src/dialectic/core.py +++ b/src/dialectic/core.py @@ -12,10 +12,15 @@ from collections.abc import AsyncIterator, Callable from typing import Any, cast from src import crud -from src.config import ReasoningLevel, settings +from src.config import ConfiguredModelSettings, ReasoningLevel, settings from src.dependencies import tracked_db from src.dialectic import prompts from src.embedding_client import embedding_client +from src.llm import ( + HonchoLLMCallResponse, + StreamingResponseWithMetadata, + honcho_llm_call, +) from src.telemetry import prometheus_metrics from src.telemetry.events import DialecticCompletedEvent, emit from src.telemetry.logging import ( @@ -30,16 +35,17 @@ from src.utils.agent_tools import ( create_tool_executor, search_memory, ) -from src.utils.clients import ( - HonchoLLMCallResponse, - StreamingResponseWithMetadata, - honcho_llm_call, -) from src.utils.formatting import format_new_turn_with_timestamp logger = logging.getLogger(__name__) +def _get_dialectic_level_model_config( + reasoning_level: ReasoningLevel, +) -> ConfiguredModelSettings: + return settings.DIALECTIC.LEVELS[reasoning_level].MODEL_CONFIG + + class DialecticAgent: """ An agentic dialectic that iteratively gathers context to answer queries. @@ -405,7 +411,7 @@ class DialecticAgent: ) response: HonchoLLMCallResponse[str] = await honcho_llm_call( - llm_settings=level_settings, + model_config=_get_dialectic_level_model_config(self.reasoning_level), prompt="", # Ignored since we pass messages max_tokens=max_tokens, tools=tools, @@ -414,7 +420,6 @@ class DialecticAgent: max_tool_iterations=level_settings.MAX_TOOL_ITERATIONS, messages=self.messages, track_name="Dialectic Agent", - thinking_budget_tokens=level_settings.THINKING_BUDGET_TOKENS, max_input_tokens=settings.DIALECTIC.MAX_INPUT_TOKENS, trace_name="dialectic_chat", ) @@ -471,7 +476,7 @@ class DialecticAgent: response = cast( StreamingResponseWithMetadata, await honcho_llm_call( - llm_settings=level_settings, + model_config=_get_dialectic_level_model_config(self.reasoning_level), prompt="", # Ignored since we pass messages max_tokens=max_tokens, stream=True, @@ -482,7 +487,6 @@ class DialecticAgent: max_tool_iterations=level_settings.MAX_TOOL_ITERATIONS, messages=self.messages, track_name="Dialectic Agent Stream", - thinking_budget_tokens=level_settings.THINKING_BUDGET_TOKENS, max_input_tokens=settings.DIALECTIC.MAX_INPUT_TOKENS, trace_name="dialectic_chat", ), diff --git a/src/dreamer/specialists.py b/src/dreamer/specialists.py index 608db3b0..c7277586 100644 --- a/src/dreamer/specialists.py +++ b/src/dreamer/specialists.py @@ -19,8 +19,10 @@ from dataclasses import dataclass from typing import Any from src import crud, schemas -from src.config import settings +from src.config import ConfiguredModelSettings, settings from src.dependencies import tracked_db +from src.exceptions import ValidationException +from src.llm import HonchoLLMCallResponse, honcho_llm_call from src.schemas import ResolvedConfiguration from src.telemetry import prometheus_metrics from src.telemetry.events import DreamSpecialistEvent, emit @@ -31,11 +33,22 @@ from src.utils.agent_tools import ( INDUCTION_SPECIALIST_TOOLS, create_tool_executor, ) -from src.utils.clients import HonchoLLMCallResponse, honcho_llm_call logger = logging.getLogger(__name__) +def _require_specialist_model_config( + model_config: ConfiguredModelSettings | None, + *, + specialist_name: str, +) -> ConfiguredModelSettings: + if model_config is None: + raise ValidationException( + f"{specialist_name} MODEL_CONFIG must be resolved before use" + ) + return model_config + + @dataclass class SpecialistResult: """Result of a specialist run for telemetry and aggregation.""" @@ -70,8 +83,8 @@ class BaseSpecialist(ABC): ... @abstractmethod - def get_model(self) -> str: - """Get the model to use for this specialist.""" + def get_model_config(self) -> ConfiguredModelSettings: + """Get the configured model to use for this specialist.""" ... def get_max_tokens(self) -> int: @@ -196,9 +209,18 @@ If you update it, send the full deduplicated list and remove stale entries. parent_category="dream", ) - # Get model with potential override - model = self.get_model() - llm_settings = settings.DREAM.model_copy(update={"MODEL": model}) + model_config = self.get_model_config() + + # Respect operator-configured max_output_tokens on the specialist's + # ModelConfig (e.g. DREAM_DEDUCTION_MODEL_CONFIG__MAX_OUTPUT_TOKENS). + # Only fall back to the specialist's hardcoded default when the + # config leaves max_output_tokens unset or non-positive. + configured_max = model_config.max_output_tokens + effective_max_tokens = ( + configured_max + if configured_max and configured_max > 0 + else self.get_max_tokens() + ) # Track iterations via callback iteration_count = 0 @@ -209,9 +231,9 @@ If you update it, send the full deduplicated list and remove stale entries. # Run the agent loop response: HonchoLLMCallResponse[str] = await honcho_llm_call( - llm_settings=llm_settings, + model_config=model_config, prompt="", # Ignored since we pass messages - max_tokens=self.get_max_tokens(), + max_tokens=effective_max_tokens, tools=self.get_tools(peer_card_enabled=peer_card_enabled), tool_choice=None, tool_executor=tool_executor, @@ -305,8 +327,11 @@ class DeductionSpecialist(BaseSpecialist): if t["name"] not in PEER_CARD_TOOL_NAMES ] - def get_model(self) -> str: - return settings.DREAM.DEDUCTION_MODEL + def get_model_config(self) -> ConfiguredModelSettings: + return _require_specialist_model_config( + settings.DREAM.DEDUCTION_MODEL_CONFIG, + specialist_name="DREAM DEDUCTION", + ) def get_max_tokens(self) -> int: return 8192 @@ -377,11 +402,12 @@ When statements can't both be true (not just updates), flag them: ## CREATING OBSERVATIONS +Use `create_observations_deductive`. + ```json {{ "observations": [{{ "content": "The logical conclusion", - "level": "deductive", // or "contradiction" "source_ids": ["id1", "id2"], "premises": ["premise 1 text", "premise 2 text"] }}] @@ -393,8 +419,9 @@ When statements can't both be true (not just updates), flag them: 1. Don't explain your reasoning - just call tools 2. Create observations based on what you ACTUALLY FIND, not what you expect 3. Always include source_ids linking to the observations you're synthesizing -4. Delete outdated observations - don't leave duplicates -5. Quality over quantity - fewer good deductions beat many weak ones""" +4. Empty or missing source_ids will be rejected +5. Delete outdated observations - don't leave duplicates +6. Quality over quantity - fewer good deductions beat many weak ones""" def build_user_prompt( self, @@ -448,8 +475,11 @@ class InductionSpecialist(BaseSpecialist): if t["name"] not in PEER_CARD_TOOL_NAMES ] - def get_model(self) -> str: - return settings.DREAM.INDUCTION_MODEL + def get_model_config(self) -> ConfiguredModelSettings: + return _require_specialist_model_config( + settings.DREAM.INDUCTION_MODEL_CONFIG, + specialist_name="DREAM INDUCTION", + ) def get_max_tokens(self) -> int: return 8192 @@ -514,11 +544,12 @@ Create inductive observations when you see patterns: ## CREATING OBSERVATIONS +Use `create_observations_inductive`. + ```json {{ "observations": [{{ "content": "The pattern or generalization", - "level": "inductive", "source_ids": ["id1", "id2", "id3"], "sources": ["evidence 1", "evidence 2"], "pattern_type": "tendency", // preference|behavior|personality|tendency|correlation @@ -533,7 +564,8 @@ Create inductive observations when you see patterns: 2. Don't just restate a single fact as a pattern 3. Confidence based on evidence count: 2=low, 3-4=medium, 5+=high 4. Look for HOW things change over time, not just static facts -5. Include source_ids - always link back to evidence""" +5. Include source_ids - always link back to evidence +6. Empty or missing source_ids will be rejected""" def build_user_prompt( self, diff --git a/src/embedding_client.py b/src/embedding_client.py index 9798dc02..e163cc66 100644 --- a/src/embedding_client.py +++ b/src/embedding_client.py @@ -6,9 +6,10 @@ from typing import NamedTuple import tiktoken from google import genai +from google.genai import types as genai_types from openai import AsyncOpenAI -from .config import settings +from .config import EmbeddingModelConfig, resolve_embedding_model_config, settings logger = logging.getLogger(__name__) @@ -26,49 +27,58 @@ class _EmbeddingClient: Embedding client supporting OpenAI and Gemini with chunking and batching support. """ - def __init__(self, api_key: str | None = None, provider: str | None = None): - self.provider: str = provider or settings.LLM.EMBEDDING_PROVIDER + def __init__( + self, + config: EmbeddingModelConfig, + *, + vector_dimensions: int, + max_input_tokens: int, + max_tokens_per_request: int, + ): + self.transport: str = config.transport + self.model: str = config.model + self.vector_dimensions: int = vector_dimensions - if self.provider == "gemini": - if api_key is None: - api_key = settings.LLM.GEMINI_API_KEY - if not api_key: + if self.transport == "gemini": + if not config.api_key: raise ValueError("Gemini API key is required") - self.client: genai.Client | AsyncOpenAI = genai.Client(api_key=api_key) - self.model: str = "gemini-embedding-001" + http_options = ( + genai_types.HttpOptions(base_url=config.base_url) + if config.base_url + else None + ) + self.client: genai.Client | AsyncOpenAI = genai.Client( + api_key=config.api_key, + http_options=http_options, + ) # Gemini has a 2048 token limit - self.max_embedding_tokens: int = min(settings.MAX_EMBEDDING_TOKENS, 2048) + self.max_embedding_tokens: int = min(max_input_tokens, 2048) # Gemini batch size is not documented, using conservative estimate self.max_batch_size: int = 100 - elif self.provider == "openrouter": - if api_key is None: - api_key = settings.LLM.OPENAI_COMPATIBLE_API_KEY - if not api_key: - raise ValueError( - "OpenRouter API key (LLM_OPENAI_COMPATIBLE_API_KEY) is required" - ) - base_url = ( - settings.LLM.OPENAI_COMPATIBLE_BASE_URL - or "https://openrouter.ai/api/v1" - ) - self.client = AsyncOpenAI(api_key=api_key, base_url=base_url) - self.model = "openai/text-embedding-3-small" - self.max_embedding_tokens = settings.MAX_EMBEDDING_TOKENS - self.max_batch_size = 2048 # Same as OpenAI else: # openai - if api_key is None: - api_key = settings.LLM.OPENAI_API_KEY - if not api_key: + if not config.api_key: raise ValueError("OpenAI API key is required") - self.client = AsyncOpenAI(api_key=api_key) - self.model = "text-embedding-3-small" - self.max_embedding_tokens = settings.MAX_EMBEDDING_TOKENS + self.client = AsyncOpenAI( + api_key=config.api_key, + base_url=config.base_url, + ) + self.max_embedding_tokens = max_input_tokens self.max_batch_size = 2048 # OpenAI batch limit self.encoding: tiktoken.Encoding = tiktoken.get_encoding("o200k_base") - self.max_embedding_tokens_per_request: int = ( - settings.MAX_EMBEDDING_TOKENS_PER_REQUEST - ) + self.max_embedding_tokens_per_request: int = max_tokens_per_request + + @property + def provider(self) -> str: + return self.transport + + def _validate_embedding_dimensions(self, embedding: list[float]) -> list[float]: + if len(embedding) != self.vector_dimensions: + raise ValueError( + f"Embedding dimension mismatch for {self.transport}:{self.model}. " + + f"Expected {self.vector_dimensions}, got {len(embedding)}." + ) + return embedding async def embed(self, query: str) -> list[float]: token_count = len(self.encoding.encode(query)) @@ -82,16 +92,16 @@ class _EmbeddingClient: response = await self.client.aio.models.embed_content( model=self.model, contents=query, - config={"output_dimensionality": 1536}, + config={"output_dimensionality": self.vector_dimensions}, ) if not response.embeddings or not response.embeddings[0].values: raise ValueError("No embedding returned from Gemini API") - return response.embeddings[0].values + return self._validate_embedding_dimensions(response.embeddings[0].values) else: # openai response = await self.client.embeddings.create( model=self.model, input=query ) - return response.data[0].embedding + return self._validate_embedding_dimensions(response.data[0].embedding) async def simple_batch_embed(self, texts: list[str]) -> list[list[float]]: """ @@ -116,18 +126,25 @@ class _EmbeddingClient: response = await self.client.aio.models.embed_content( model=self.model, contents=batch, # pyright: ignore[reportArgumentType] - config={"output_dimensionality": 1536}, + config={"output_dimensionality": self.vector_dimensions}, ) if response.embeddings: for emb in response.embeddings: if emb.values: - embeddings.append(emb.values) + embeddings.append( + self._validate_embedding_dimensions(emb.values) + ) else: # openai response = await self.client.embeddings.create( input=batch, model=self.model, ) - embeddings.extend([data.embedding for data in response.data]) + embeddings.extend( + [ + self._validate_embedding_dimensions(data.embedding) + for data in response.data + ] + ) except Exception as e: # Check if it's a token limit error and re-raise as ValueError for consistency if "token" in str(e).lower(): @@ -252,7 +269,7 @@ class _EmbeddingClient: response = await self.client.aio.models.embed_content( model=self.model, contents=[item.text for item in batch], - config={"output_dimensionality": 1536}, + config={"output_dimensionality": self.vector_dimensions}, ) if response.embeddings: for item, embedding in zip( @@ -260,15 +277,19 @@ class _EmbeddingClient: ): if embedding.values: result[item.text_id][item.chunk_index] = ( - embedding.values + self._validate_embedding_dimensions( + embedding.values + ) ) - else: # openai / openrouter + else: # openai response = await self.client.embeddings.create( model=self.model, input=[item.text for item in batch] ) for item, embedding_data in zip(batch, response.data, strict=True): result[item.text_id][item.chunk_index] = ( - embedding_data.embedding + self._validate_embedding_dimensions( + embedding_data.embedding + ) ) return dict(result) @@ -358,6 +379,7 @@ class EmbeddingClient: """ _instance: "_EmbeddingClient | None" = None + _instance_signature: tuple[object, ...] | None = None _lock: threading.Lock = threading.Lock() _wrapper_instance: "EmbeddingClient | None" = None @@ -374,26 +396,41 @@ class EmbeddingClient: Uses double-checked locking for thread-safe lazy initialization. """ - if self._instance is None: + signature = self._get_settings_signature() + if self._instance is None or self._instance_signature != signature: with self._lock: - if self._instance is None: - provider = settings.LLM.EMBEDDING_PROVIDER - if provider == "gemini": - api_key = settings.LLM.GEMINI_API_KEY - elif provider == "openrouter": - api_key = settings.LLM.OPENAI_COMPATIBLE_API_KEY - else: - api_key = settings.LLM.OPENAI_API_KEY - + if self._instance is None or self._instance_signature != signature: + runtime_config = self._resolve_runtime_config() self._instance = _EmbeddingClient( - api_key=api_key, provider=provider + runtime_config, + vector_dimensions=settings.EMBEDDING.VECTOR_DIMENSIONS, + max_input_tokens=settings.EMBEDDING.MAX_INPUT_TOKENS, + max_tokens_per_request=settings.EMBEDDING.MAX_TOKENS_PER_REQUEST, ) + self._instance_signature = signature logger.debug( - f"Initialized embedding client with provider: {provider}" + "Initialized embedding client with transport: %s model: %s", + runtime_config.transport, + runtime_config.model, ) return self._instance + def _resolve_runtime_config(self) -> EmbeddingModelConfig: + return resolve_embedding_model_config(settings.EMBEDDING.MODEL_CONFIG) + + def _get_settings_signature(self) -> tuple[object, ...]: + runtime_config = self._resolve_runtime_config() + return ( + runtime_config.transport, + runtime_config.model, + runtime_config.api_key, + runtime_config.base_url, + settings.EMBEDDING.VECTOR_DIMENSIONS, + settings.EMBEDDING.MAX_INPUT_TOKENS, + settings.EMBEDDING.MAX_TOKENS_PER_REQUEST, + ) + async def embed(self, query: str) -> list[float]: """Embed a single query string.""" return await self._get_client().embed(query) @@ -418,11 +455,21 @@ class EmbeddingClient: """Get the model name.""" return self._get_client().model + @property + def transport(self) -> str: + """Get the transport name.""" + return self._get_client().transport + @property def max_embedding_tokens(self) -> int: """Get the maximum embedding tokens.""" return self._get_client().max_embedding_tokens + @property + def vector_dimensions(self) -> int: + """Get the configured embedding dimensions.""" + return self._get_client().vector_dimensions + @property def encoding(self) -> tiktoken.Encoding: """Get the tiktoken encoding.""" diff --git a/src/llm/__init__.py b/src/llm/__init__.py new file mode 100644 index 00000000..ae47bc53 --- /dev/null +++ b/src/llm/__init__.py @@ -0,0 +1,66 @@ +"""Honcho LLM orchestration package — stable public surface. + +Application code should import from `src.llm` (or specific submodules like +`src.llm.api` / `src.llm.types`). The old `src/utils/clients.py` entrypoint +is gone; everything lives here now. +""" + +from __future__ import annotations + +from .api import honcho_llm_call +from .backend import CompletionResult, ProviderBackend, StreamChunk, ToolCallResult +from .credentials import default_transport_api_key, resolve_credentials +from .executor import honcho_llm_call_inner +from .registry import ( + CLIENTS, + backend_for_provider, + client_for_model_config, + get_anthropic_client, + get_anthropic_override_client, + get_backend, + get_gemini_client, + get_gemini_override_client, + get_openai_client, + get_openai_override_client, + history_adapter_for_provider, +) +from .types import ( + HonchoLLMCallResponse, + HonchoLLMCallStreamChunk, + IterationCallback, + IterationData, + ProviderClient, + ReasoningEffortType, + StreamingResponseWithMetadata, + VerbosityType, +) + +__all__ = [ + "CLIENTS", + "CompletionResult", + "HonchoLLMCallResponse", + "HonchoLLMCallStreamChunk", + "IterationCallback", + "IterationData", + "ProviderBackend", + "ProviderClient", + "ReasoningEffortType", + "StreamChunk", + "StreamingResponseWithMetadata", + "ToolCallResult", + "VerbosityType", + "backend_for_provider", + "client_for_model_config", + "default_transport_api_key", + "get_anthropic_client", + "get_anthropic_override_client", + "get_backend", + "get_gemini_client", + "get_gemini_override_client", + "get_openai_client", + "get_openai_override_client", + "history_adapter_for_provider", + "honcho_llm_call", + "honcho_llm_call_inner", + "resolve_credentials", +] diff --git a/src/llm/api.py b/src/llm/api.py new file mode 100644 index 00000000..4639cad4 --- /dev/null +++ b/src/llm/api.py @@ -0,0 +1,359 @@ +"""Public LLM entrypoint: `honcho_llm_call`. + +Orchestrates: +- Runtime config resolution from ConfiguredModelSettings → ModelConfig. +- Per-attempt planning (primary vs fallback selection). +- Retry with exponential backoff via tenacity. +- Tool-loop delegation when tools are supplied. +- Single-call delegation to the executor otherwise. +- Reasoning-trace telemetry emission. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator, Callable +from typing import Any, Literal, TypeVar, cast, overload + +from pydantic import BaseModel +from sentry_sdk.ai.monitoring import ai_track +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.config import ConfiguredModelSettings, ModelConfig +from src.exceptions import ValidationException +from src.telemetry.logging import conditional_observe +from src.telemetry.reasoning_traces import log_reasoning_trace + +from .executor import honcho_llm_call_inner +from .runtime import ( + AttemptPlan, + current_attempt, + effective_temperature, + plan_attempt, + resolve_runtime_model_config, +) +from .tool_loop import execute_tool_loop +from .types import ( + HonchoLLMCallResponse, + HonchoLLMCallStreamChunk, + IterationCallback, + ReasoningEffortType, + StreamingResponseWithMetadata, +) + +logger = logging.getLogger(__name__) + +M = TypeVar("M", bound=BaseModel) + + +@overload +async def honcho_llm_call( + *, + model_config: ModelConfig | ConfiguredModelSettings, + prompt: str, + max_tokens: int, + track_name: str | None = None, + response_model: type[M], + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + enable_retry: bool = True, + retry_attempts: int = 3, + stream: Literal[False] = False, + stream_final_only: bool = False, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, + max_tool_iterations: int = 10, + messages: list[dict[str, Any]] | None = None, + max_input_tokens: int | None = None, + trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, +) -> HonchoLLMCallResponse[M]: ... + + +@overload +async def honcho_llm_call( + *, + model_config: ModelConfig | ConfiguredModelSettings, + prompt: str, + max_tokens: int, + track_name: str | None = None, + response_model: None = None, + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + enable_retry: bool = True, + retry_attempts: int = 3, + stream: Literal[False] = False, + stream_final_only: bool = False, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, + max_tool_iterations: int = 10, + messages: list[dict[str, Any]] | None = None, + max_input_tokens: int | None = None, + trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, +) -> HonchoLLMCallResponse[str]: ... + + +@overload +async def honcho_llm_call( + *, + model_config: ModelConfig | ConfiguredModelSettings, + prompt: str, + max_tokens: int, + track_name: str | None = None, + response_model: type[BaseModel] | None = None, + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + enable_retry: bool = True, + retry_attempts: int = 3, + stream: Literal[True] = ..., + stream_final_only: bool = False, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, + max_tool_iterations: int = 10, + messages: list[dict[str, Any]] | None = None, + max_input_tokens: int | None = None, + trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, +) -> AsyncIterator[HonchoLLMCallStreamChunk] | StreamingResponseWithMetadata: ... + + +@conditional_observe(name="LLM Call") +async def honcho_llm_call( + *, + model_config: ModelConfig | ConfiguredModelSettings, + prompt: str, + max_tokens: int, + track_name: str | None = None, + response_model: type[BaseModel] | None = None, + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + enable_retry: bool = True, + retry_attempts: int = 3, + stream: bool = False, + stream_final_only: bool = False, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, + max_tool_iterations: int = 10, + messages: list[dict[str, Any]] | None = None, + max_input_tokens: int | None = None, + trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, +) -> ( + HonchoLLMCallResponse[Any] + | AsyncIterator[HonchoLLMCallStreamChunk] + | StreamingResponseWithMetadata +): + """Make an LLM call with retry, optional backup failover, and optional tool loop. + + Backup provider/model (if configured on the primary ModelConfig's + `fallback`) is used on the final retry attempt, which is 3 by default. + + Raises: + ValidationException: If streaming and tool calling are combined + without `stream_final_only=True`. + """ + runtime_model_config = resolve_runtime_model_config(model_config) + + # Caller kwargs left at None are resolved downstream by + # effective_config_for_call against whichever ModelConfig wins the + # attempt (primary or fallback). Defaulting here from + # runtime_model_config would clobber a fallback config's own + # temperature/thinking params on the final retry, so we deliberately + # keep the locals as the caller supplied them. + + if stream and tools and not stream_final_only: + raise ValidationException( + "Streaming is not supported with tool calling. " + + "Set stream=False when using tools, or use stream_final_only=True " + + "to stream only the final response after tool calls." + ) + + # tenacity uses 1-indexed attempts. + current_attempt.set(1) + + def _get_attempt_plan() -> AttemptPlan: + return plan_attempt( + runtime_model_config=runtime_model_config, + attempt=current_attempt.get(), + retry_attempts=retry_attempts, + call_thinking_budget_tokens=thinking_budget_tokens, + call_reasoning_effort=reasoning_effort, + ) + + async def _call_with_provider_selection() -> ( + HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk] + ): + """Select provider/model based on current attempt, then call once. + + This closure is what tenacity wraps, so selection re-runs per attempt + (and the fallback kicks in on the final attempt automatically). + """ + plan = _get_attempt_plan() + + if stream: + return await honcho_llm_call_inner( + plan.provider, + plan.model, + prompt, + max_tokens, + response_model, + json_mode, + effective_temperature(temperature), + stop_seqs, + plan.reasoning_effort, + verbosity, + plan.thinking_budget_tokens, + stream=True, + client_override=plan.client, + tools=tools, + tool_choice=tool_choice, + selected_config=plan.selected_config, + ) + return await honcho_llm_call_inner( + plan.provider, + plan.model, + prompt, + max_tokens, + response_model, + json_mode, + effective_temperature(temperature), + stop_seqs, + plan.reasoning_effort, + verbosity, + plan.thinking_budget_tokens, + stream=False, + client_override=plan.client, + tools=tools, + tool_choice=tool_choice, + selected_config=plan.selected_config, + ) + + decorated = _call_with_provider_selection + + if track_name: + decorated = ai_track(track_name)(decorated) + + def before_retry_callback(retry_state: Any) -> None: + """Update attempt counter before each retry + log transient failures. + + tenacity's before_sleep fires AFTER an attempt fails, BEFORE sleeping, + so we increment to the next attempt number here. + """ + next_attempt = retry_state.attempt_number + 1 + current_attempt.set(next_attempt) + exc = retry_state.outcome.exception() if retry_state.outcome else None + if exc: + logger.warning( + f"Error on attempt {retry_state.attempt_number}/{retry_attempts} with " + + f"{runtime_model_config.transport}/{runtime_model_config.model}: {exc}" + ) + logger.info(f"Will retry with attempt {next_attempt}/{retry_attempts}") + + if enable_retry: + decorated = retry( + stop=stop_after_attempt(retry_attempts), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=before_retry_callback, + )(decorated) + + def _trace_thinking_budget() -> int | None: + # Trace log should reflect what got applied, so fall back to the + # runtime config's value when the caller left the kwarg unset. + return ( + thinking_budget_tokens + if thinking_budget_tokens is not None + else runtime_model_config.thinking_budget_tokens + ) + + def _trace_reasoning_effort() -> ReasoningEffortType: + if reasoning_effort is not None: + return reasoning_effort + config_effort = runtime_model_config.thinking_effort + return cast(ReasoningEffortType, config_effort) if config_effort else None + + def _trace_stop_seqs() -> list[str] | None: + return ( + stop_seqs if stop_seqs is not None else runtime_model_config.stop_sequences + ) + + # Tool-less path: call once and return. + if not tools or not tool_executor: + result: ( + HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk] + ) = await decorated() + if trace_name and isinstance(result, HonchoLLMCallResponse): + log_reasoning_trace( + task_type=trace_name, + model_config=runtime_model_config, + prompt=prompt, + response=result, + max_tokens=max_tokens, + thinking_budget_tokens=_trace_thinking_budget(), + reasoning_effort=_trace_reasoning_effort(), + json_mode=json_mode, + stop_seqs=_trace_stop_seqs(), + messages=messages, + ) + return result + + # execute_tool_loop raises ValidationException on out-of-range + # max_tool_iterations; fail-fast is cheaper than silent clamping here. + result = await execute_tool_loop( + prompt=prompt, + max_tokens=max_tokens, + messages=messages, + tools=tools, + tool_choice=tool_choice, + tool_executor=tool_executor, + max_tool_iterations=max_tool_iterations, + response_model=response_model, + json_mode=json_mode, + temperature=temperature, + stop_seqs=stop_seqs, + verbosity=verbosity, + enable_retry=enable_retry, + retry_attempts=retry_attempts, + max_input_tokens=max_input_tokens, + get_attempt_plan=_get_attempt_plan, + before_retry_callback=before_retry_callback, + stream_final=stream_final_only, + iteration_callback=iteration_callback, + ) + if trace_name and isinstance(result, HonchoLLMCallResponse): + log_reasoning_trace( + task_type=trace_name, + model_config=runtime_model_config, + prompt=prompt, + response=result, + max_tokens=max_tokens, + thinking_budget_tokens=_trace_thinking_budget(), + reasoning_effort=_trace_reasoning_effort(), + json_mode=json_mode, + stop_seqs=_trace_stop_seqs(), + messages=messages, + ) + return result + + +__all__ = ["honcho_llm_call"] diff --git a/src/llm/backend.py b/src/llm/backend.py new file mode 100644 index 00000000..5645998c --- /dev/null +++ b/src/llm/backend.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +from pydantic import BaseModel + + +@dataclass(slots=True) +class ToolCallResult: + """Normalized tool call from any provider.""" + + id: str + name: str + input: dict[str, Any] + thought_signature: str | None = None + + +@dataclass(slots=True) +class CompletionResult: + """Normalized completion result returned by provider backends.""" + + content: Any = "" + input_tokens: int = 0 + output_tokens: int = 0 + cache_creation_input_tokens: int = 0 + cache_read_input_tokens: int = 0 + finish_reason: str = "stop" + tool_calls: list[ToolCallResult] = field(default_factory=list) + thinking_content: str | None = None + thinking_blocks: list[dict[str, Any]] = field(default_factory=list) + reasoning_details: list[dict[str, Any]] = field(default_factory=list) + raw_response: Any = None + + +@dataclass(slots=True) +class StreamChunk: + """A single chunk in a streaming response.""" + + content: str = "" + is_done: bool = False + finish_reason: str | None = None + output_tokens: int | None = None + + +@runtime_checkable +class ProviderBackend(Protocol): + """Transport-agnostic interface for LLM providers. + + Credentials are baked into the underlying SDK client at backend construction + time (see src/llm/registry.py), so these method signatures deliberately do + not accept api_key / api_base. + """ + + async def complete( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> CompletionResult: ... + + def stream( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> AsyncIterator[StreamChunk]: ... diff --git a/src/llm/backends/__init__.py b/src/llm/backends/__init__.py new file mode 100644 index 00000000..dfba81ec --- /dev/null +++ b/src/llm/backends/__init__.py @@ -0,0 +1,9 @@ +from .anthropic import AnthropicBackend +from .gemini import GeminiBackend +from .openai import OpenAIBackend + +__all__ = [ + "AnthropicBackend", + "GeminiBackend", + "OpenAIBackend", +] diff --git a/src/llm/backends/anthropic.py b/src/llm/backends/anthropic.py new file mode 100644 index 00000000..cdf775be --- /dev/null +++ b/src/llm/backends/anthropic.py @@ -0,0 +1,347 @@ +from __future__ import annotations + +import copy +import json +from collections.abc import AsyncIterator +from typing import Any + +from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock +from pydantic import BaseModel, ValidationError + +from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.structured_output import repair_response_model_json + + +class AnthropicBackend: + """Provider backend wrapping the native Anthropic SDK.""" + + def __init__(self, client: Any) -> None: + self._client: Any = client + + async def complete( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> CompletionResult: + del max_output_tokens + if thinking_effort is not None: + raise ValueError( + "Anthropic backend does not support thinking_effort; use thinking_budget_tokens instead" + ) + + request_messages, system_messages = self._extract_system(messages) + params: dict[str, Any] = { + "model": model, + "max_tokens": max_tokens, + "messages": request_messages, + } + + if temperature is not None: + params["temperature"] = temperature + if stop: + params["stop_sequences"] = stop + if system_messages: + params["system"] = [ + { + "type": "text", + "text": "\n\n".join(system_messages), + "cache_control": {"type": "ephemeral"}, + } + ] + if tools: + params["tools"] = tools + converted_tool_choice = self._convert_tool_choice(tool_choice) + if converted_tool_choice is not None: + params["tool_choice"] = converted_tool_choice + if thinking_budget_tokens: + params["thinking"] = { + "type": "enabled", + "budget_tokens": thinking_budget_tokens, + } + if extra_params: + for key in ("top_p", "top_k"): + if key in extra_params: + params[key] = extra_params[key] + + use_json_prefill = ( + bool(response_format or self._json_mode(extra_params)) + and not thinking_budget_tokens + and self._supports_assistant_prefill(model) + ) + if use_json_prefill and params["messages"]: + if response_format and isinstance(response_format, type): + schema_json = json.dumps(response_format.model_json_schema(), indent=2) + self._append_text_to_last_message( + params["messages"], + f"\n\nRespond with valid JSON matching this schema:\n{schema_json}", + ) + params["messages"].append({"role": "assistant", "content": "{"}) + elif ( + response_format and isinstance(response_format, type) and params["messages"] + ): + schema_json = json.dumps(response_format.model_json_schema(), indent=2) + self._append_text_to_last_message( + params["messages"], + f"\n\nRespond with valid JSON matching this schema:\n{schema_json}", + ) + + response = await self._client.messages.create(**params) + return self._normalize_response( + response=response, + response_format=response_format + if isinstance(response_format, type) + else None, + prefilled_json=use_json_prefill, + model_name=model, + ) + + async def stream( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> AsyncIterator[StreamChunk]: + is_json_mode = self._json_mode(extra_params) + del max_output_tokens + if thinking_effort is not None: + raise ValueError( + "Anthropic backend does not support thinking_effort; use thinking_budget_tokens instead" + ) + + request_messages, system_messages = self._extract_system(messages) + params: dict[str, Any] = { + "model": model, + "max_tokens": max_tokens, + "messages": request_messages, + } + if temperature is not None: + params["temperature"] = temperature + if stop: + params["stop_sequences"] = stop + if tools: + params["tools"] = tools + converted_tool_choice = self._convert_tool_choice(tool_choice) + if converted_tool_choice is not None: + params["tool_choice"] = converted_tool_choice + if system_messages: + params["system"] = [ + { + "type": "text", + "text": "\n\n".join(system_messages), + "cache_control": {"type": "ephemeral"}, + } + ] + if extra_params: + for key in ("top_p", "top_k"): + if key in extra_params: + params[key] = extra_params[key] + use_json_prefill = ( + bool(response_format or is_json_mode) + and not thinking_budget_tokens + and self._supports_assistant_prefill(model) + ) + if use_json_prefill and params["messages"]: + if response_format and isinstance(response_format, type): + schema_json = json.dumps(response_format.model_json_schema(), indent=2) + self._append_text_to_last_message( + params["messages"], + f"\n\nRespond with valid JSON matching this schema:\n{schema_json}", + ) + params["messages"].append({"role": "assistant", "content": "{"}) + elif ( + response_format and isinstance(response_format, type) and params["messages"] + ): + schema_json = json.dumps(response_format.model_json_schema(), indent=2) + self._append_text_to_last_message( + params["messages"], + f"\n\nRespond with valid JSON matching this schema:\n{schema_json}", + ) + if thinking_budget_tokens: + params["thinking"] = { + "type": "enabled", + "budget_tokens": thinking_budget_tokens, + } + + async with self._client.messages.stream(**params) as stream: + async for chunk in stream: + if ( + chunk.type == "content_block_delta" + and hasattr(chunk, "delta") + and hasattr(chunk.delta, "text") + ): + yield StreamChunk(content=getattr(chunk.delta, "text", "")) + + final_message = await stream.get_final_message() + output_tokens = ( + final_message.usage.output_tokens if final_message.usage else None + ) + yield StreamChunk( + is_done=True, + finish_reason=final_message.stop_reason, + output_tokens=output_tokens, + ) + + def _normalize_response( + self, + *, + response: Any, + response_format: type[BaseModel] | None, + prefilled_json: bool, + model_name: str, + ) -> CompletionResult: + text_blocks: list[str] = [] + thinking_text_blocks: list[str] = [] + thinking_full_blocks: list[dict[str, Any]] = [] + tool_calls: list[ToolCallResult] = [] + + for block in response.content: + if isinstance(block, TextBlock): + text_blocks.append(block.text) + elif isinstance(block, ThinkingBlock): + thinking_text_blocks.append(block.thinking) + thinking_full_blocks.append( + { + "type": "thinking", + "thinking": block.thinking, + "signature": block.signature, + } + ) + elif isinstance(block, ToolUseBlock): + tool_calls.append( + ToolCallResult( + id=block.id, + name=block.name, + input=dict(block.input), + ) + ) + + usage = response.usage + cache_creation_tokens = ( + getattr(usage, "cache_creation_input_tokens", 0) or 0 if usage else 0 + ) + cache_read_tokens = ( + getattr(usage, "cache_read_input_tokens", 0) or 0 if usage else 0 + ) + uncached_tokens = usage.input_tokens if usage else 0 + total_input_tokens = uncached_tokens + cache_creation_tokens + cache_read_tokens + + text_content = "\n".join(text_blocks) + thinking_content = ( + "\n".join(thinking_text_blocks) if thinking_text_blocks else None + ) + + content: Any = text_content + if response_format is not None: + raw_content = f"{{{text_content}" if prefilled_json else text_content + try: + if prefilled_json: + parsed_json = json.loads(raw_content) + content = response_format.model_validate(parsed_json) + else: + content = response_format.model_validate_json(raw_content) + except (json.JSONDecodeError, ValidationError, ValueError): + content = repair_response_model_json( + raw_content, + response_format, + model_name, + ) + + return CompletionResult( + content=content, + input_tokens=total_input_tokens, + output_tokens=usage.output_tokens if usage else 0, + cache_creation_input_tokens=cache_creation_tokens, + cache_read_input_tokens=cache_read_tokens, + finish_reason=response.stop_reason or "stop", + tool_calls=tool_calls, + thinking_content=thinking_content, + thinking_blocks=thinking_full_blocks, + raw_response=response, + ) + + @staticmethod + def _supports_assistant_prefill(model: str) -> bool: + # Claude 4-class models reject assistant-prefill and require the + # conversation to end with a user message. + return not model.startswith( + ( + "claude-opus-4", + "claude-sonnet-4", + "claude-haiku-4", + ) + ) + + @staticmethod + def _extract_system( + messages: list[dict[str, Any]], + ) -> tuple[list[dict[str, Any]], list[str]]: + system_messages: list[str] = [] + non_system_messages: list[dict[str, Any]] = [] + for message in messages: + if message.get("role") == "system" and isinstance( + message.get("content"), + str, + ): + system_messages.append(message["content"]) + else: + non_system_messages.append(copy.deepcopy(message)) + return non_system_messages, system_messages + + @staticmethod + def _convert_tool_choice( + tool_choice: str | dict[str, Any] | None, + ) -> dict[str, Any] | None: + if tool_choice is None: + return None + if isinstance(tool_choice, dict): + return tool_choice + if tool_choice == "auto": + return {"type": "auto"} + if tool_choice in {"any", "required"}: + return {"type": "any"} + if tool_choice == "none": + return {"type": "none"} + return {"type": "tool", "name": tool_choice} + + @staticmethod + def _append_text_to_last_message( + messages: list[dict[str, Any]], suffix: str + ) -> None: + """Append text to the last message, handling both string and list content.""" + last = messages[-1] + content = last.get("content") + if isinstance(content, str): + last["content"] = content + suffix + elif isinstance(content, list): + # Content block list — append to the last text block or add one + blocks: list[dict[str, Any]] = content # pyright: ignore[reportUnknownVariableType] + for block in reversed(blocks): + if block.get("type") == "text": + block["text"] = block["text"] + suffix + return + blocks.append({"type": "text", "text": suffix}) + + @staticmethod + def _json_mode(extra_params: dict[str, Any] | None) -> bool: + return bool(extra_params and extra_params.get("json_mode")) diff --git a/src/llm/backends/gemini.py b/src/llm/backends/gemini.py new file mode 100644 index 00000000..b14cefe4 --- /dev/null +++ b/src/llm/backends/gemini.py @@ -0,0 +1,577 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator +from datetime import datetime, timedelta, timezone +from typing import Any, ClassVar, cast + +from pydantic import BaseModel + +from src.exceptions import LLMError, ValidationException +from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.caching import ( + GeminiCacheHandle, + PromptCachePolicy, + build_cache_key, + gemini_cache_store, +) +from src.llm.structured_output import repair_response_model_json + +GEMINI_BLOCKED_FINISH_REASONS = { + "SAFETY", + "RECITATION", + "PROHIBITED_CONTENT", + "BLOCKLIST", +} + + +class GeminiBackend: + """Provider backend wrapping the Google GenAI SDK.""" + + def __init__(self, client: Any) -> None: + self._client: Any = client + + async def complete( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> CompletionResult: + contents, system_instruction = self._convert_messages(messages) + config = self._build_config( + max_tokens=max_output_tokens or max_tokens, + temperature=temperature, + stop=stop, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + thinking_budget_tokens=thinking_budget_tokens, + thinking_effort=thinking_effort, + extra_params=extra_params, + ) + if system_instruction: + config["system_instruction"] = system_instruction + + cache_policy = ( + extra_params.get("cache_policy") + if extra_params and "cache_policy" in extra_params + else None + ) + if isinstance(cache_policy, PromptCachePolicy) and isinstance(contents, list): + # Cache the history prefix; only the last turn is sent as new input. + cacheable = contents[:-1] if contents else [] + await self._attach_cached_content( + model=model, + config=config, + cache_policy=cache_policy, + contents=cacheable, + tools=tools, + ) + if "cached_content" in config and contents: + contents = contents[-1:] + + if isinstance(contents, list) and not contents: + raise LLMError( + "No non-system messages to send to Gemini", + provider="gemini", + model=model, + ) + + response = await self._client.aio.models.generate_content( + model=model, + contents=contents, + config=config or None, + ) + return self._normalize_response( + response=response, + response_format=response_format + if isinstance(response_format, type) + else None, + model_name=model, + ) + + async def stream( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> AsyncIterator[StreamChunk]: + contents, system_instruction = self._convert_messages(messages) + config = self._build_config( + max_tokens=max_output_tokens or max_tokens, + temperature=temperature, + stop=stop, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + thinking_budget_tokens=thinking_budget_tokens, + thinking_effort=thinking_effort, + extra_params=extra_params, + ) + if system_instruction: + config["system_instruction"] = system_instruction + + cache_policy = ( + extra_params.get("cache_policy") + if extra_params and "cache_policy" in extra_params + else None + ) + if isinstance(cache_policy, PromptCachePolicy) and isinstance(contents, list): + # Cache the history prefix; only the last turn is sent as new input. + cacheable = contents[:-1] if contents else [] + await self._attach_cached_content( + model=model, + config=config, + cache_policy=cache_policy, + contents=cacheable, + tools=tools, + ) + if "cached_content" in config and contents: + contents = contents[-1:] + + if isinstance(contents, list) and not contents: + raise LLMError( + "No non-system messages to send to Gemini", + provider="gemini", + model=model, + ) + + stream = await self._client.aio.models.generate_content_stream( + model=model, + contents=contents, + config=config or None, + ) + + final_chunk = None + any_text = False + async for chunk in stream: + if chunk.text: + any_text = True + yield StreamChunk(content=chunk.text) + final_chunk = chunk + + finish_reason = "stop" + output_tokens: int | None = None + if ( + final_chunk + and getattr(final_chunk, "candidates", None) + and final_chunk.candidates[0].finish_reason + ): + finish_reason = final_chunk.candidates[0].finish_reason.name + if ( + final_chunk + and getattr(final_chunk, "usage_metadata", None) + and getattr(final_chunk.usage_metadata, "candidates_token_count", None) + ): + output_tokens = final_chunk.usage_metadata.candidates_token_count or None + + # Mirror complete()'s behavior on SAFETY / RECITATION / etc. — if + # Gemini blocked the response and produced no usable text, raise + # LLMError rather than silently yielding a terminal chunk carrying + # the blocked finish_reason. Downstream callers should get a clean + # exception and a chance to retry / fall back. + if not any_text and finish_reason in GEMINI_BLOCKED_FINISH_REASONS: + raise LLMError( + f"Gemini response blocked (finish_reason={finish_reason})", + provider="gemini", + model=model, + finish_reason=finish_reason, + ) + + yield StreamChunk( + is_done=True, + finish_reason=finish_reason, + output_tokens=output_tokens, + ) + + def _build_config( + self, + *, + max_tokens: int, + temperature: float | None, + stop: list[str] | None, + tools: list[dict[str, Any]] | None, + tool_choice: str | dict[str, Any] | None, + response_format: type[BaseModel] | dict[str, Any] | None, + thinking_budget_tokens: int | None, + thinking_effort: str | None, + extra_params: dict[str, Any] | None, + ) -> dict[str, Any]: + config: dict[str, Any] = { + "max_output_tokens": max_tokens, + } + if temperature is not None: + config["temperature"] = temperature + if stop: + config["stop_sequences"] = stop + if tools: + config["tools"] = self._convert_tools(tools) + if tool_choice: + config["tool_config"] = self._convert_tool_choice(tool_choice) + if response_format is not None: + config["response_mime_type"] = "application/json" + config["response_schema"] = response_format + elif extra_params and extra_params.get("json_mode") and not tools: + config["response_mime_type"] = "application/json" + thinking_config: dict[str, Any] = {} + if thinking_budget_tokens is not None: + thinking_config["thinking_budget"] = thinking_budget_tokens + if thinking_effort is not None: + thinking_config["thinking_level"] = thinking_effort + if len(thinking_config) > 1: + raise ValidationException( + "Gemini backend does not support sending both thinking_budget_tokens and thinking_effort in the same request" + ) + if thinking_config: + config["thinking_config"] = thinking_config + for key in ("top_p", "top_k", "frequency_penalty", "presence_penalty", "seed"): + if extra_params and key in extra_params: + config[key] = extra_params[key] + return config + + def _normalize_response( + self, + *, + response: Any, + response_format: type[BaseModel] | None, + model_name: str, + ) -> CompletionResult: + candidate = response.candidates[0] if response.candidates else None + finish_reason = ( + candidate.finish_reason.name + if candidate is not None and candidate.finish_reason + else "stop" + ) + + text_parts: list[str] = [] + tool_calls: list[ToolCallResult] = [] + candidate_parts = ( + cast(list[Any] | None, getattr(candidate.content, "parts", None)) + if candidate is not None and getattr(candidate, "content", None) + else None + ) + if isinstance(candidate_parts, list): + for part in candidate_parts: + part_text = getattr(part, "text", None) + if isinstance(part_text, str) and part_text: + text_parts.append(part_text) + function_call = getattr(part, "function_call", None) + if function_call is not None: + function_name = getattr(function_call, "name", None) + function_args = getattr(function_call, "args", None) + if not isinstance(function_name, str): + continue + tool_calls.append( + ToolCallResult( + id=f"call_{function_name}_{len(tool_calls)}", + name=function_name, + input=dict(cast(dict[str, Any], function_args)) + if function_args + else {}, + thought_signature=getattr(part, "thought_signature", None), + ) + ) + response_text = getattr(response, "text", None) + if not text_parts and isinstance(response_text, str) and response_text: + text_parts.append(response_text) + response_function_calls = cast( + list[Any] | None, + getattr(response, "function_calls", None), + ) + if not tool_calls and isinstance(response_function_calls, list): + for function_call in response_function_calls: + function_name = getattr(function_call, "name", None) + function_args = getattr(function_call, "args", None) + if not isinstance(function_name, str): + continue + tool_calls.append( + ToolCallResult( + id=f"call_{function_name}_{len(tool_calls)}", + name=function_name, + input=dict(cast(dict[str, Any], function_args)) + if function_args + else {}, + ) + ) + + content: Any = "\n".join(text_parts) if text_parts else "" + if response_format is not None: + parsed_response = getattr(response, "parsed", None) + if isinstance(parsed_response, response_format): + content = parsed_response + elif isinstance(parsed_response, dict): + content = response_format.model_validate(parsed_response) + elif isinstance(parsed_response, str): + content = response_format.model_validate_json(parsed_response) + else: + if finish_reason in GEMINI_BLOCKED_FINISH_REASONS: + raise LLMError( + f"Gemini response blocked (finish_reason={finish_reason})", + provider="gemini", + model=model_name, + finish_reason=finish_reason, + ) + raw_text = "".join(text_parts) + content = repair_response_model_json( + raw_text, + response_format, + model_name, + ) + elif ( + not content + and not tool_calls + and finish_reason in GEMINI_BLOCKED_FINISH_REASONS + ): + raise LLMError( + f"Gemini response blocked (finish_reason={finish_reason})", + provider="gemini", + model=model_name, + finish_reason=finish_reason, + ) + + usage = response.usage_metadata + cache_read_input_tokens = 0 + if usage is not None: + cached_tokens = getattr(usage, "cached_content_token_count", 0) + if isinstance(cached_tokens, int): + cache_read_input_tokens = cached_tokens + return CompletionResult( + content=content, + input_tokens=usage.prompt_token_count if usage else 0, + output_tokens=usage.candidates_token_count if usage else 0, + cache_read_input_tokens=cache_read_input_tokens, + finish_reason=finish_reason, + tool_calls=tool_calls, + raw_response=response, + ) + + async def _attach_cached_content( + self, + *, + model: str, + config: dict[str, Any], + cache_policy: PromptCachePolicy, + contents: list[dict[str, Any]], + tools: list[dict[str, Any]] | None, + ) -> None: + if cache_policy.mode != "gemini_cached_content": + return + # Worth caching if there are history messages, system instruction, or tools + has_cacheable = bool( + contents or config.get("system_instruction") or config.get("tools") + ) + if not has_cacheable: + return + + cache_key = build_cache_key( + config=self._cache_model_config(model), + cache_policy=cache_policy, + cacheable_messages=contents, + tools=tools, + system_instruction=config.get("system_instruction"), + tool_config=config.get("tool_config"), + ) + cached_handle = gemini_cache_store.get(cache_key) + if cached_handle is None: + ttl_seconds = cache_policy.ttl_seconds or 300 + cache_config: dict[str, Any] = { + "system_instruction": config.get("system_instruction"), + "tools": config.get("tools"), + "tool_config": config.get("tool_config"), + "ttl": f"{ttl_seconds}s", + } + if contents: + cache_config["contents"] = contents + cached_content = await self._client.aio.caches.create( + model=model, + config=cache_config, + ) + expires_at = getattr(cached_content, "expire_time", None) + if expires_at is None: + expires_at = datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds) + cached_handle = gemini_cache_store.set( + GeminiCacheHandle( + key=cache_key, + cached_content_name=cached_content.name, + expires_at=expires_at, + ) + ) + # Once a cached-content handle is attached, Gemini rejects repeating + # system/tool configuration on the generate call. + config.pop("system_instruction", None) + config.pop("tools", None) + config.pop("tool_config", None) + config["cached_content"] = cached_handle.cached_content_name + + @staticmethod + def _cache_model_config(model: str): + from src.config import ModelConfig + + return ModelConfig(transport="gemini", model=model) + + @staticmethod + def _convert_messages( + messages: list[dict[str, Any]], + ) -> tuple[list[dict[str, Any]] | str, str | None]: + system_messages: list[str] = [] + contents: list[dict[str, Any]] = [] + + for message in messages: + role = message.get("role", "user") + if role == "system": + if isinstance(message.get("content"), str): + system_messages.append(message["content"]) + continue + + if role == "assistant": + role = "model" + + if isinstance(message.get("parts"), list): + message_copy = message.copy() + message_copy["role"] = role + contents.append(message_copy) + continue + + if isinstance(message.get("content"), str): + contents.append({"role": role, "parts": [{"text": message["content"]}]}) + continue + + if isinstance(message.get("content"), list): + parts: list[dict[str, Any]] = [] + for block in message["content"]: + block_type = block.get("type") + if block_type == "text": + parts.append({"text": block["text"]}) + else: + # Silently dropping non-"text" blocks would mask real + # input-shape bugs — e.g., an Anthropic-shaped + # tool_use/tool_result payload accidentally routed to + # the Gemini backend without going through the + # history adapter. Fail fast so the caller knows. + raise ValidationException( + "Gemini backend cannot translate content block " + + f"of type {block_type!r}; translate to " + + "Gemini-native 'parts' via the history adapter " + + "before passing to the backend" + ) + if parts: + contents.append({"role": role, "parts": parts}) + + system_instruction = "\n\n".join(system_messages) if system_messages else None + return contents, system_instruction + + @staticmethod + def _convert_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + if tools and "function_declarations" in tools[0]: + return tools + return [ + { + "function_declarations": [ + { + "name": tool["name"], + "description": tool["description"], + "parameters": GeminiBackend._sanitize_schema( + tool["input_schema"] + ), + } + for tool in tools + ] + } + ] + + # JSON-Schema keywords Gemini's function_declarations validator accepts. + # See https://ai.google.dev/api/caching#Schema. Anything outside this set + # (e.g. additionalProperties, allOf, if/then/else, $ref, anyOf, oneOf, + # patternProperties) triggers an INVALID_ARGUMENT 400 at call time, so we + # strip on the way out. Other backends keep the richer schema. + _GEMINI_ALLOWED_SCHEMA_KEYS: ClassVar[frozenset[str]] = frozenset( + { + "type", + "format", + "description", + "nullable", + "enum", + "properties", + "required", + "items", + "minItems", + "maxItems", + "minimum", + "maximum", + "title", + } + ) + + @staticmethod + def _sanitize_schema(schema: Any) -> Any: + """Recursively strip JSON-Schema keywords Gemini rejects. + + ``properties`` holds user-supplied field names → sub-schemas, so we + recurse into its values but preserve its keys. ``required`` and + ``enum`` are lists of literals (field names / allowed values) and are + passed through verbatim. Everything else is a scalar schema keyword. + """ + if not isinstance(schema, dict): + return schema + schema_dict = cast(dict[str, Any], schema) + cleaned: dict[str, Any] = {} + for key, value in schema_dict.items(): + if key not in GeminiBackend._GEMINI_ALLOWED_SCHEMA_KEYS: + continue + if key == "properties" and isinstance(value, dict): + cleaned["properties"] = { + prop_name: GeminiBackend._sanitize_schema(prop_schema) + for prop_name, prop_schema in cast(dict[str, Any], value).items() + } + elif key == "items": + cleaned["items"] = GeminiBackend._sanitize_schema(value) + elif key == "required" and isinstance(value, list): + cleaned["required"] = list(cast(list[Any], value)) + elif key == "enum" and isinstance(value, list): + cleaned["enum"] = list(cast(list[Any], value)) + else: + cleaned[key] = value + return cleaned + + @staticmethod + def _convert_tool_choice( + tool_choice: str | dict[str, Any], + ) -> dict[str, Any]: + if isinstance(tool_choice, dict) and "name" in tool_choice: + return { + "function_calling_config": { + "mode": "ANY", + "allowed_function_names": [tool_choice["name"]], + } + } + if tool_choice == "auto": + return {"function_calling_config": {"mode": "AUTO"}} + if tool_choice in {"any", "required"}: + return {"function_calling_config": {"mode": "ANY"}} + if tool_choice == "none": + return {"function_calling_config": {"mode": "NONE"}} + return { + "function_calling_config": { + "mode": "ANY", + "allowed_function_names": [tool_choice], + } + } diff --git a/src/llm/backends/openai.py b/src/llm/backends/openai.py new file mode 100644 index 00000000..1e01e78a --- /dev/null +++ b/src/llm/backends/openai.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import json +import logging +from collections.abc import AsyncIterator +from typing import Any, cast + +from openai import BadRequestError, LengthFinishReasonError +from pydantic import BaseModel, ValidationError + +from src.exceptions import ValidationException +from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult +from src.llm.structured_output import ( + repair_response_model_json, + validate_structured_output, +) + +logger = logging.getLogger(__name__) + + +def _uses_max_completion_tokens(model: str) -> bool: + """OpenAI reasoning models (gpt-5 family + o-series) require + ``max_completion_tokens`` instead of the classic ``max_tokens`` parameter. + + Matches: gpt-5, gpt-5-anything, gpt-5.anything (incl. gpt-5.4, gpt-5.4-mini), + o1*, o3*, o4*. Anything else (gpt-4.x, gpt-4o, chat models on proxies) + stays on ``max_tokens``. + """ + m = model.lower() + if m == "gpt-5" or m.startswith("gpt-5-") or m.startswith("gpt-5."): + return True + for prefix in ("o1", "o3", "o4"): + if m == prefix or m.startswith(prefix + "-"): + return True + return False + + +def extract_openai_reasoning_content(response: Any) -> str | None: + try: + message = response.choices[0].message + if hasattr(message, "reasoning_details") and message.reasoning_details: + reasoning_parts: list[str] = [] + for detail in message.reasoning_details: + detail_content = getattr(detail, "content", None) + if isinstance(detail_content, str) and detail_content: + reasoning_parts.append(detail_content) + elif isinstance(detail, dict): + detail_dict = cast(dict[str, Any], detail) + dict_content = detail_dict.get("content") + if isinstance(dict_content, str) and dict_content: + reasoning_parts.append(dict_content) + if reasoning_parts: + return "\n".join(reasoning_parts) + if hasattr(message, "reasoning_content") and message.reasoning_content: + return message.reasoning_content + except (AttributeError, IndexError, TypeError): + return None + return None + + +def extract_openai_reasoning_details(response: Any) -> list[dict[str, Any]]: + try: + message = response.choices[0].message + if hasattr(message, "reasoning_details") and message.reasoning_details: + details: list[dict[str, Any]] = [] + for detail in message.reasoning_details: + if hasattr(detail, "model_dump"): + dumped = detail.model_dump() + if isinstance(dumped, dict): + details.append(cast(dict[str, Any], dumped)) + elif isinstance(detail, dict): + details.append(cast(dict[str, Any], detail)) + else: + detail_content = getattr(detail, "content", None) + if isinstance(detail_content, str) and detail_content: + details.append({"content": detail_content}) + return details + except (AttributeError, IndexError, TypeError): + return [] + return [] + + +def extract_openai_cache_tokens(usage: Any) -> tuple[int, int]: + if not usage: + return 0, 0 + + cache_read = 0 + if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: + details = usage.prompt_tokens_details + if hasattr(details, "cached_tokens") and details.cached_tokens: + cache_read = details.cached_tokens + + if cache_read == 0: + if hasattr(usage, "cache_read_input_tokens") and usage.cache_read_input_tokens: + cache_read = usage.cache_read_input_tokens + elif hasattr(usage, "cached_tokens") and usage.cached_tokens: + cache_read = usage.cached_tokens + + cache_creation = 0 + if ( + hasattr(usage, "cache_creation_input_tokens") + and usage.cache_creation_input_tokens + ): + cache_creation = usage.cache_creation_input_tokens + + return cache_creation, cache_read + + +class OpenAIBackend: + """Provider backend wrapping AsyncOpenAI.""" + + def __init__(self, client: Any) -> None: + self._client: Any = client + + async def complete( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> CompletionResult: + if thinking_budget_tokens is not None: + raise ValidationException( + "OpenAI backend does not support thinking_budget_tokens; use thinking_effort instead" + ) + + params = self._build_params( + model=model, + messages=messages, + max_tokens=max_output_tokens or max_tokens, + temperature=temperature, + stop=stop, + tools=tools, + tool_choice=tool_choice, + thinking_effort=thinking_effort, + extra_params=extra_params, + ) + + if isinstance(response_format, type): + params["response_format"] = response_format + try: + response = await self._client.chat.completions.parse(**params) + except LengthFinishReasonError as exc: + truncated = exc.completion + raw_content = truncated.choices[0].message.content or "" + content = repair_response_model_json( + raw_content, + response_format, + model, + ) + return self._normalize_response( + truncated, + content_override=content, + ) + except (BadRequestError, json.JSONDecodeError, ValidationError): + fallback_response = await self._create_structured_response( + params=params, + response_format=response_format, + ) + content = self._parse_or_repair_structured_content( + fallback_response, + response_format, + model, + ) + return self._normalize_response( + fallback_response, + content_override=content, + ) + parsed = response.choices[0].message.parsed + raw_content = response.choices[0].message.content or "" + if parsed is None and raw_content: + content = repair_response_model_json( + raw_content, + response_format, + model, + ) + return self._normalize_response(response, content_override=content) + if parsed is None: + refusal = getattr(response.choices[0].message, "refusal", None) + if refusal: + return self._normalize_response( + response, + content_override=refusal, + ) + raise ValidationException("No parsed content in structured response") + return self._normalize_response( + response, + content_override=validate_structured_output(parsed, response_format), + ) + if response_format is not None: + params["response_format"] = response_format + + if extra_params and extra_params.get("json_mode"): + params["response_format"] = {"type": "json_object"} + + response = await self._client.chat.completions.create(**params) + return self._normalize_response(response) + + async def stream( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None = None, + stop: list[str] | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + thinking_budget_tokens: int | None = None, + thinking_effort: str | None = None, + max_output_tokens: int | None = None, + extra_params: dict[str, Any] | None = None, + ) -> AsyncIterator[StreamChunk]: + if thinking_budget_tokens is not None: + raise ValidationException( + "OpenAI backend does not support thinking_budget_tokens; use thinking_effort instead" + ) + + params = self._build_params( + model=model, + messages=messages, + max_tokens=max_output_tokens or max_tokens, + temperature=temperature, + stop=stop, + tools=tools, + tool_choice=tool_choice, + thinking_effort=thinking_effort, + extra_params=extra_params, + ) + params["stream"] = True + params["stream_options"] = {"include_usage": True} + if isinstance(response_format, type): + # parse() supports BaseModel types but streaming create() does not — + # convert to a json_schema dict so the streaming path works. + params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": response_format.__name__, + "schema": response_format.model_json_schema(), + }, + } + elif response_format is not None: + params["response_format"] = response_format + elif extra_params and extra_params.get("json_mode"): + params["response_format"] = {"type": "json_object"} + + response_stream = await self._client.chat.completions.create(**params) + finish_reason: str | None = None + usage_chunk_received = False + async for chunk in response_stream: + if chunk.choices and chunk.choices[0].delta.content: + yield StreamChunk(content=chunk.choices[0].delta.content) + if chunk.choices and chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + if hasattr(chunk, "usage") and chunk.usage: + yield StreamChunk( + is_done=True, + finish_reason=finish_reason, + output_tokens=chunk.usage.completion_tokens, + ) + usage_chunk_received = True + + if not usage_chunk_received and finish_reason: + yield StreamChunk(is_done=True, finish_reason=finish_reason) + + def _build_params( + self, + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float | None, + stop: list[str] | None, + tools: list[dict[str, Any]] | None, + tool_choice: str | dict[str, Any] | None, + thinking_effort: str | None, + extra_params: dict[str, Any] | None, + ) -> dict[str, Any]: + params: dict[str, Any] = { + "model": model, + "messages": messages, + } + + if _uses_max_completion_tokens(model): + params["max_completion_tokens"] = max_tokens + if extra_params and extra_params.get("verbosity"): + params["verbosity"] = extra_params["verbosity"] + else: + params["max_tokens"] = max_tokens + + if temperature is not None: + params["temperature"] = temperature + + if thinking_effort: + params["reasoning_effort"] = thinking_effort + + if stop: + params["stop"] = stop + if tools: + params["tools"] = self._convert_tools(tools) + if tool_choice is not None: + params["tool_choice"] = tool_choice + if extra_params: + for key in ( + "top_p", + "frequency_penalty", + "presence_penalty", + "seed", + ): + if key in extra_params: + params[key] = extra_params[key] + return params + + def _normalize_response( + self, + response: Any, + *, + content_override: Any | None = None, + ) -> CompletionResult: + usage = response.usage + finish_reason = response.choices[0].finish_reason + tool_calls: list[ToolCallResult] = [] + message = response.choices[0].message + if getattr(message, "tool_calls", None): + for tool_call in message.tool_calls: + tool_input: dict[str, Any] = {} + if tool_call.function.arguments: + try: + tool_input = json.loads(tool_call.function.arguments) + except (json.JSONDecodeError, TypeError) as exc: + # Don't log the raw arguments payload — LLM-generated + # tool calls can mirror user PII from the prompt into + # their arguments, and this runs at WARN level. + logger.warning( + "Malformed tool arguments for %s (id=%s): %s", + tool_call.function.name, + tool_call.id, + exc.__class__.__name__, + ) + tool_calls.append( + ToolCallResult( + id=tool_call.id, + name=tool_call.function.name, + input=tool_input, + ) + ) + + cache_creation, cache_read = extract_openai_cache_tokens(usage) + return CompletionResult( + content=content_override + if content_override is not None + else (message.content or ""), + input_tokens=usage.prompt_tokens if usage else 0, + output_tokens=usage.completion_tokens if usage else 0, + cache_creation_input_tokens=cache_creation, + cache_read_input_tokens=cache_read, + finish_reason=finish_reason or "stop", + tool_calls=tool_calls, + thinking_content=extract_openai_reasoning_content(response), + reasoning_details=extract_openai_reasoning_details(response), + raw_response=response, + ) + + async def _create_structured_response( + self, + *, + params: dict[str, Any], + response_format: type[BaseModel], + ) -> Any: + structured_params = dict(params) + structured_params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": response_format.__name__, + "schema": response_format.model_json_schema(), + }, + } + return await self._client.chat.completions.create(**structured_params) + + @staticmethod + def _parse_or_repair_structured_content( + response: Any, + response_format: type[BaseModel], + model: str, + ) -> BaseModel | str: + raw_content = response.choices[0].message.content or "" + if raw_content: + return repair_response_model_json(raw_content, response_format, model) + refusal = getattr(response.choices[0].message, "refusal", None) + if refusal: + return refusal + raise ValidationException( + "No raw content available for structured output repair" + ) + + @staticmethod + def _convert_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + if not tools or tools[0].get("type") == "function": + return tools + # Tool schemas in src/utils/agent_tools.py use optional fields with + # defaults and don't declare additionalProperties: false. OpenAI's + # strict function-calling mode forbids both, so we intentionally + # don't set strict: True. Standard function calling on GPT-4.x / + # GPT-5 remains reliable, and this stays compatible with + # OpenAI-compatible proxies (OpenRouter, Together, vLLM, Ollama) + # whose strict-mode support is inconsistent. + return [ + { + "type": "function", + "function": { + "name": tool["name"], + "description": tool["description"], + "parameters": tool["input_schema"], + }, + } + for tool in tools + ] diff --git a/src/llm/caching.py b/src/llm/caching.py new file mode 100644 index 00000000..75020dce --- /dev/null +++ b/src/llm/caching.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import hashlib +import json +from collections import OrderedDict +from datetime import datetime, timezone +from threading import Lock +from typing import Any + +from pydantic import BaseModel + +from src.config import ModelConfig, PromptCachePolicy + +__all__ = [ + "GeminiCacheHandle", + "InMemoryGeminiCacheStore", + "PromptCachePolicy", + "build_cache_key", + "gemini_cache_store", +] + + +class GeminiCacheHandle(BaseModel): + key: str + cached_content_name: str + expires_at: datetime + + +def build_cache_key( + *, + config: ModelConfig, + cache_policy: PromptCachePolicy, + cacheable_messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None, + system_instruction: str | None = None, + tool_config: dict[str, Any] | None = None, +) -> str: + """Deterministic key over the cacheable shape of a request. + + ``system_instruction`` and ``tool_config`` must be part of the key + because the provider's cached-content handle captures them at creation + time — two requests that differ only by system prompt or tool + constraints would otherwise hit the same cached handle and silently get + the wrong system prompt / tool policy. + """ + payload = { + "transport": config.transport, + "model": config.model, + "cache_policy": cache_policy.model_dump(mode="json"), + "messages": cacheable_messages, + "tools": tools, + "system_instruction": system_instruction, + "tool_config": tool_config, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest() + return f"llm-cache:{cache_policy.key_version}:{digest}" + + +class InMemoryGeminiCacheStore: + """Best-effort local cache for Gemini cached-content handles. + + Uses LRU eviction with a max entry limit to prevent unbounded growth. + """ + + MAX_ENTRIES: int = 1024 + + def __init__(self) -> None: + self._handles: OrderedDict[str, GeminiCacheHandle] = OrderedDict() + self._lock: Lock = Lock() + + def get(self, key: str) -> GeminiCacheHandle | None: + with self._lock: + handle = self._handles.get(key) + if handle is None: + return None + if handle.expires_at <= datetime.now(timezone.utc): + self._handles.pop(key, None) + return None + self._handles.move_to_end(key) + return handle + + def set(self, handle: GeminiCacheHandle) -> GeminiCacheHandle: + with self._lock: + now = datetime.now(timezone.utc) + expired = [k for k, h in self._handles.items() if h.expires_at <= now] + for k in expired: + self._handles.pop(k, None) + if handle.key in self._handles: + self._handles.move_to_end(handle.key) + self._handles[handle.key] = handle + while len(self._handles) > self.MAX_ENTRIES: + self._handles.popitem(last=False) + return handle + + +gemini_cache_store = InMemoryGeminiCacheStore() diff --git a/src/llm/conversation.py b/src/llm/conversation.py new file mode 100644 index 00000000..4697b819 --- /dev/null +++ b/src/llm/conversation.py @@ -0,0 +1,185 @@ +"""Conversation-shaping helpers: token counting + tool-aware truncation. + +Moved out of src/utils/clients.py as part of the migration into src/llm/. +These are pure helpers with no orchestration dependencies. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, cast + +from src.utils.tokens import estimate_tokens + +logger = logging.getLogger(__name__) + + +def count_message_tokens(messages: list[dict[str, Any]]) -> int: + """Count tokens in a list of messages using tiktoken.""" + total = 0 + for msg in messages: + content = msg.get("content", "") + if isinstance(content, str): + total += estimate_tokens(content) + elif isinstance(content, list): + # Anthropic-style content blocks + total += estimate_tokens(json.dumps(content)) + if "parts" in msg: + try: + total += estimate_tokens(json.dumps(msg["parts"])) + except TypeError: + # Non-JSON-serializable content (e.g. bytes) — estimate from repr. + total += estimate_tokens(str(msg["parts"])) + return total + + +def _is_tool_use_message(msg: dict[str, Any]) -> bool: + """Check if a message contains tool calls (any format). + + Recognizes: + - Anthropic: ``content`` is a list containing a ``{"type": "tool_use"}`` block. + - Gemini: ``parts`` is a list containing a ``{"function_call": …}`` entry. + - OpenAI: assistant message with a non-empty ``tool_calls`` field. + """ + content = msg.get("content") + if isinstance(content, list): + for block in cast(list[dict[str, Any]], content): + if block.get("type") == "tool_use": + return True + parts = msg.get("parts") + if isinstance(parts, list): + for part in cast(list[dict[str, Any]], parts): + if "function_call" in part: + return True + return bool(msg.get("tool_calls")) + + +def _is_tool_result_message(msg: dict[str, Any]) -> bool: + """Check if a message contains tool results (any format). + + Recognizes: + - Anthropic: ``content`` is a list containing a ``{"type": "tool_result"}`` block. + - Gemini: ``parts`` is a list containing a ``{"function_response": …}`` entry. + - OpenAI: message with ``role == "tool"``. + """ + content = msg.get("content") + if isinstance(content, list): + for block in cast(list[dict[str, Any]], content): + if block.get("type") == "tool_result": + return True + parts = msg.get("parts") + if isinstance(parts, list): + for part in cast(list[dict[str, Any]], parts): + if "function_response" in part: + return True + return msg.get("role") == "tool" + + +def _group_into_units( + messages: list[dict[str, Any]], +) -> list[list[dict[str, Any]]]: + """Group messages into logical conversation units. + + A unit is either: + - A tool_use message + ALL consecutive tool_result messages that follow + - A single non-tool message + + Keeps tool_use / tool_result pairs together so truncation never breaks + them apart. + """ + units: list[list[dict[str, Any]]] = [] + i = 0 + + while i < len(messages): + msg = messages[i] + + if _is_tool_use_message(msg): + j = i + 1 + while j < len(messages) and _is_tool_result_message(messages[j]): + j += 1 + unit = messages[i:j] + if len(unit) > 1: + units.append(unit) + i = j + else: + # Orphaned tool_use with no results — skip it. + logger.debug(f"Skipping orphaned tool_use at index {i}") + i += 1 + elif _is_tool_result_message(msg): + # Orphaned tool_result — skip it. + logger.debug(f"Skipping orphaned tool_result at index {i}") + i += 1 + else: + units.append([msg]) + i += 1 + + return units + + +def truncate_messages_to_fit( + messages: list[dict[str, Any]], + max_tokens: int, + preserve_system: bool = True, +) -> list[dict[str, Any]]: + """Truncate messages to fit within a token limit while maintaining valid structure. + + Strategy: + 1. Group messages into units (tool_use + results together, or single messages) + 2. Remove oldest units first to preserve recent context + 3. Units stay intact so tool_use/tool_result pairs are never broken + """ + current_tokens = count_message_tokens(messages) + if current_tokens <= max_tokens: + return messages + + logger.info(f"Truncating: {current_tokens} tokens exceeds {max_tokens} limit") + + system_messages: list[dict[str, Any]] = [] + conversation: list[dict[str, Any]] = [] + + for msg in messages: + if msg.get("role") == "system" and preserve_system: + system_messages.append(msg) + else: + conversation.append(msg) + + system_tokens = count_message_tokens(system_messages) + available_tokens = max_tokens - system_tokens + + if available_tokens <= 0: + logger.warning("System message exceeds max_input_tokens") + return messages + + units = _group_into_units(conversation) + + if not units: + logger.warning("No valid conversation units") + return system_messages + + # Drop oldest units until conversation fits, but keep at least one unit so + # we never erase the entire non-system conversation. + while len(units) > 1: + flat_messages = [m for unit in units for m in unit] + if count_message_tokens(flat_messages) <= available_tokens: + break + removed_unit = units.pop(0) + logger.debug( + "Dropping conversation unit with " + + f"{len(removed_unit)} messages " + + f"(~{count_message_tokens(removed_unit)} tokens)" + ) + + result = system_messages + [m for unit in units for m in unit] + result_tokens = count_message_tokens(result) + logger.info( + f"Truncation complete: {current_tokens} → {result_tokens} tokens " + + f"({len(messages)} → {len(result)} messages)" + ) + return result + + +__all__ = [ + "count_message_tokens", + "truncate_messages_to_fit", +] diff --git a/src/llm/credentials.py b/src/llm/credentials.py new file mode 100644 index 00000000..9b41e77d --- /dev/null +++ b/src/llm/credentials.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from src.config import ModelConfig, settings +from src.exceptions import ValidationException + + +def resolve_credentials(config: ModelConfig) -> dict[str, str | None]: + """Resolve credentials for the effective model transport.""" + + default_api_key = default_transport_api_key(config.transport) + return { + "api_key": config.api_key or default_api_key, + "api_base": config.base_url, + } + + +def default_transport_api_key(transport: str) -> str | None: + """Fall back to the global LLM API key for the matching transport.""" + if transport == "anthropic": + return settings.LLM.ANTHROPIC_API_KEY + if transport == "openai": + return settings.LLM.OPENAI_API_KEY + if transport == "gemini": + return settings.LLM.GEMINI_API_KEY + raise ValidationException(f"Unknown transport: {transport}") diff --git a/src/llm/executor.py b/src/llm/executor.py new file mode 100644 index 00000000..d96008af --- /dev/null +++ b/src/llm/executor.py @@ -0,0 +1,226 @@ +"""Single-call executor: the inner LLM-call path without tool-loop orchestration. + +`honcho_llm_call_inner` handles one backend call (complete or stream), building +the effective ModelConfig and delegating to request_builder. Result / stream +chunk types are bridged to the public Honcho* shapes here. + +Used by: +- src/llm/api.py (the public entrypoint, for both tool-less and tool-enabled paths) +- src/llm/tool_loop.py (each iteration of the tool loop calls this) +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any, Literal, TypeVar, overload + +from pydantic import BaseModel + +from src.config import ModelConfig, ModelTransport + +from .backend import CompletionResult as BackendCompletionResult +from .backend import StreamChunk as BackendStreamChunk +from .backend import ToolCallResult +from .registry import CLIENTS, backend_for_provider +from .request_builder import execute_completion, execute_stream +from .runtime import effective_config_for_call +from .types import ( + HonchoLLMCallResponse, + HonchoLLMCallStreamChunk, + ProviderClient, + ReasoningEffortType, +) + +M = TypeVar("M", bound=BaseModel) + + +def _tool_call_result_to_dict(tool_call: ToolCallResult) -> dict[str, Any]: + result = { + "id": tool_call.id, + "name": tool_call.name, + "input": tool_call.input, + } + if tool_call.thought_signature is not None: + result["thought_signature"] = tool_call.thought_signature + return result + + +def completion_result_to_response( + result: BackendCompletionResult, +) -> HonchoLLMCallResponse[Any]: + return HonchoLLMCallResponse( + content=result.content, + input_tokens=result.input_tokens, + output_tokens=result.output_tokens, + cache_creation_input_tokens=result.cache_creation_input_tokens, + cache_read_input_tokens=result.cache_read_input_tokens, + finish_reasons=[result.finish_reason] if result.finish_reason else [], + tool_calls_made=[_tool_call_result_to_dict(tc) for tc in result.tool_calls], + thinking_content=result.thinking_content, + thinking_blocks=result.thinking_blocks, + reasoning_details=result.reasoning_details, + ) + + +def stream_chunk_to_response_chunk( + chunk: BackendStreamChunk, +) -> HonchoLLMCallStreamChunk: + return HonchoLLMCallStreamChunk( + content=chunk.content, + is_done=chunk.is_done, + finish_reasons=[chunk.finish_reason] if chunk.finish_reason else [], + output_tokens=chunk.output_tokens, + ) + + +@overload +async def honcho_llm_call_inner( + provider: ModelTransport, + model: str, + prompt: str, + max_tokens: int, + response_model: type[M], + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + stream: Literal[False] = False, + client_override: ProviderClient | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + messages: list[dict[str, Any]] | None = None, + selected_config: ModelConfig | None = None, +) -> HonchoLLMCallResponse[M]: ... + + +@overload +async def honcho_llm_call_inner( + provider: ModelTransport, + model: str, + prompt: str, + max_tokens: int, + response_model: None = None, + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + stream: Literal[False] = False, + client_override: ProviderClient | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + messages: list[dict[str, Any]] | None = None, + selected_config: ModelConfig | None = None, +) -> HonchoLLMCallResponse[str]: ... + + +@overload +async def honcho_llm_call_inner( + provider: ModelTransport, + model: str, + prompt: str, + max_tokens: int, + response_model: type[BaseModel] | None = None, + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + stream: Literal[True] = ..., + client_override: ProviderClient | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + messages: list[dict[str, Any]] | None = None, + selected_config: ModelConfig | None = None, +) -> AsyncIterator[HonchoLLMCallStreamChunk]: ... + + +async def honcho_llm_call_inner( + provider: ModelTransport, + model: str, + prompt: str, + max_tokens: int, + response_model: type[BaseModel] | None = None, + json_mode: bool = False, + temperature: float | None = None, + stop_seqs: list[str] | None = None, + reasoning_effort: ReasoningEffortType = None, + verbosity: Literal["low", "medium", "high"] | None = None, + thinking_budget_tokens: int | None = None, + stream: bool = False, + client_override: ProviderClient | None = None, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + messages: list[dict[str, Any]] | None = None, + selected_config: ModelConfig | None = None, +) -> HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk]: + """One backend call. No retry, no fallback, no tool loop. + + The outer src/llm/api.py `honcho_llm_call` handles retry + fallback + + tool orchestration on top of this. + """ + client = client_override or CLIENTS.get(provider) + if client is None: + raise ValueError(f"Missing client for {provider}") + + if messages is None: + messages = [{"role": "user", "content": prompt}] + + backend = backend_for_provider(provider, client) + + effective_config = effective_config_for_call( + selected_config=selected_config, + provider=provider, + model=model, + temperature=temperature, + stop_seqs=stop_seqs, + thinking_budget_tokens=thinking_budget_tokens, + reasoning_effort=reasoning_effort, + ) + # json_mode + verbosity are per-call transport toggles, not ModelConfig + # knobs — they pass through extra_params. execute_completion merges + # build_config_extra_params(effective_config) on top for top_p/seed/etc. + call_extras: dict[str, Any] = {"json_mode": json_mode, "verbosity": verbosity} + + if stream: + + async def _stream() -> AsyncIterator[HonchoLLMCallStreamChunk]: + stream_iter = await execute_stream( + backend, + effective_config, + messages=messages, + max_tokens=max_tokens, + tools=tools, + tool_choice=tool_choice, + response_format=response_model, + cache_policy=effective_config.cache_policy, + extra_params=call_extras, + ) + async for chunk in stream_iter: + yield stream_chunk_to_response_chunk(chunk) + + return _stream() + + result = await execute_completion( + backend, + effective_config, + messages=messages, + max_tokens=max_tokens, + tools=tools, + tool_choice=tool_choice, + response_format=response_model, + cache_policy=effective_config.cache_policy, + extra_params=call_extras, + ) + return completion_result_to_response(result) + + +__all__ = [ + "completion_result_to_response", + "honcho_llm_call_inner", + "stream_chunk_to_response_chunk", +] diff --git a/src/llm/history_adapters.py b/src/llm/history_adapters.py new file mode 100644 index 00000000..02d2ea05 --- /dev/null +++ b/src/llm/history_adapters.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import json +from typing import Any, Protocol + +from .backend import CompletionResult + + +class HistoryAdapter(Protocol): + def format_assistant_tool_message( + self, + result: CompletionResult, + ) -> dict[str, Any]: ... + + def format_tool_results( + self, + tool_results: list[dict[str, Any]], + ) -> list[dict[str, Any]]: ... + + +class AnthropicHistoryAdapter: + def format_assistant_tool_message( + self, + result: CompletionResult, + ) -> dict[str, Any]: + content_blocks: list[dict[str, Any]] = [] + if result.thinking_blocks: + content_blocks.extend(result.thinking_blocks) + if isinstance(result.content, str) and result.content: + content_blocks.append({"type": "text", "text": result.content}) + for tool_call in result.tool_calls: + content_blocks.append( + { + "type": "tool_use", + "id": tool_call.id, + "name": tool_call.name, + "input": tool_call.input, + } + ) + return {"role": "assistant", "content": content_blocks} + + def format_tool_results( + self, + tool_results: list[dict[str, Any]], + ) -> list[dict[str, Any]]: + return [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": tr["tool_id"], + "content": str(tr["result"]), + "is_error": tr.get("is_error", False), + } + for tr in tool_results + ], + } + ] + + +class GeminiHistoryAdapter: + def format_assistant_tool_message( + self, + result: CompletionResult, + ) -> dict[str, Any]: + parts: list[dict[str, Any]] = [] + if isinstance(result.content, str) and result.content: + parts.append({"text": result.content}) + for tool_call in result.tool_calls: + part: dict[str, Any] = { + "function_call": { + "name": tool_call.name, + "args": tool_call.input, + } + } + if tool_call.thought_signature is not None: + part["thought_signature"] = tool_call.thought_signature + parts.append(part) + return {"role": "model", "parts": parts} + + def format_tool_results( + self, + tool_results: list[dict[str, Any]], + ) -> list[dict[str, Any]]: + return [ + { + "role": "user", + "parts": [ + { + "function_response": { + "name": tr["tool_name"], + "response": {"result": str(tr["result"])}, + } + } + for tr in tool_results + ], + } + ] + + +class OpenAIHistoryAdapter: + def format_assistant_tool_message( + self, + result: CompletionResult, + ) -> dict[str, Any]: + message: dict[str, Any] = { + "role": "assistant", + "content": result.content if isinstance(result.content, str) else None, + "tool_calls": [ + { + "id": tool_call.id, + "type": "function", + "function": { + "name": tool_call.name, + "arguments": json.dumps(tool_call.input), + }, + } + for tool_call in result.tool_calls + ], + } + if result.reasoning_details: + message["reasoning_details"] = result.reasoning_details + return message + + def format_tool_results( + self, + tool_results: list[dict[str, Any]], + ) -> list[dict[str, Any]]: + return [ + { + "role": "tool", + "tool_call_id": tr["tool_id"], + "content": str(tr["result"]), + } + for tr in tool_results + ] diff --git a/src/llm/registry.py b/src/llm/registry.py new file mode 100644 index 00000000..73cf60c8 --- /dev/null +++ b/src/llm/registry.py @@ -0,0 +1,185 @@ +"""Single owner of provider runtime objects: clients, backends, history adapters. + +Consolidates wiring that previously lived in both `src/llm/__init__.py` and +`src/utils/clients.py`. Everything that touches provider SDKs at runtime +(default client construction, override client caching, backend selection, +history adapter selection) lives here now. +""" + +from __future__ import annotations + +from functools import lru_cache +from typing import assert_never + +from anthropic import AsyncAnthropic +from google import genai +from google.genai import types as genai_types +from openai import AsyncOpenAI + +from src.config import ModelConfig, ModelTransport, settings +from src.exceptions import ValidationException + +from .backend import ProviderBackend +from .backends.anthropic import AnthropicBackend +from .backends.gemini import GeminiBackend +from .backends.openai import OpenAIBackend +from .credentials import default_transport_api_key +from .history_adapters import ( + AnthropicHistoryAdapter, + GeminiHistoryAdapter, + HistoryAdapter, + OpenAIHistoryAdapter, +) +from .types import ProviderClient + + +@lru_cache(maxsize=1) +def get_anthropic_client() -> AsyncAnthropic: + """Default Anthropic client built from settings.LLM.ANTHROPIC_API_KEY.""" + return AsyncAnthropic( + api_key=settings.LLM.ANTHROPIC_API_KEY, + timeout=600.0, + ) + + +@lru_cache(maxsize=1) +def get_openai_client() -> AsyncOpenAI: + """Default OpenAI client built from settings.LLM.OPENAI_API_KEY.""" + return AsyncOpenAI( + api_key=settings.LLM.OPENAI_API_KEY, + ) + + +@lru_cache(maxsize=1) +def get_gemini_client() -> genai.Client: + """Default Gemini client built from settings.LLM.GEMINI_API_KEY.""" + return genai.Client(api_key=settings.LLM.GEMINI_API_KEY) + + +# Bounded cache — in practice the (base_url, api_key) key space is small +# and process-scoped, but maxsize=128 keeps worst-case memory predictable. +@lru_cache(maxsize=128) +def get_openai_override_client( + base_url: str | None, api_key: str | None +) -> AsyncOpenAI: + """OpenAI client for a specific (base_url, api_key) pair. Cached by key.""" + return AsyncOpenAI(api_key=api_key, base_url=base_url) + + +@lru_cache(maxsize=128) +def get_anthropic_override_client( + base_url: str | None, + api_key: str | None, +) -> AsyncAnthropic: + """Anthropic client for a specific (base_url, api_key) pair. Cached by key.""" + return AsyncAnthropic(api_key=api_key, base_url=base_url, timeout=600.0) + + +@lru_cache(maxsize=128) +def get_gemini_override_client( + base_url: str | None, api_key: str | None +) -> genai.Client: + """Gemini client for a specific (base_url, api_key) pair. Cached by key.""" + http_options = genai_types.HttpOptions(base_url=base_url) if base_url else None + return genai.Client(api_key=api_key, http_options=http_options) + + +# Module-level default-client registry, populated at import time. Tests patch +# this dict via `patch.dict(CLIENTS, {...})` to inject mock provider clients. +CLIENTS: dict[ModelTransport, ProviderClient] = {} + +if settings.LLM.ANTHROPIC_API_KEY: + CLIENTS["anthropic"] = AsyncAnthropic( + api_key=settings.LLM.ANTHROPIC_API_KEY, + timeout=600.0, + ) + +if settings.LLM.OPENAI_API_KEY: + CLIENTS["openai"] = AsyncOpenAI( + api_key=settings.LLM.OPENAI_API_KEY, + ) + +if settings.LLM.GEMINI_API_KEY: + CLIENTS["gemini"] = genai.client.Client( + api_key=settings.LLM.GEMINI_API_KEY, + ) + + +def client_for_model_config( + provider: ModelTransport, + model_config: ModelConfig, +) -> ProviderClient: + """Resolve the provider client for a ModelConfig. + + Fast path: no overrides → reuse the module-level default client from + CLIENTS (the test-mockable seam). Otherwise route through the cached + override factories. + """ + if model_config.api_key is None and model_config.base_url is None: + existing_client = CLIENTS.get(provider) + if existing_client is not None: + return existing_client + + api_key = model_config.api_key or default_transport_api_key(provider) + base_url = model_config.base_url + if not api_key: + raise ValidationException(f"Missing API key for {provider} model config") + + if provider == "anthropic": + return get_anthropic_override_client(base_url, api_key) + if provider == "openai": + return get_openai_override_client(base_url, api_key) + if provider == "gemini": + return get_gemini_override_client(base_url, api_key) + assert_never(provider) + + +def backend_for_provider( + provider: ModelTransport, + client: ProviderClient, +) -> ProviderBackend: + """Wrap a raw provider SDK client in the matching ProviderBackend adapter.""" + if provider == "anthropic": + return AnthropicBackend(client) + if provider == "openai": + return OpenAIBackend(client) + if provider == "gemini": + return GeminiBackend(client) + assert_never(provider) + + +def history_adapter_for_provider(provider: ModelTransport) -> HistoryAdapter: + """Provider-appropriate HistoryAdapter for assistant/tool message formatting.""" + if provider == "anthropic": + return AnthropicHistoryAdapter() + if provider == "gemini": + return GeminiHistoryAdapter() + return OpenAIHistoryAdapter() + + +def get_backend(config: ModelConfig) -> ProviderBackend: + """High-level one-shot backend factory: ModelConfig → ProviderBackend. + + Delegates client resolution to ``client_for_model_config``, which owns + the CLIENTS fast-path and the missing-API-key validation. Both the + production path (via ``honcho_llm_call_inner``) and the live-test path + (via this function) now construct clients through the same helper, so + validation behavior stays consistent. + """ + client = client_for_model_config(config.transport, config) + return backend_for_provider(config.transport, client) + + +__all__ = [ + "CLIENTS", + "backend_for_provider", + "client_for_model_config", + "get_anthropic_client", + "get_anthropic_override_client", + "get_backend", + "get_gemini_client", + "get_gemini_override_client", + "get_openai_client", + "get_openai_override_client", + "history_adapter_for_provider", +] diff --git a/src/llm/request_builder.py b/src/llm/request_builder.py new file mode 100644 index 00000000..d6be5a22 --- /dev/null +++ b/src/llm/request_builder.py @@ -0,0 +1,119 @@ +"""Low-level request assembly: flatten a ModelConfig into backend calls. + +Does NOT own: retry, fallback, tool loop, provider selection. Those live in +src/llm/api.py, src/llm/tool_loop.py, src/llm/runtime.py. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from pydantic import BaseModel + +from src.config import ModelConfig, PromptCachePolicy + +from .backend import CompletionResult, ProviderBackend, StreamChunk + + +def build_config_extra_params(config: ModelConfig) -> dict[str, Any]: + """Flatten ModelConfig's optional knobs and provider_params into extra_params. + + Backends read per-call tuning parameters (top_p, top_k, frequency_penalty, + presence_penalty, seed) and the free-form provider_params passthrough out + of ``extra_params``. Single source of truth for that translation. + """ + extra_params: dict[str, Any] = {} + if config.top_p is not None: + extra_params["top_p"] = config.top_p + if config.top_k is not None: + extra_params["top_k"] = config.top_k + if config.frequency_penalty is not None: + extra_params["frequency_penalty"] = config.frequency_penalty + if config.presence_penalty is not None: + extra_params["presence_penalty"] = config.presence_penalty + if config.seed is not None: + extra_params["seed"] = config.seed + + if config.provider_params: + extra_params.update(config.provider_params) + + return extra_params + + +async def execute_completion( + backend: ProviderBackend, + config: ModelConfig, + *, + messages: list[dict[str, Any]], + max_tokens: int, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + stop: list[str] | None = None, + cache_policy: PromptCachePolicy | None = None, + extra_params: dict[str, Any] | None = None, +) -> CompletionResult: + # Preserve 0 as an explicit "disable thinking" value (used by Gemini); + # only convert to None when the field is truly unset. + effective_max_tokens = config.max_output_tokens or max_tokens + + merged_extra_params = { + **build_config_extra_params(config), + **(extra_params or {}), + } + if cache_policy is not None: + merged_extra_params["cache_policy"] = cache_policy + + return await backend.complete( + model=config.model, + messages=messages, + max_tokens=effective_max_tokens, + temperature=config.temperature, + stop=stop if stop is not None else config.stop_sequences, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + thinking_budget_tokens=config.thinking_budget_tokens, + thinking_effort=config.thinking_effort, + max_output_tokens=effective_max_tokens, + extra_params=merged_extra_params, + ) + + +async def execute_stream( + backend: ProviderBackend, + config: ModelConfig, + *, + messages: list[dict[str, Any]], + max_tokens: int, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, + stop: list[str] | None = None, + cache_policy: PromptCachePolicy | None = None, + extra_params: dict[str, Any] | None = None, +) -> AsyncIterator[StreamChunk]: + effective_max_tokens = config.max_output_tokens or max_tokens + + merged_extra_params = { + **build_config_extra_params(config), + **(extra_params or {}), + } + if cache_policy is not None: + merged_extra_params["cache_policy"] = cache_policy + + return backend.stream( + model=config.model, + messages=messages, + max_tokens=effective_max_tokens, + temperature=config.temperature, + stop=stop if stop is not None else config.stop_sequences, + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + thinking_budget_tokens=config.thinking_budget_tokens, + thinking_effort=config.thinking_effort, + max_output_tokens=effective_max_tokens, + extra_params=merged_extra_params, + ) diff --git a/src/llm/runtime.py b/src/llm/runtime.py new file mode 100644 index 00000000..27bc56d9 --- /dev/null +++ b/src/llm/runtime.py @@ -0,0 +1,207 @@ +"""Runtime config planning and retry/fallback selection. + +Owns: +- Resolution of ConfiguredModelSettings → ModelConfig. +- Per-attempt planning (AttemptPlan) including primary/fallback selection and + reasoning-effort/thinking-budget resolution. +- Per-call effective config construction (applying caller kwarg overrides onto + the selected ModelConfig). +- Retry attempt tracking via a ContextVar, plus the temperature-bump heuristic. +""" + +from __future__ import annotations + +import logging +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Any + +from src.config import ( + ConfiguredModelSettings, + ModelConfig, + ModelTransport, + resolve_model_config, +) + +from .registry import backend_for_provider, client_for_model_config +from .types import ProviderClient, ReasoningEffortType + +logger = logging.getLogger(__name__) + +# ContextVar tracking the current retry attempt for provider switching. +current_attempt: ContextVar[int] = ContextVar("current_attempt", default=0) + + +@dataclass(frozen=True) +class AttemptPlan: + """Per-attempt plan produced by `plan_attempt`. + + Replaces the old loose tuple-of-six (`ProviderSelection`) with a single + dataclass. Carries everything the executor / tool loop needs to make one + backend call without re-resolving configuration mid-call. + """ + + provider: ModelTransport + model: str + client: ProviderClient + thinking_budget_tokens: int | None + reasoning_effort: ReasoningEffortType + selected_config: ModelConfig + + +def resolve_runtime_model_config( + model_config: ModelConfig | ConfiguredModelSettings, +) -> ModelConfig: + """Return a runtime ModelConfig, resolving settings-shape inputs if needed.""" + if isinstance(model_config, ModelConfig): + return model_config + return resolve_model_config(model_config) + + +def select_model_config_for_attempt( + model_config: ModelConfig, + *, + attempt: int, + retry_attempts: int, +) -> ModelConfig: + """Pick the effective config for this attempt. + + Primary config on all attempts except the last, which swaps to the + resolved fallback (if any). + """ + if attempt != retry_attempts or model_config.fallback is None: + return model_config + + fb = model_config.fallback + return ModelConfig( + model=fb.model, + transport=fb.transport, + fallback=None, + api_key=fb.api_key, + base_url=fb.base_url, + temperature=fb.temperature, + top_p=fb.top_p, + top_k=fb.top_k, + frequency_penalty=fb.frequency_penalty, + presence_penalty=fb.presence_penalty, + seed=fb.seed, + thinking_effort=fb.thinking_effort, + thinking_budget_tokens=fb.thinking_budget_tokens, + provider_params=fb.provider_params, + max_output_tokens=fb.max_output_tokens, + stop_sequences=fb.stop_sequences, + cache_policy=fb.cache_policy, + ) + + +def plan_attempt( + *, + runtime_model_config: ModelConfig, + attempt: int, + retry_attempts: int, + call_thinking_budget_tokens: int | None, + call_reasoning_effort: ReasoningEffortType, +) -> AttemptPlan: + """Build the AttemptPlan for `attempt`. + + Reasoning params are drawn from the caller when we're still on the + primary config, and from the fallback config otherwise, so cross-transport + fallbacks use provider-appropriate params. + """ + selected = select_model_config_for_attempt( + runtime_model_config, + attempt=attempt, + retry_attempts=retry_attempts, + ) + provider = selected.transport + client = client_for_model_config(provider, selected) + + is_primary = selected is runtime_model_config + attempt_thinking_budget = ( + call_thinking_budget_tokens if is_primary else selected.thinking_budget_tokens + ) + attempt_reasoning_effort: ReasoningEffortType = ( + call_reasoning_effort if is_primary else selected.thinking_effort + ) + + if attempt == retry_attempts and runtime_model_config.fallback is not None: + logger.warning( + f"Final retry attempt {attempt}/{retry_attempts}: switching from " + + f"{runtime_model_config.transport}/{runtime_model_config.model} to " + + f"backup {provider}/{selected.model}" + ) + + return AttemptPlan( + provider=provider, + model=selected.model, + client=client, + thinking_budget_tokens=attempt_thinking_budget, + reasoning_effort=attempt_reasoning_effort, + selected_config=selected, + ) + + +def effective_config_for_call( + *, + selected_config: ModelConfig | None, + provider: ModelTransport, + model: str, + temperature: float | None, + stop_seqs: list[str] | None, + thinking_budget_tokens: int | None, + reasoning_effort: ReasoningEffortType, +) -> ModelConfig: + """Build the ModelConfig passed to the executor / request_builder. + + Per-call kwargs (temperature, stop_seqs, thinking_*) win when set; otherwise + the selected_config's values are used. When selected_config is None + (test-only callers passing provider+model directly) a minimal ModelConfig + is synthesized. + + max_output_tokens is forced to None so the per-call max_tokens kwarg is + authoritative — matching historical honcho_llm_call_inner behavior. + """ + if selected_config is None: + return ModelConfig( + model=model, + transport=provider, + temperature=temperature, + stop_sequences=stop_seqs, + thinking_budget_tokens=thinking_budget_tokens, + thinking_effort=reasoning_effort, + ) + updates: dict[str, Any] = {"max_output_tokens": None} + if temperature is not None: + updates["temperature"] = temperature + if stop_seqs is not None: + updates["stop_sequences"] = stop_seqs + if thinking_budget_tokens is not None: + updates["thinking_budget_tokens"] = thinking_budget_tokens + if reasoning_effort is not None: + updates["thinking_effort"] = reasoning_effort + return selected_config.model_copy(update=updates) + + +def effective_temperature(temperature: float | None) -> float | None: + """Bump temperature from 0.0 → 0.2 on retry attempts for variety.""" + if temperature == 0.0 and current_attempt.get() > 1: + logger.debug("Bumping temperature from 0.0 to 0.2 on retry") + return 0.2 + return temperature + + +def resolve_backend_for_plan(plan: AttemptPlan) -> Any: + """Convenience helper: plan → ready-to-call ProviderBackend.""" + return backend_for_provider(plan.provider, plan.client) + + +__all__ = [ + "AttemptPlan", + "current_attempt", + "effective_config_for_call", + "effective_temperature", + "plan_attempt", + "resolve_backend_for_plan", + "resolve_runtime_model_config", + "select_model_config_for_attempt", +] diff --git a/src/llm/structured_output.py b/src/llm/structured_output.py new file mode 100644 index 00000000..76c0690a --- /dev/null +++ b/src/llm/structured_output.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable +from typing import Literal + +from pydantic import BaseModel, ValidationError + +from src.utils.json_parser import validate_and_repair_json +from src.utils.representation import PromptRepresentation + +from .backend import CompletionResult + +StructuredOutputFailurePolicy = Literal[ + "raise", + "repair_then_raise", + "repair_then_empty", +] + + +class StructuredOutputError(ValueError): + """Raised when structured output cannot be validated or repaired.""" + + +def repair_response_model_json( + raw_content: str, + response_model: type[BaseModel], + _model: str, +) -> BaseModel: + """Repair truncated or malformed JSON and validate against the response model.""" + + try: + final = validate_and_repair_json(raw_content) + repaired_data = json.loads(final) + + if ( + response_model is PromptRepresentation + and "deductive" in repaired_data + and isinstance(repaired_data["deductive"], list) + ): + for item in repaired_data["deductive"]: + if isinstance(item, dict): + if "conclusion" not in item and "premises" in item: + if item["premises"]: + item["conclusion"] = ( + f"[Incomplete reasoning from premises: {item['premises'][0][:100]}...]" + ) + else: + item["conclusion"] = ( + "[Incomplete reasoning - conclusion missing]" + ) + if "premises" not in item: + item["premises"] = [] + + final = json.dumps(repaired_data) + except (json.JSONDecodeError, KeyError, TypeError, ValueError): + final = "" + + try: + return response_model.model_validate_json(final) + except ValidationError: + if response_model is PromptRepresentation: + return PromptRepresentation(explicit=[]) + raise + + +def validate_structured_output( + content: object, + response_model: type[BaseModel], +) -> BaseModel: + if isinstance(content, response_model): + return content + if isinstance(content, str): + return response_model.model_validate_json(content) + if isinstance(content, dict): + return response_model.model_validate(content) + raise StructuredOutputError( + f"Unsupported structured output payload: {type(content).__name__}" + ) + + +def attempt_structured_output_repair( + content: object, + response_model: type[BaseModel], + model: str, +) -> BaseModel | None: + if not isinstance(content, str): + return None + try: + return repair_response_model_json(content, response_model, model) + except (StructuredOutputError, ValidationError): + return None + + +def empty_structured_output(response_model: type[BaseModel]) -> BaseModel: + if response_model is PromptRepresentation: + return PromptRepresentation(explicit=[]) + return response_model.model_validate({}) + + +async def execute_structured_output_call( + executor: Callable[[], Awaitable[CompletionResult]], + *, + response_model: type[BaseModel], + model_name: str, + failure_policy: StructuredOutputFailurePolicy = "repair_then_raise", +) -> CompletionResult: + result = await executor() + + try: + result.content = validate_structured_output(result.content, response_model) + return result + except (StructuredOutputError, ValidationError): + if failure_policy == "raise": + raise + + repaired = attempt_structured_output_repair( + result.content, + response_model, + model_name, + ) + if repaired is not None: + result.content = repaired + return result + + if failure_policy == "repair_then_empty": + result.content = empty_structured_output(response_model) + return result + + raise StructuredOutputError( + f"Failed to produce valid structured output for {model_name}" + ) diff --git a/src/llm/tool_loop.py b/src/llm/tool_loop.py new file mode 100644 index 00000000..2db87e9a --- /dev/null +++ b/src/llm/tool_loop.py @@ -0,0 +1,491 @@ +"""Agentic/tool orchestration — the multi-iteration tool execution loop. + +`execute_tool_loop` owns: +- initial tool-enabled call +- tool execution +- conversation augmentation with assistant messages + tool results +- max-iteration handling and synthesis call +- stream-final-only mode +- empty-response retry (one retry nudge when the model returns empty content) +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator, Callable +from typing import Any + +from pydantic import BaseModel +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.config import ModelTransport +from src.exceptions import ValidationException +from src.utils.types import set_current_iteration + +from .executor import honcho_llm_call_inner +from .registry import history_adapter_for_provider +from .runtime import ( + AttemptPlan, + current_attempt, + effective_temperature, +) +from .types import ( + HonchoLLMCallResponse, + HonchoLLMCallStreamChunk, + IterationCallback, + IterationData, + StreamingResponseWithMetadata, + VerbosityType, +) + +logger = logging.getLogger(__name__) + +# Bounds for max_tool_iterations to prevent runaway loops. +MIN_TOOL_ITERATIONS = 1 +MAX_TOOL_ITERATIONS = 100 + + +def format_assistant_tool_message( + provider: ModelTransport, + content: Any, + tool_calls: list[dict[str, Any]], + thinking_blocks: list[dict[str, Any]] | None = None, + reasoning_details: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Format an assistant message with tool calls in provider-native shape.""" + from .backend import CompletionResult as BackendCompletionResult + from .backend import ToolCallResult + + adapter = history_adapter_for_provider(provider) + result = BackendCompletionResult( + content=content, + tool_calls=[ + ToolCallResult( + id=tool_call["id"], + name=tool_call["name"], + input=tool_call["input"], + thought_signature=tool_call.get("thought_signature"), + ) + for tool_call in tool_calls + ], + thinking_blocks=thinking_blocks or [], + reasoning_details=reasoning_details or [], + ) + return adapter.format_assistant_tool_message(result) + + +def append_tool_results( + provider: ModelTransport, + tool_results: list[dict[str, Any]], + conversation_messages: list[dict[str, Any]], +) -> None: + """Append tool results to `conversation_messages` in provider-native shape.""" + adapter = history_adapter_for_provider(provider) + conversation_messages.extend(adapter.format_tool_results(tool_results)) + + +async def stream_final_response( + *, + winning_plan: AttemptPlan, + prompt: str, + max_tokens: int, + conversation_messages: list[dict[str, Any]], + response_model: type[BaseModel] | None, + json_mode: bool, + temperature: float | None, + stop_seqs: list[str] | None, + verbosity: VerbosityType, + enable_retry: bool, + retry_attempts: int, + before_retry_callback: Callable[[Any], None], +) -> AsyncIterator[HonchoLLMCallStreamChunk]: + """Stream the final response after tool execution is complete. + + Uses the AttemptPlan captured at the moment streaming began (typically + the plan whose inner LLM call just succeeded) and pins it across any + retries of the stream setup. Re-running provider selection here would + bleed the outer current_attempt ContextVar into streaming retries, + potentially rolling the selection back to primary after the tool loop + had already settled on fallback. Tenacity retries re-issue the same + streaming call against the same pinned model for transient errors. + """ + + async def _setup_stream() -> AsyncIterator[HonchoLLMCallStreamChunk]: + return await honcho_llm_call_inner( + winning_plan.provider, + winning_plan.model, + prompt, + max_tokens, + response_model, + json_mode, + effective_temperature(temperature), + stop_seqs, + winning_plan.reasoning_effort, + verbosity, + winning_plan.thinking_budget_tokens, + stream=True, + client_override=winning_plan.client, + tools=None, + tool_choice=None, + messages=conversation_messages, + selected_config=winning_plan.selected_config, + ) + + if enable_retry: + wrapped = retry( + stop=stop_after_attempt(retry_attempts), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=before_retry_callback, + )(_setup_stream) + stream = await wrapped() + else: + stream = await _setup_stream() + + async for chunk in stream: + yield chunk + + +async def execute_tool_loop( + *, + prompt: str, + max_tokens: int, + messages: list[dict[str, Any]] | None, + tools: list[dict[str, Any]], + tool_choice: str | dict[str, Any] | None, + tool_executor: Callable[[str, dict[str, Any]], Any], + max_tool_iterations: int, + response_model: type[BaseModel] | None, + json_mode: bool, + temperature: float | None, + stop_seqs: list[str] | None, + verbosity: VerbosityType, + enable_retry: bool, + retry_attempts: int, + max_input_tokens: int | None, + get_attempt_plan: Callable[[], AttemptPlan], + before_retry_callback: Callable[[Any], None], + stream_final: bool = False, + iteration_callback: IterationCallback | None = None, +) -> HonchoLLMCallResponse[Any] | StreamingResponseWithMetadata: + """Run the iterative tool calling loop for agentic LLM interactions. + + Loop per iteration: + 1. Make an LLM call with tools available + 2. Execute any tool calls the LLM requests + 3. Append tool results to the conversation + 4. Repeat until the LLM stops calling tools or max iterations reached + + Returns: + Final HonchoLLMCallResponse with accumulated token counts and tool call + history, or a StreamingResponseWithMetadata if stream_final=True. + """ + from .conversation import truncate_messages_to_fit + + if not MIN_TOOL_ITERATIONS <= max_tool_iterations <= MAX_TOOL_ITERATIONS: + raise ValidationException( + "max_tool_iterations must be in " + + f"[{MIN_TOOL_ITERATIONS}, {MAX_TOOL_ITERATIONS}]; " + + f"got {max_tool_iterations}" + ) + + conversation_messages: list[dict[str, Any]] = ( + messages.copy() if messages else [{"role": "user", "content": prompt}] + ) + + iteration = 0 + all_tool_calls: list[dict[str, Any]] = [] + total_input_tokens = 0 + total_output_tokens = 0 + total_cache_creation_tokens = 0 + total_cache_read_tokens = 0 + empty_response_retries = 0 + # Track effective tool_choice — switches from "required"/"any" to "auto" after iter 1. + effective_tool_choice = tool_choice + + while iteration < max_tool_iterations: + # Reset attempt counter so each iteration starts with the primary provider. + current_attempt.set(1) + logger.debug(f"Tool execution iteration {iteration + 1}/{max_tool_iterations}") + + if max_input_tokens is not None: + conversation_messages = truncate_messages_to_fit( + conversation_messages, max_input_tokens + ) + + async def _call_with_messages( + effective_tool_choice: str | dict[str, Any] | None = effective_tool_choice, + conversation_messages: list[dict[str, Any]] = conversation_messages, + ) -> HonchoLLMCallResponse[Any]: + plan = get_attempt_plan() + return await honcho_llm_call_inner( + plan.provider, + plan.model, + prompt, # ignored when messages is passed + max_tokens, + response_model, + json_mode, + effective_temperature(temperature), + stop_seqs, + plan.reasoning_effort, + verbosity, + plan.thinking_budget_tokens, + stream=False, + client_override=plan.client, + tools=tools, + tool_choice=effective_tool_choice, + messages=conversation_messages, + selected_config=plan.selected_config, + ) + + if enable_retry: + call_func = retry( + stop=stop_after_attempt(retry_attempts), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=before_retry_callback, + )(_call_with_messages) + else: + call_func = _call_with_messages + + response = await call_func() + + total_input_tokens += response.input_tokens + total_output_tokens += response.output_tokens + total_cache_creation_tokens += response.cache_creation_input_tokens + total_cache_read_tokens += response.cache_read_input_tokens + + if not response.tool_calls_made: + logger.debug("No tool calls in response, finishing") + + if ( + isinstance(response.content, str) + and not response.content.strip() + and empty_response_retries < 1 + and iteration < max_tool_iterations - 1 + ): + empty_response_retries += 1 + conversation_messages.append( + { + "role": "user", + "content": ( + "Your last response was empty. Provide a concise answer " + "to the original query using the available context." + ), + } + ) + iteration += 1 + continue + + if stream_final: + # Snapshot the plan that just succeeded — streaming retries + # pin to this exact client/model so we don't bounce back to + # primary after the tool loop settled on fallback. + winning_plan = get_attempt_plan() + stream = stream_final_response( + winning_plan=winning_plan, + prompt=prompt, + max_tokens=max_tokens, + conversation_messages=conversation_messages, + response_model=response_model, + json_mode=json_mode, + temperature=temperature, + stop_seqs=stop_seqs, + verbosity=verbosity, + enable_retry=enable_retry, + retry_attempts=retry_attempts, + before_retry_callback=before_retry_callback, + ) + return StreamingResponseWithMetadata( + stream=stream, + tool_calls_made=all_tool_calls, + input_tokens=total_input_tokens, + output_tokens=total_output_tokens, + cache_creation_input_tokens=total_cache_creation_tokens, + cache_read_input_tokens=total_cache_read_tokens, + thinking_content=response.thinking_content, + iterations=iteration + 1, + ) + + response.tool_calls_made = all_tool_calls + response.input_tokens = total_input_tokens + response.output_tokens = total_output_tokens + response.cache_creation_input_tokens = total_cache_creation_tokens + response.cache_read_input_tokens = total_cache_read_tokens + response.iterations = iteration + 1 + return response + + current_provider = get_attempt_plan().provider + + assistant_message = format_assistant_tool_message( + current_provider, + response.content, + response.tool_calls_made, + response.thinking_blocks, + response.reasoning_details, + ) + conversation_messages.append(assistant_message) + + # Telemetry context — 1-indexed iteration. + set_current_iteration(iteration + 1) + + tool_results: list[dict[str, Any]] = [] + for tool_call in response.tool_calls_made: + tool_name = tool_call["name"] + tool_input = tool_call["input"] + tool_id = tool_call.get("id", "") + + logger.debug(f"Executing tool: {tool_name}") + + try: + tool_result = await tool_executor(tool_name, tool_input) + tool_results.append( + { + "tool_id": tool_id, + "tool_name": tool_name, + "result": tool_result, + } + ) + all_tool_calls.append( + { + "tool_name": tool_name, + "tool_input": tool_input, + "tool_result": tool_result, + } + ) + except Exception as e: + logger.error(f"Tool execution failed for {tool_name}: {e}") + tool_results.append( + { + "tool_id": tool_id, + "tool_name": tool_name, + "result": f"Error: {str(e)}", + "is_error": True, + } + ) + + append_tool_results(current_provider, tool_results, conversation_messages) + + if iteration_callback is not None: + try: + iteration_data = IterationData( + iteration=iteration + 1, + tool_calls=[tc["name"] for tc in response.tool_calls_made], + input_tokens=response.input_tokens, + output_tokens=response.output_tokens, + cache_read_tokens=response.cache_read_input_tokens or 0, + cache_creation_tokens=response.cache_creation_input_tokens or 0, + ) + iteration_callback(iteration_data) + except Exception: + logger.warning("iteration_callback failed", exc_info=True) + + # After first iteration, switch "required"/"any" → "auto" so the model can stop. + if iteration == 0 and effective_tool_choice in ("required", "any"): + effective_tool_choice = "auto" + logger.debug( + "Switched tool_choice from 'required'/'any' to 'auto' after first iteration" + ) + + iteration += 1 + + logger.warning( + f"Tool execution loop reached max iterations ({max_tool_iterations})" + ) + + synthesis_prompt = ( + "You have reached the maximum number of tool calls. " + "Based on all the information you have gathered, provide your final response now. " + "Do not attempt to call any more tools." + ) + conversation_messages.append({"role": "user", "content": synthesis_prompt}) + + # Truncate again — the per-iteration truncate ran before the last tool + # call, so appending synthesis_prompt could nudge us back over the cap. + if max_input_tokens is not None: + conversation_messages = truncate_messages_to_fit( + conversation_messages, max_input_tokens + ) + + if stream_final: + # Snapshot the plan the loop settled on — streaming retries pin to + # this exact client/model rather than re-running provider selection. + winning_plan = get_attempt_plan() + stream = stream_final_response( + winning_plan=winning_plan, + prompt=prompt, + max_tokens=max_tokens, + conversation_messages=conversation_messages, + response_model=response_model, + json_mode=json_mode, + temperature=temperature, + stop_seqs=stop_seqs, + verbosity=verbosity, + enable_retry=enable_retry, + retry_attempts=retry_attempts, + before_retry_callback=before_retry_callback, + ) + return StreamingResponseWithMetadata( + stream=stream, + tool_calls_made=all_tool_calls, + input_tokens=total_input_tokens, + output_tokens=total_output_tokens, + cache_creation_input_tokens=total_cache_creation_tokens, + cache_read_input_tokens=total_cache_read_tokens, + thinking_content=None, + iterations=iteration + 1, + ) + + current_attempt.set(1) + + async def _final_call() -> HonchoLLMCallResponse[Any]: + plan = get_attempt_plan() + return await honcho_llm_call_inner( + plan.provider, + plan.model, + prompt, + max_tokens, + response_model, + json_mode, + effective_temperature(temperature), + stop_seqs, + plan.reasoning_effort, + verbosity, + plan.thinking_budget_tokens, + stream=False, + client_override=plan.client, + tools=None, + tool_choice=None, + messages=conversation_messages, + selected_config=plan.selected_config, + ) + + if enable_retry: + final_call_func = retry( + stop=stop_after_attempt(retry_attempts), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=before_retry_callback, + )(_final_call) + else: + final_call_func = _final_call + + final_response = await final_call_func() + final_response.tool_calls_made = all_tool_calls + final_response.iterations = iteration + 1 + final_response.input_tokens = total_input_tokens + final_response.input_tokens + final_response.output_tokens = total_output_tokens + final_response.output_tokens + final_response.cache_creation_input_tokens = ( + total_cache_creation_tokens + final_response.cache_creation_input_tokens + ) + final_response.cache_read_input_tokens = ( + total_cache_read_tokens + final_response.cache_read_input_tokens + ) + return final_response + + +__all__ = [ + "MAX_TOOL_ITERATIONS", + "MIN_TOOL_ITERATIONS", + "append_tool_results", + "execute_tool_loop", + "format_assistant_tool_message", + "stream_final_response", +] diff --git a/src/llm/types.py b/src/llm/types.py new file mode 100644 index 00000000..7af5372d --- /dev/null +++ b/src/llm/types.py @@ -0,0 +1,138 @@ +"""Public response/stream/iteration types for the LLM API. + +These used to live in src/utils/clients.py and have been moved here as part +of the migration toward src/llm/ owning all non-embedding LLM orchestration. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Callable +from dataclasses import dataclass +from typing import Any, Generic, Literal, TypeVar + +from anthropic import AsyncAnthropic +from google import genai +from openai import AsyncOpenAI +from pydantic import BaseModel, Field + +T = TypeVar("T") + +# OpenAI GPT-5 specific reasoning levels. +ReasoningEffortType = ( + Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] | None +) +VerbosityType = Literal["low", "medium", "high"] | None + +# Raw SDK client union used by the provider-selection layer. +ProviderClient = AsyncAnthropic | AsyncOpenAI | genai.Client + + +@dataclass +class IterationData: + """Data passed to iteration callbacks after each tool execution loop iteration.""" + + iteration: int + """1-indexed iteration number.""" + tool_calls: list[str] + """List of tool names called in this iteration.""" + input_tokens: int + """Input tokens used in this iteration's LLM call.""" + output_tokens: int + """Output tokens generated in this iteration's LLM call.""" + cache_read_tokens: int = 0 + """Tokens read from cache in this iteration.""" + cache_creation_tokens: int = 0 + """Tokens written to cache in this iteration.""" + + +IterationCallback = Callable[[IterationData], None] + + +class HonchoLLMCallResponse(BaseModel, Generic[T]): + """Response object for LLM calls. + + Note: + Uncached input tokens = input_tokens - cache_read_input_tokens + + cache_creation_input_tokens + (cache_creation costs 25% more, cache_read costs 90% less) + """ + + content: T + input_tokens: int = 0 + output_tokens: int + cache_creation_input_tokens: int = 0 + cache_read_input_tokens: int = 0 + finish_reasons: list[str] + tool_calls_made: list[dict[str, Any]] = Field(default_factory=list) + iterations: int = 0 + """Number of LLM calls made in the tool execution loop.""" + thinking_content: str | None = None + # Full thinking blocks with signatures for multi-turn replay (Anthropic only). + thinking_blocks: list[dict[str, Any]] = Field(default_factory=list) + # OpenRouter reasoning_details for Gemini models — must be preserved across turns. + reasoning_details: list[dict[str, Any]] = Field(default_factory=list) + + +class HonchoLLMCallStreamChunk(BaseModel): + """A single chunk in a streaming LLM response.""" + + content: str + is_done: bool = False + finish_reasons: list[str] = Field(default_factory=list) + output_tokens: int | None = None + + +class StreamingResponseWithMetadata: + """Streaming response wrapper carrying metadata from a completed tool loop. + + Lets callers read tool_calls_made / token counts / thinking_content from + the tool-execution phase while still iterating the final streamed answer. + """ + + _stream: AsyncIterator[HonchoLLMCallStreamChunk] + tool_calls_made: list[dict[str, Any]] + input_tokens: int + output_tokens: int + cache_creation_input_tokens: int + cache_read_input_tokens: int + thinking_content: str | None + iterations: int + + def __init__( + self, + stream: AsyncIterator[HonchoLLMCallStreamChunk], + tool_calls_made: list[dict[str, Any]], + input_tokens: int, + output_tokens: int, + cache_creation_input_tokens: int, + cache_read_input_tokens: int, + thinking_content: str | None = None, + iterations: int = 0, + ): + self._stream = stream + self.tool_calls_made = tool_calls_made + self.input_tokens = input_tokens + self.output_tokens = output_tokens + self.cache_creation_input_tokens = cache_creation_input_tokens + self.cache_read_input_tokens = cache_read_input_tokens + self.thinking_content = thinking_content + self.iterations = iterations + + def __aiter__(self) -> AsyncIterator[HonchoLLMCallStreamChunk]: + return self._stream.__aiter__() + + async def __anext__(self) -> HonchoLLMCallStreamChunk: + return await self._stream.__anext__() + + +__all__ = [ + "HonchoLLMCallResponse", + "HonchoLLMCallStreamChunk", + "IterationCallback", + "IterationData", + "ProviderClient", + "ReasoningEffortType", + "StreamingResponseWithMetadata", + "T", + "VerbosityType", +] diff --git a/src/schemas/api.py b/src/schemas/api.py index 307b12e0..8be19492 100644 --- a/src/schemas/api.py +++ b/src/schemas/api.py @@ -501,9 +501,10 @@ class ConclusionCreate(BaseModel): tokens = encoding.encode(self.content) self._token_count = len(tokens) - if self._token_count > settings.MAX_EMBEDDING_TOKENS: + if self._token_count > settings.EMBEDDING.MAX_INPUT_TOKENS: raise ValueError( - f"Content exceeds maximum embedding token limit of {settings.MAX_EMBEDDING_TOKENS} " + "Content exceeds maximum embedding token limit of " + + f"{settings.EMBEDDING.MAX_INPUT_TOKENS} " + f"(got {self._token_count} tokens)" ) return self diff --git a/src/telemetry/reasoning_traces.py b/src/telemetry/reasoning_traces.py index b208d08b..1f4f03e3 100644 --- a/src/telemetry/reasoning_traces.py +++ b/src/telemetry/reasoning_traces.py @@ -12,7 +12,11 @@ from typing import Any from pydantic import BaseModel -from src.config import LLMComponentSettings, settings +from src.config import ( + ConfiguredModelSettings, + ModelConfig, + settings, +) def get_reasoning_traces_file_path() -> Path | None: @@ -24,7 +28,7 @@ def get_reasoning_traces_file_path() -> Path | None: def log_reasoning_trace( task_type: str, - llm_settings: LLMComponentSettings, + model_config: ModelConfig | ConfiguredModelSettings, prompt: str, response: Any, *, @@ -40,7 +44,7 @@ def log_reasoning_trace( Args: task_type: Type of task (e.g., "minimal_deriver", "dialectic_chat") - llm_settings: LLM settings used for the call + model_config: Model configuration used for the call prompt: The full prompt text sent to the LLM (used if messages is None) response: HonchoLLMCallResponse object with the LLM response max_tokens: Max output tokens setting @@ -62,8 +66,8 @@ def log_reasoning_trace( trace_entry: dict[str, Any] = { "timestamp": time.time(), "task_type": task_type, - "provider": llm_settings.PROVIDER, - "model": llm_settings.MODEL, + "provider": model_config.transport, + "model": model_config.model, "settings": { "max_tokens": max_tokens, "thinking_budget_tokens": thinking_budget_tokens, diff --git a/src/utils/agent_tools.py b/src/utils/agent_tools.py index 21132397..36168f34 100644 --- a/src/utils/agent_tools.py +++ b/src/utils/agent_tools.py @@ -33,6 +33,205 @@ logger = logging.getLogger(__name__) MAX_PEER_CARD_FACTS = 40 +def _base_observation_properties() -> dict[str, Any]: + return { + "content": { + "type": "string", + "description": "The observation content", + }, + "level": { + "type": "string", + "enum": [ + "explicit", + "deductive", + "inductive", + "contradiction", + ], + "description": ( + "Level: 'explicit' for direct facts, 'deductive' for logical " + + "necessities, 'inductive' for patterns, 'contradiction' for " + + "conflicting statements" + ), + }, + "source_ids": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Document IDs of source or premise observations. Required and " + + "must be non-empty for deductive, inductive, and contradiction " + + "observations." + ), + }, + "premises": { + "type": "array", + "items": {"type": "string"}, + "description": "(For deductive) Human-readable premise text for display", + }, + "sources": { + "type": "array", + "items": {"type": "string"}, + "description": "(For inductive/contradiction) Human-readable source text for display", + }, + "pattern_type": { + "type": "string", + "enum": [ + "preference", + "behavior", + "personality", + "tendency", + "correlation", + ], + "description": "(For inductive only) Type of pattern being identified", + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"], + "description": ( + "(For inductive only) Confidence level: 'high' for 5+ sources, " + + "'medium' for 3-4, 'low' for 2" + ), + }, + } + + +def _generic_observation_item_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": _base_observation_properties(), + "required": ["content", "level"], + "additionalProperties": False, + "allOf": [ + { + "if": {"properties": {"level": {"const": "deductive"}}}, + "then": { + "required": ["source_ids", "premises"], + "properties": { + "source_ids": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + }, + "premises": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + }, + }, + }, + }, + { + "if": {"properties": {"level": {"const": "inductive"}}}, + "then": { + "required": [ + "source_ids", + "sources", + "pattern_type", + "confidence", + ], + "properties": { + "source_ids": { + "type": "array", + "items": {"type": "string"}, + "minItems": 2, + }, + "sources": { + "type": "array", + "items": {"type": "string"}, + "minItems": 2, + }, + }, + }, + }, + { + "if": {"properties": {"level": {"const": "contradiction"}}}, + "then": { + "required": ["source_ids", "sources"], + "properties": { + "source_ids": { + "type": "array", + "items": {"type": "string"}, + "minItems": 2, + }, + "sources": { + "type": "array", + "items": {"type": "string"}, + "minItems": 2, + }, + }, + }, + }, + ], + } + + +def _deductive_observation_item_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "The deductive conclusion as a self-contained statement", + }, + "source_ids": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "description": "Required non-empty list of source observation IDs supporting the deduction", + }, + "premises": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "description": "Required human-readable premise text matching the source observations", + }, + }, + "required": ["content", "source_ids", "premises"], + "additionalProperties": False, + } + + +def _inductive_observation_item_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "The inductive pattern or generalization as a self-contained statement", + }, + "source_ids": { + "type": "array", + "items": {"type": "string"}, + "minItems": 2, + "description": "Required list of at least two source observation IDs supporting the pattern", + }, + "sources": { + "type": "array", + "items": {"type": "string"}, + "minItems": 2, + "description": "Required human-readable evidence text matching the source observations", + }, + "pattern_type": { + "type": "string", + "enum": [ + "preference", + "behavior", + "personality", + "tendency", + "correlation", + ], + "description": "Required pattern category", + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"], + "description": "Required confidence level based on evidence count", + }, + }, + "required": ["content", "source_ids", "sources", "pattern_type", "confidence"], + "additionalProperties": False, + } + + def _safe_int(value: Any, default: int) -> int: """Coerce a tool input value to int, returning default on failure. @@ -177,88 +376,44 @@ def _extract_pattern_snippet( TOOLS: dict[str, dict[str, Any]] = { "create_observations": { "name": "create_observations", - "description": "Create observations at any level: explicit (facts), deductive (logical necessities), inductive (patterns), or contradiction (conflicting statements). Use this to record facts, logical inferences, patterns, or note when the user has said contradictory things.", + "description": "Create observations at any level: explicit (facts), deductive (logical necessities), inductive (patterns), or contradiction (conflicting statements). For deductive, inductive, and contradiction observations, missing or empty source_ids are invalid and will be rejected.", "input_schema": { "type": "object", "properties": { "observations": { "type": "array", "description": "List of observations to create", - "items": { - "type": "object", - "properties": { - "content": { - "type": "string", - "description": "The observation content", - }, - "level": { - "type": "string", - "enum": [ - "explicit", - "deductive", - "inductive", - "contradiction", - ], - "description": "Level: 'explicit' for direct facts, 'deductive' for logical necessities, 'inductive' for patterns, 'contradiction' for conflicting statements", - }, - "source_ids": { - "type": "array", - "items": {"type": "string"}, - "description": "(For deductive/inductive/contradiction) Document IDs of source/premise observations - REQUIRED", - }, - "premises": { - "type": "array", - "items": {"type": "string"}, - "description": "(For deductive) Human-readable premise text for display", - }, - "sources": { - "type": "array", - "items": {"type": "string"}, - "description": "(For inductive/contradiction) Human-readable source text for display", - }, - "pattern_type": { - "type": "string", - "enum": [ - "preference", - "behavior", - "personality", - "tendency", - "correlation", - ], - "description": "(For inductive only) Type of pattern being identified", - }, - "confidence": { - "type": "string", - "enum": ["high", "medium", "low"], - "description": "(For inductive only) Confidence level: 'high' for 3+ sources, 'medium' for 2+, 'low' for tentative", - }, - }, - "required": ["content", "level"], - }, + "items": _generic_observation_item_schema(), }, }, "required": ["observations"], }, }, "create_observations_deductive": { - "name": "create_observations", - "description": "Create new deductive observations discovered while answering the query. Use this when you infer something new about the peer that isn't already captured in existing observations. Only use for novel deductions - not for restating existing facts.", + "name": "create_observations_deductive", + "description": "Create new deductive observations discovered while answering the query. Every observation must include non-empty source_ids and premise text. Use this only for novel deductions grounded in existing observations.", "input_schema": { "type": "object", "properties": { "observations": { "type": "array", "description": "List of new deductive observations to create", - "items": { - "type": "object", - "properties": { - "content": { - "type": "string", - "description": "The observation content - should be a self-contained statement about the peer", - }, - }, - "required": ["content"], - }, + "items": _deductive_observation_item_schema(), + }, + }, + "required": ["observations"], + }, + }, + "create_observations_inductive": { + "name": "create_observations_inductive", + "description": "Create new inductive observations discovered while answering the query. Every observation must include source_ids, source text, pattern_type, and confidence. Use this only for patterns supported by multiple observations.", + "input_schema": { + "type": "object", + "properties": { + "observations": { + "type": "array", + "description": "List of new inductive observations to create", + "items": _inductive_observation_item_schema(), }, }, "required": ["observations"], @@ -595,7 +750,7 @@ DEDUCTION_SPECIALIST_TOOLS: list[dict[str, Any]] = [ TOOLS["search_memory"], TOOLS["search_messages"], # Action tools - TOOLS["create_observations"], + TOOLS["create_observations_deductive"], TOOLS["delete_observations"], TOOLS["update_peer_card"], ] @@ -610,7 +765,7 @@ INDUCTION_SPECIALIST_TOOLS: list[dict[str, Any]] = [ TOOLS["search_memory"], TOOLS["search_messages"], # Action tools - TOOLS["create_observations"], + TOOLS["create_observations_inductive"], TOOLS["update_peer_card"], ] @@ -1033,8 +1188,11 @@ class ToolContext: parent_category: str | None = None # Parent category for CloudEvents -async def _handle_create_observations( - ctx: ToolContext, tool_input: dict[str, Any] +async def _handle_create_observations_impl( + ctx: ToolContext, + tool_input: dict[str, Any], + *, + forced_level: str | None = None, ) -> str: """Handle create_observations tool.""" raw_observations = tool_input.get("observations", []) @@ -1045,7 +1203,10 @@ async def _handle_create_observations( # Set context-specific default level before Pydantic validation default_level = "explicit" if ctx.current_messages else "deductive" for obs in raw_observations: - obs.setdefault("level", default_level) + if forced_level is not None: + obs["level"] = forced_level + else: + obs.setdefault("level", default_level) # Validate observations individually so valid ones are still processed observations: list[schemas.ObservationInput] = [] @@ -1139,6 +1300,32 @@ async def _handle_create_observations( return response +async def _handle_create_observations( + ctx: ToolContext, tool_input: dict[str, Any] +) -> str: + return await _handle_create_observations_impl(ctx, tool_input) + + +async def _handle_create_observations_deductive( + ctx: ToolContext, tool_input: dict[str, Any] +) -> str: + return await _handle_create_observations_impl( + ctx, + tool_input, + forced_level="deductive", + ) + + +async def _handle_create_observations_inductive( + ctx: ToolContext, tool_input: dict[str, Any] +) -> str: + return await _handle_create_observations_impl( + ctx, + tool_input, + forced_level="inductive", + ) + + async def _handle_update_peer_card(ctx: ToolContext, tool_input: dict[str, Any]) -> str: """Handle update_peer_card tool.""" # Check if peer card creation is disabled via configuration @@ -1263,7 +1450,10 @@ async def _handle_search_memory(ctx: ToolContext, tool_input: dict[str, Any]) -> try: query_embedding = await embedding_client.embed(query) except ValueError: - return f"ERROR: Query exceeds maximum token limit of {settings.MAX_EMBEDDING_TOKENS}. Please use a shorter query." + return ( + "ERROR: Query exceeds maximum token limit of " + + f"{settings.EMBEDDING.MAX_INPUT_TOKENS}. Please use a shorter query." + ) documents = await crud.query_documents( db=None, @@ -1814,6 +2004,8 @@ async def _handle_get_reasoning_chain( # Tool handler dispatch table _TOOL_HANDLERS: dict[str, Callable[[ToolContext, dict[str, Any]], Any]] = { "create_observations": _handle_create_observations, + "create_observations_deductive": _handle_create_observations_deductive, + "create_observations_inductive": _handle_create_observations_inductive, "update_peer_card": _handle_update_peer_card, "get_recent_history": _handle_get_recent_history, "search_memory": _handle_search_memory, diff --git a/src/utils/clients.py b/src/utils/clients.py deleted file mode 100644 index 1c042bff..00000000 --- a/src/utils/clients.py +++ /dev/null @@ -1,2575 +0,0 @@ -import json -import logging -from collections.abc import AsyncIterator, Callable -from contextvars import ContextVar -from dataclasses import dataclass -from typing import Any, Generic, Literal, TypeVar, cast, overload - -from anthropic import AsyncAnthropic -from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock -from anthropic.types.message import Message as AnthropicMessage -from anthropic.types.usage import Usage -from google import genai -from google.genai.types import ( - ContentListUnionDict, - GenerateContentConfigDict, - GenerateContentResponse, -) -from groq import AsyncGroq -from openai import AsyncOpenAI -from openai.types.chat import ChatCompletion, ChatCompletionChunk -from pydantic import BaseModel, Field, ValidationError -from sentry_sdk.ai.monitoring import ai_track -from tenacity import retry, stop_after_attempt, wait_exponential - -from src.config import LLMComponentSettings, settings -from src.exceptions import LLMError -from src.telemetry.logging import conditional_observe -from src.telemetry.reasoning_traces import log_reasoning_trace -from src.utils.json_parser import validate_and_repair_json -from src.utils.representation import PromptRepresentation -from src.utils.tokens import estimate_tokens -from src.utils.types import SupportedProviders, set_current_iteration - -logger = logging.getLogger(__name__) - -# Gemini finish reasons that indicate the response was blocked by safety or policy -# filters. When these occur, the response typically has no usable text content and -# retrying with a backup provider is appropriate. -GEMINI_BLOCKED_FINISH_REASONS = { - "SAFETY", - "RECITATION", - "PROHIBITED_CONTENT", - "BLOCKLIST", -} - - -@dataclass -class IterationData: - """Data passed to iteration callbacks after each tool execution loop iteration.""" - - iteration: int - """1-indexed iteration number.""" - tool_calls: list[str] - """List of tool names called in this iteration.""" - input_tokens: int - """Input tokens used in this iteration's LLM call.""" - output_tokens: int - """Output tokens generated in this iteration's LLM call.""" - cache_read_tokens: int = 0 - """Tokens read from cache in this iteration.""" - cache_creation_tokens: int = 0 - """Tokens written to cache in this iteration.""" - - -# Type alias for iteration callback -IterationCallback = Callable[[IterationData], None] - -T = TypeVar("T") - -# Type aliases for OpenAI GPT-5 specific parameters -ReasoningEffortType = Literal["low", "medium", "high", "minimal"] | None -VerbosityType = Literal["low", "medium", "high"] | None - - -def count_message_tokens(messages: list[dict[str, Any]]) -> int: - """Count tokens in a list of messages using tiktoken.""" - total = 0 - for msg in messages: - content = msg.get("content", "") - if isinstance(content, str): - total += estimate_tokens(content) - elif isinstance(content, list): - # Handle Anthropic-style content blocks - total += estimate_tokens(json.dumps(content)) - # Also count parts for Google format - if "parts" in msg: - try: - total += estimate_tokens(json.dumps(msg["parts"])) - except TypeError: - # Handle non-JSON-serializable content (e.g., bytes) by estimating based on string representation - total += estimate_tokens(str(msg["parts"])) - return total - - -def _is_tool_use_message(msg: dict[str, Any]) -> bool: - """Check if a message contains tool calls (any format).""" - # Anthropic format: content is a list with tool_use blocks - content = msg.get("content") - if isinstance(content, list): - for block in cast(list[dict[str, Any]], content): - if block.get("type") == "tool_use": - return True - - # OpenAI format: tool_calls field on assistant message - return bool(msg.get("tool_calls")) - - -def _is_tool_result_message(msg: dict[str, Any]) -> bool: - """Check if a message contains tool results (any format).""" - # Anthropic format: content is a list with tool_result blocks - content = msg.get("content") - if isinstance(content, list): - for block in cast(list[dict[str, Any]], content): - if block.get("type") == "tool_result": - return True - - # OpenAI format: role is "tool" - return msg.get("role") == "tool" - - -def _group_into_units(messages: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: - """ - Group messages into logical conversation units. - - A unit is either: - - A tool_use message + ALL consecutive tool_result messages that follow - - A single non-tool message - - This ensures tool_use and tool_results stay together. - """ - units: list[list[dict[str, Any]]] = [] - i = 0 - - while i < len(messages): - msg = messages[i] - - if _is_tool_use_message(msg): - # Collect this tool_use and ALL following tool_results - j = i + 1 - while j < len(messages) and _is_tool_result_message(messages[j]): - j += 1 - - # Create unit with tool_use + all tool_results - unit = messages[i:j] - if len(unit) > 1: # Has at least one tool_result - units.append(unit) - i = j - else: - # Orphaned tool_use (no results) - skip it - logger.debug(f"Skipping orphaned tool_use at index {i}") - i += 1 - elif _is_tool_result_message(msg): - # Orphaned tool_result - skip it - logger.debug(f"Skipping orphaned tool_result at index {i}") - i += 1 - else: - # Regular message - its own unit - units.append([msg]) - i += 1 - - return units - - -def truncate_messages_to_fit( - messages: list[dict[str, Any]], - max_tokens: int, - preserve_system: bool = True, -) -> list[dict[str, Any]]: - """ - Truncate messages to fit within a token limit while maintaining valid structure. - - Strategy: - 1. Group messages into units (tool_use + results together, or single messages) - 2. Remove oldest units first to preserve recent context - 3. Units stay intact so tool_use/tool_result pairs are never broken - """ - current_tokens = count_message_tokens(messages) - if current_tokens <= max_tokens: - return messages - - logger.info(f"Truncating: {current_tokens} tokens exceeds {max_tokens} limit") - - # Separate system messages from conversation - system_messages: list[dict[str, Any]] = [] - conversation: list[dict[str, Any]] = [] - - for msg in messages: - if msg.get("role") == "system" and preserve_system: - system_messages.append(msg) - else: - conversation.append(msg) - - system_tokens = count_message_tokens(system_messages) - available_tokens = max_tokens - system_tokens - - if available_tokens <= 0: - logger.warning("System message exceeds max_input_tokens") - return messages - - # Group messages into units - units = _group_into_units(conversation) - - if not units: - logger.warning("No valid conversation units") - return system_messages - - # Remove oldest units until we fit - while len(units) > 1: # Keep at least one unit - # Calculate current token count - flat_messages = [msg for unit in units for msg in unit] - if count_message_tokens(flat_messages) <= available_tokens: - break - - # Remove the oldest unit - removed_unit = units.pop(0) - logger.debug( - f"Removed unit with {len(removed_unit)} messages " - + f"(~{count_message_tokens(removed_unit)} tokens)" - ) - - # Flatten remaining units - result_conversation = [msg for unit in units for msg in unit] - - result = system_messages + result_conversation - result_tokens = count_message_tokens(result) - logger.info( - f"Truncation complete: {len(messages)} -> {len(result)} messages, " - + f"{current_tokens} -> {result_tokens} tokens, " - + f"{len(units)} units kept" - ) - return result - - -M = TypeVar("M", bound=BaseModel) - -# Context variable to track retry attempts for provider switching -_current_attempt: ContextVar[int] = ContextVar("current_attempt", default=0) - - -def _get_effective_temperature(temperature: float | None) -> float | None: - """Adjust temperature on retries - bump 0.0 to 0.2 to get different results.""" - if temperature == 0.0 and _current_attempt.get() > 1: - logger.debug("Bumping temperature from 0.0 to 0.2 on retry") - return 0.2 - return temperature - - -CLIENTS: dict[ - SupportedProviders, - AsyncAnthropic | AsyncOpenAI | genai.Client | AsyncGroq, -] = {} - -if settings.LLM.ANTHROPIC_API_KEY: - anthropic = AsyncAnthropic( - api_key=settings.LLM.ANTHROPIC_API_KEY, - timeout=600.0, # 10 minutes timeout for long-running operations - ) - CLIENTS["anthropic"] = anthropic - -if settings.LLM.OPENAI_API_KEY: - openai_client = AsyncOpenAI( - api_key=settings.LLM.OPENAI_API_KEY, - ) - CLIENTS["openai"] = openai_client - -if settings.LLM.OPENAI_COMPATIBLE_API_KEY and settings.LLM.OPENAI_COMPATIBLE_BASE_URL: - CLIENTS["custom"] = AsyncOpenAI( - api_key=settings.LLM.OPENAI_COMPATIBLE_API_KEY, - base_url=settings.LLM.OPENAI_COMPATIBLE_BASE_URL, - ) - -# vLLM uses separate settings for local model serving -if settings.LLM.VLLM_API_KEY and settings.LLM.VLLM_BASE_URL: - CLIENTS["vllm"] = AsyncOpenAI( - api_key=settings.LLM.VLLM_API_KEY, - base_url=settings.LLM.VLLM_BASE_URL, - ) - -if settings.LLM.GEMINI_API_KEY: - google = genai.client.Client(api_key=settings.LLM.GEMINI_API_KEY) - CLIENTS["google"] = google - -if settings.LLM.GROQ_API_KEY: - groq = AsyncGroq(api_key=settings.LLM.GROQ_API_KEY) - CLIENTS["groq"] = groq - -SELECTED_PROVIDERS = [ - ("Summary", settings.SUMMARY.PROVIDER), - ("Deriver", settings.DERIVER.PROVIDER), -] - -# Add all dialectic level providers -for level, level_settings in settings.DIALECTIC.LEVELS.items(): - SELECTED_PROVIDERS.append((f"Dialectic ({level})", level_settings.PROVIDER)) - -for provider_name, provider_value in SELECTED_PROVIDERS: - if provider_value not in CLIENTS: - raise ValueError(f"Missing client for {provider_name}: {provider_value}") - -# Validate backup providers are initialized if configured -BACKUP_PROVIDERS: list[tuple[str, SupportedProviders | None]] = [ - ("Deriver", settings.DERIVER.BACKUP_PROVIDER), - ("Summary", settings.SUMMARY.BACKUP_PROVIDER), - ("Dream", settings.DREAM.BACKUP_PROVIDER), -] - -# Add all dialectic level backup providers -for level, level_settings in settings.DIALECTIC.LEVELS.items(): - BACKUP_PROVIDERS.append((f"Dialectic ({level})", level_settings.BACKUP_PROVIDER)) - -for component_name, backup_provider in BACKUP_PROVIDERS: - if backup_provider is not None and backup_provider not in CLIENTS: - raise ValueError( - f"Backup provider for {component_name} is set to {backup_provider}, " - + "but this provider is not initialized. Please set the required API key/URL environment " - + "variables or remove the backup configuration." - ) - - -def convert_tools_for_provider( - tools: list[dict[str, Any]], - provider: SupportedProviders, -) -> list[dict[str, Any]]: - """ - Convert tool definitions to provider-specific format. - - Args: - tools: List of tool definitions in Anthropic format (with input_schema) - provider: The target provider to convert tools for - - Returns: - List of tool definitions in the provider's native format - """ - if provider == "anthropic": - # Anthropic format: input_schema - return tools - elif provider in ("openai", "custom", "vllm"): - # OpenAI format: parameters instead of input_schema - # custom and vllm use AsyncOpenAI client so need OpenAI format - return [ - { - "type": "function", - "function": { - "name": tool["name"], - "description": tool["description"], - "parameters": tool["input_schema"], - }, - } - for tool in tools - ] - elif provider == "google": - # Google format: function_declarations wrapped in a tool object - return [ - { - "function_declarations": [ - { - "name": tool["name"], - "description": tool["description"], - "parameters": tool["input_schema"], - } - for tool in tools - ] - } - ] - else: - # For unsupported providers, return as-is (will likely error if tools are used) - logger.warning( - f"Tool calling not implemented for provider {provider}, returning tools as-is" - ) - return tools - - -def extract_openai_reasoning_content(response: Any) -> str | None: - """ - Extract reasoning/thinking content from an OpenAI ChatCompletion response. - - GPT-5 and o1 models include reasoning_details in the response message. - Custom OpenAI-compatible providers may also include this field. - - Args: - response: OpenAI ChatCompletion response object - - Returns: - Concatenated reasoning content string, or None if not present - """ - try: - message = response.choices[0].message - # Check for reasoning_details (GPT-5/o1 models) - if hasattr(message, "reasoning_details") and message.reasoning_details: - # reasoning_details is a list of reasoning steps - reasoning_parts: list[Any] = [] - for detail in message.reasoning_details: - if hasattr(detail, "content") and detail.content: - reasoning_parts.append(detail.content) - elif isinstance(detail, dict) and detail.get("content"): # pyright: ignore[reportUnknownMemberType] - reasoning_parts.append(detail["content"]) - if reasoning_parts: - return "\n".join(reasoning_parts) - # Check for reasoning_content (some custom providers) - if hasattr(message, "reasoning_content") and message.reasoning_content: - return message.reasoning_content - except (AttributeError, IndexError, TypeError): - pass - return None - - -def extract_openai_reasoning_details(response: Any) -> list[dict[str, Any]]: - """ - Extract reasoning_details array from an OpenAI/OpenRouter ChatCompletion response. - - OpenRouter returns reasoning blocks in reasoning_details that must be preserved - and passed back in subsequent requests for Gemini models with tool use. - - Args: - response: OpenAI ChatCompletion response object - - Returns: - List of reasoning detail objects, or empty list if not present - """ - try: - message = response.choices[0].message - # Check for reasoning_details (OpenRouter/Gemini) - if hasattr(message, "reasoning_details") and message.reasoning_details: - # Return the full array for preservation - return [ - detail.model_dump() if hasattr(detail, "model_dump") else dict(detail) - for detail in message.reasoning_details - ] - except (AttributeError, IndexError, TypeError): - pass - return [] - - -def extract_openai_cache_tokens(usage: Any) -> tuple[int, int]: - """ - Extract cache token counts from OpenAI-style usage objects. - - OpenAI reports cached tokens in usage.prompt_tokens_details.cached_tokens. - OpenRouter and some proxies may report in different locations. - - Args: - usage: OpenAI CompletionUsage object or similar - - Returns: - Tuple of (cache_creation_tokens, cache_read_tokens). - For OpenAI-style APIs, cache_creation is always 0 (automatic caching), - and cache_read is the cached_tokens count. - """ - if not usage: - return 0, 0 - - cache_read = 0 - - # OpenAI native: usage.prompt_tokens_details.cached_tokens - if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: - details = usage.prompt_tokens_details - if hasattr(details, "cached_tokens") and details.cached_tokens: - cache_read = details.cached_tokens - - # OpenRouter style: usage.cache_read_input_tokens or usage.cached_tokens - if cache_read == 0: - if hasattr(usage, "cache_read_input_tokens") and usage.cache_read_input_tokens: - cache_read = usage.cache_read_input_tokens - elif hasattr(usage, "cached_tokens") and usage.cached_tokens: - cache_read = usage.cached_tokens - - # OpenRouter/Anthropic-proxy style: cache_creation_input_tokens - cache_creation = 0 - if ( - hasattr(usage, "cache_creation_input_tokens") - and usage.cache_creation_input_tokens - ): - cache_creation = usage.cache_creation_input_tokens - - return cache_creation, cache_read - - -class HonchoLLMCallResponse(BaseModel, Generic[T]): - """ - Response object for LLM calls. - - Args: - content: The response content. When a response_model is provided, this will be - the parsed object of that type. Otherwise, it will be a string. - input_tokens: Total number of input tokens (including cached). - output_tokens: Number of tokens generated in the response. - cache_creation_input_tokens: Number of tokens written to cache. - cache_read_input_tokens: Number of tokens read from cache. - finish_reasons: List of finish reasons for the response. - tool_calls_made: Optional list of all tool calls executed during the request. - - Note: - Uncached input tokens = input_tokens - cache_read_input_tokens + cache_creation_input_tokens - (cache_creation costs 25% more, cache_read costs 90% less) - """ - - content: T - input_tokens: int = 0 - output_tokens: int - cache_creation_input_tokens: int = 0 - cache_read_input_tokens: int = 0 - finish_reasons: list[str] - tool_calls_made: list[dict[str, Any]] = Field(default_factory=list) - iterations: int = 0 - """Number of LLM calls made in the tool execution loop (1 = single response, 2+ = tool use iterations plus final synthesis).""" - thinking_content: str | None = None - # Full thinking blocks with signatures for multi-turn conversation replay (Anthropic only) - thinking_blocks: list[dict[str, Any]] = Field(default_factory=list) - # OpenRouter reasoning_details for Gemini models - must be preserved across turns - reasoning_details: list[dict[str, Any]] = Field(default_factory=list) - - -class HonchoLLMCallStreamChunk(BaseModel): - """ - A single chunk in a streaming LLM response. - - Args: - content: The text content for this chunk. Empty for chunks that only contain metadata. - is_done: Whether this is the final chunk in the stream. - finish_reasons: List of finish reasons if the stream is complete. - output_tokens: Number of tokens generated in the response. Only set on the final chunk. - """ - - content: str - is_done: bool = False - finish_reasons: list[str] = Field(default_factory=list) - output_tokens: int | None = None - - -class StreamingResponseWithMetadata: - """ - Wrapper for streaming responses that includes metadata from the tool execution phase. - - This allows callers to access tool call counts, token usage, and thinking content - from the tool loop while still streaming the final response. - """ - - _stream: AsyncIterator[HonchoLLMCallStreamChunk] - tool_calls_made: list[dict[str, Any]] - input_tokens: int - output_tokens: int - cache_creation_input_tokens: int - cache_read_input_tokens: int - thinking_content: str | None - iterations: int - - def __init__( - self, - stream: AsyncIterator[HonchoLLMCallStreamChunk], - tool_calls_made: list[dict[str, Any]], - input_tokens: int, - output_tokens: int, - cache_creation_input_tokens: int, - cache_read_input_tokens: int, - thinking_content: str | None = None, - iterations: int = 0, - ): - self._stream = stream - self.tool_calls_made = tool_calls_made - self.input_tokens = input_tokens - self.output_tokens = output_tokens - self.cache_creation_input_tokens = cache_creation_input_tokens - self.cache_read_input_tokens = cache_read_input_tokens - self.thinking_content = thinking_content - self.iterations = iterations - - def __aiter__(self) -> AsyncIterator[HonchoLLMCallStreamChunk]: - return self._stream.__aiter__() - - async def __anext__(self) -> HonchoLLMCallStreamChunk: - return await self._stream.__anext__() - - -# Bounds for max_tool_iterations to prevent runaway loops -MIN_TOOL_ITERATIONS = 1 -MAX_TOOL_ITERATIONS = 100 - - -async def _stream_final_response( - llm_settings: "LLMComponentSettings", - prompt: str, - max_tokens: int, - conversation_messages: list[dict[str, Any]], - response_model: type[BaseModel] | None, - json_mode: bool, - temperature: float | None, - stop_seqs: list[str] | None, - reasoning_effort: ReasoningEffortType, - verbosity: VerbosityType, - thinking_budget_tokens: int | None, -) -> AsyncIterator[HonchoLLMCallStreamChunk]: - """ - Stream the final response after tool execution is complete. - - Makes a streaming LLM call with the accumulated conversation messages - (which include all tool call results) to generate the final answer. - - Args: - llm_settings: Settings for the LLM provider - prompt: Original prompt (used as fallback) - max_tokens: Maximum tokens to generate - conversation_messages: Full conversation history including tool results - response_model: Optional Pydantic model for structured output - json_mode: Whether to use JSON mode - temperature: Temperature for the LLM - stop_seqs: Stop sequences - reasoning_effort: OpenAI reasoning effort (GPT-5 only) - verbosity: OpenAI verbosity (GPT-5 only) - thinking_budget_tokens: Anthropic thinking budget - - Yields: - HonchoLLMCallStreamChunk objects containing the streaming response - """ - provider = llm_settings.PROVIDER - model = llm_settings.MODEL - - client = CLIENTS.get(provider) - if not client: - raise ValueError(f"Missing client for {provider}") - - # Make a streaming call without tools - stream_response = await honcho_llm_call_inner( - provider, - model, - prompt, - max_tokens, - response_model, - json_mode, - _get_effective_temperature(temperature), - stop_seqs, - reasoning_effort, - verbosity, - thinking_budget_tokens, - True, # stream=True - None, # No tools - None, # No tool_choice - conversation_messages, - ) - - # Yield chunks from the streaming response - async for chunk in stream_response: - yield chunk - - -async def _execute_tool_loop( - llm_settings: "LLMComponentSettings", - prompt: str, - max_tokens: int, - messages: list[dict[str, Any]] | None, - tools: list[dict[str, Any]], - tool_choice: str | dict[str, Any] | None, - tool_executor: Callable[[str, dict[str, Any]], Any], - max_tool_iterations: int, - response_model: type[BaseModel] | None, - json_mode: bool, - temperature: float | None, - stop_seqs: list[str] | None, - reasoning_effort: ReasoningEffortType, - verbosity: VerbosityType, - thinking_budget_tokens: int | None, - enable_retry: bool, - retry_attempts: int, - max_input_tokens: int | None, - get_provider_and_model: Callable[ - [], - tuple[SupportedProviders, str, int | None, ReasoningEffortType, VerbosityType], - ], - before_retry_callback: Callable[[Any], None], - stream_final: bool = False, - iteration_callback: IterationCallback | None = None, -) -> HonchoLLMCallResponse[Any] | StreamingResponseWithMetadata: - """ - Execute the tool calling loop for agentic LLM interactions. - - This function handles the iterative process of: - 1. Making an LLM call with tools available - 2. Executing any tool calls the LLM requests - 3. Feeding tool results back to the LLM - 4. Repeating until the LLM stops calling tools or max iterations reached - - Args: - llm_settings: Settings for the LLM provider - prompt: Initial prompt (used if messages is None) - max_tokens: Maximum tokens to generate per call - messages: Conversation history - tools: Tool definitions in Anthropic format - tool_choice: Tool selection strategy - tool_executor: Async function to execute tools - max_tool_iterations: Maximum iterations before forcing completion - response_model: Optional Pydantic model for structured output - json_mode: Whether to use JSON mode - temperature: Temperature for the LLM (default **none**, only some models support this) - stop_seqs: Stop sequences - reasoning_effort: OpenAI reasoning effort (GPT-5 only) - verbosity: OpenAI verbosity (GPT-5 only) - thinking_budget_tokens: Anthropic thinking budget - enable_retry: Whether to enable retry with exponential backoff - retry_attempts: Number of retry attempts - max_input_tokens: Maximum input tokens (for truncation) - get_provider_and_model: Function to get current provider/model based on attempt - before_retry_callback: Callback for retry events - stream_final: If True, stream the final response instead of returning it synchronously - iteration_callback: Optional callback invoked after each iteration with IterationData - - Returns: - Final HonchoLLMCallResponse with accumulated token counts and tool call history, - or an AsyncIterator of HonchoLLMCallStreamChunk if stream_final=True - """ - # Initialize conversation messages - conversation_messages: list[dict[str, Any]] = ( - messages.copy() if messages else [{"role": "user", "content": prompt}] - ) - - iteration = 0 - all_tool_calls: list[dict[str, Any]] = [] - total_input_tokens = 0 - total_output_tokens = 0 - total_cache_creation_tokens = 0 - total_cache_read_tokens = 0 - empty_response_retries = 0 - # Track effective tool_choice - switches from "required" to "auto" after first iteration - effective_tool_choice = tool_choice - - while iteration < max_tool_iterations: - # Reset attempt counter so each iteration starts with the primary provider - _current_attempt.set(1) - logger.debug(f"Tool execution iteration {iteration + 1}/{max_tool_iterations}") - - # Truncate BEFORE making the API call to avoid context length errors - if max_input_tokens is not None: - conversation_messages = truncate_messages_to_fit( - conversation_messages, max_input_tokens - ) - - # Create a wrapper that injects our messages - async def _call_with_messages( - effective_tool_choice: str | dict[str, Any] | None = effective_tool_choice, - conversation_messages: list[dict[str, Any]] = conversation_messages, - ) -> HonchoLLMCallResponse[Any]: - # Use shared provider selection helper - provider, model, thinking_budget, gpt5_reasoning_effort, gpt5_verbosity = ( - get_provider_and_model() - ) - - client = CLIENTS.get(provider) - if not client: - raise ValueError(f"Missing client for {provider}") - - converted_tools = ( - convert_tools_for_provider(tools, provider) if tools else None - ) - - return await honcho_llm_call_inner( - provider, - model, - prompt, # Will be ignored since we pass messages - max_tokens, - response_model, - json_mode, - _get_effective_temperature(temperature), - stop_seqs, - gpt5_reasoning_effort, - gpt5_verbosity, - thinking_budget, - False, - converted_tools, - effective_tool_choice, - conversation_messages, - ) - - # Apply retry if enabled - if enable_retry: - call_func = retry( - stop=stop_after_attempt(retry_attempts), - wait=wait_exponential(multiplier=1, min=4, max=10), - before_sleep=before_retry_callback, - )(_call_with_messages) - else: - call_func = _call_with_messages - - # Make the call - response = await call_func() - - # Accumulate tokens from this iteration - total_input_tokens += response.input_tokens - total_output_tokens += response.output_tokens - total_cache_creation_tokens += response.cache_creation_input_tokens - total_cache_read_tokens += response.cache_read_input_tokens - - # Check if there are tool calls - if not response.tool_calls_made: - # No tool calls, return final response - logger.debug("No tool calls in response, finishing") - - if ( - isinstance(response.content, str) - and not response.content.strip() - and empty_response_retries < 1 - and iteration < max_tool_iterations - 1 - ): - empty_response_retries += 1 - conversation_messages.append( - { - "role": "user", - "content": ( - "Your last response was empty. Provide a concise answer " - "to the original query using the available context." - ), - } - ) - iteration += 1 - continue - - if stream_final: - # Stream the final response with metadata from tool execution - stream = _stream_final_response( - llm_settings=llm_settings, - prompt=prompt, - max_tokens=max_tokens, - conversation_messages=conversation_messages, - response_model=response_model, - json_mode=json_mode, - temperature=temperature, - stop_seqs=stop_seqs, - reasoning_effort=reasoning_effort, - verbosity=verbosity, - thinking_budget_tokens=thinking_budget_tokens, - ) - return StreamingResponseWithMetadata( - stream=stream, - tool_calls_made=all_tool_calls, - input_tokens=total_input_tokens, - output_tokens=total_output_tokens, - cache_creation_input_tokens=total_cache_creation_tokens, - cache_read_input_tokens=total_cache_read_tokens, - thinking_content=response.thinking_content, - iterations=iteration + 1, - ) - - response.tool_calls_made = all_tool_calls - response.input_tokens = total_input_tokens - response.output_tokens = total_output_tokens - response.cache_creation_input_tokens = total_cache_creation_tokens - response.cache_read_input_tokens = total_cache_read_tokens - response.iterations = iteration + 1 - return response - - # Determine which provider we're using (reuse the helper) - current_provider, _, _, _, _ = get_provider_and_model() - - # Add assistant message with tool calls to conversation - assistant_message = _format_assistant_tool_message( - current_provider, - response.content, - response.tool_calls_made, - response.thinking_blocks, - response.reasoning_details, - ) - conversation_messages.append(assistant_message) - - # Set current iteration for telemetry context (1-indexed) - set_current_iteration(iteration + 1) - - # Execute tools and add results - tool_results: list[dict[str, Any]] = [] - for tool_call in response.tool_calls_made: - tool_name = tool_call["name"] - tool_input = tool_call["input"] - tool_id = tool_call.get("id", "") - - logger.debug(f"Executing tool: {tool_name}") - - try: - # Execute the tool - tool_result = await tool_executor(tool_name, tool_input) - - # Store for Anthropic format - tool_results.append( - { - "tool_id": tool_id, - "tool_name": tool_name, - "result": tool_result, - } - ) - - all_tool_calls.append( - { - "tool_name": tool_name, - "tool_input": tool_input, - "tool_result": tool_result, - } - ) - - except Exception as e: - logger.error(f"Tool execution failed for {tool_name}: {e}") - tool_results.append( - { - "tool_id": tool_id, - "tool_name": tool_name, - "result": f"Error: {str(e)}", - "is_error": True, - } - ) - - # Add tool result message in provider-specific format - _append_tool_results(current_provider, tool_results, conversation_messages) - - # Call iteration callback if provided - if iteration_callback is not None: - try: - iteration_data = IterationData( - iteration=iteration + 1, # 1-indexed - tool_calls=[tc["name"] for tc in response.tool_calls_made], - input_tokens=response.input_tokens, - output_tokens=response.output_tokens, - cache_read_tokens=response.cache_read_input_tokens or 0, - cache_creation_tokens=response.cache_creation_input_tokens or 0, - ) - iteration_callback(iteration_data) - except Exception: - logger.warning("iteration_callback failed", exc_info=True) - - # After first iteration, switch from "required" to "auto" to allow model to stop - if iteration == 0 and effective_tool_choice in ("required", "any"): - effective_tool_choice = "auto" - logger.debug( - "Switched tool_choice from 'required'/'any' to 'auto' after first iteration" - ) - - iteration += 1 - - # Max iterations reached - logger.warning( - f"Tool execution loop reached max iterations ({max_tool_iterations})" - ) - - # Add a synthesis prompt to help the model generate a response - # without tool calls - the conversation currently ends with tool results - # and the model may not know to produce text output - synthesis_prompt = ( - "You have reached the maximum number of tool calls. " - "Based on all the information you have gathered, provide your final response now. " - "Do not attempt to call any more tools." - ) - conversation_messages.append({"role": "user", "content": synthesis_prompt}) - - # If streaming the final response, use the streaming helper with metadata - if stream_final: - stream = _stream_final_response( - llm_settings=llm_settings, - prompt=prompt, - max_tokens=max_tokens, - conversation_messages=conversation_messages, - response_model=response_model, - json_mode=json_mode, - temperature=temperature, - stop_seqs=stop_seqs, - reasoning_effort=reasoning_effort, - verbosity=verbosity, - thinking_budget_tokens=thinking_budget_tokens, - ) - return StreamingResponseWithMetadata( - stream=stream, - tool_calls_made=all_tool_calls, - input_tokens=total_input_tokens, - output_tokens=total_output_tokens, - cache_creation_input_tokens=total_cache_creation_tokens, - cache_read_input_tokens=total_cache_read_tokens, - thinking_content=None, # No thinking content at max iterations - iterations=iteration + 1, # +1 for the synthesis call - ) - - # Make one final call to get a text response - _current_attempt.set(1) # Reset attempt counter - - async def _final_call() -> HonchoLLMCallResponse[Any]: - # Use shared provider selection helper for backup failover support - provider, model, thinking_budget, gpt5_reasoning_effort, gpt5_verbosity = ( - get_provider_and_model() - ) - - client = CLIENTS.get(provider) - if not client: - raise ValueError(f"Missing client for {provider}") - - # No tools for final call - return await honcho_llm_call_inner( - provider, - model, - prompt, - max_tokens, - response_model, - json_mode, - _get_effective_temperature(temperature), - stop_seqs, - gpt5_reasoning_effort, - gpt5_verbosity, - thinking_budget, - False, - None, # No tools - None, # No tool_choice - conversation_messages, - ) - - if enable_retry: - final_call_func = retry( - stop=stop_after_attempt(retry_attempts), - wait=wait_exponential(multiplier=1, min=4, max=10), - before_sleep=before_retry_callback, - )(_final_call) - else: - final_call_func = _final_call - - final_response = await final_call_func() - final_response.tool_calls_made = all_tool_calls - final_response.iterations = iteration + 1 # +1 for the synthesis call - # Include accumulated tokens from all iterations plus the final call - final_response.input_tokens = total_input_tokens + final_response.input_tokens - final_response.output_tokens = total_output_tokens + final_response.output_tokens - final_response.cache_creation_input_tokens = ( - total_cache_creation_tokens + final_response.cache_creation_input_tokens - ) - final_response.cache_read_input_tokens = ( - total_cache_read_tokens + final_response.cache_read_input_tokens - ) - return final_response - - -def _format_assistant_tool_message( - provider: SupportedProviders, - content: Any, - tool_calls: list[dict[str, Any]], - thinking_blocks: list[dict[str, Any]] | None = None, - reasoning_details: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """ - Format an assistant message with tool calls for a specific provider. - - Args: - provider: The LLM provider - content: The text content from the response - tool_calls: List of tool call dicts with id, name, input keys - thinking_blocks: Full thinking blocks with signatures for multi-turn replay (Anthropic only) - reasoning_details: OpenRouter reasoning_details for Gemini models (must be preserved) - - Returns: - Provider-formatted assistant message dict - """ - if provider == "anthropic": - # Anthropic requires content to be a list of blocks including tool use blocks - content_blocks: list[dict[str, Any]] = [] - - # Add thinking blocks FIRST if present (required by Anthropic when extended thinking is enabled) - # These include signatures which are required for multi-turn conversation replay - if thinking_blocks: - content_blocks.extend(thinking_blocks) - - # Add text content if present - if isinstance(content, str) and content: - content_blocks.append({"type": "text", "text": content}) - - # Add tool use blocks - for tool_call in tool_calls: - content_blocks.append( - { - "type": "tool_use", - "id": tool_call["id"], - "name": tool_call["name"], - "input": tool_call["input"], - } - ) - - return { - "role": "assistant", - "content": content_blocks, - } - elif provider == "google": - # Google format: model role with function_call parts - parts: list[dict[str, Any]] = [] - - # Add text content if present - if isinstance(content, str) and content: - parts.append({"text": content}) - - # Add function call parts with thought_signature if present - for tool_call in tool_calls: - part_data: dict[str, Any] = { - "function_call": { - "name": tool_call["name"], - "args": tool_call["input"], - } - } - # Include thought_signature if present (required by Gemini) - if "thought_signature" in tool_call: - part_data["thought_signature"] = tool_call["thought_signature"] - parts.append(part_data) - - return { - "role": "model", - "parts": parts, - } - else: - # OpenAI format - must include tool_calls in the assistant message - openai_tool_calls: list[Any] = [] - for tool_call in tool_calls: - openai_tool_calls.append( - { - "id": tool_call["id"], - "type": "function", - "function": { - "name": tool_call["name"], - "arguments": json.dumps(tool_call["input"]), - }, - } - ) - msg: dict[str, Any] = { - "role": "assistant", - "content": content if isinstance(content, str) else None, - "tool_calls": openai_tool_calls, - } - # Include reasoning_details for OpenRouter/Gemini (required for multi-turn tool use) - if reasoning_details: - msg["reasoning_details"] = reasoning_details - return msg - - -def _append_tool_results( - provider: SupportedProviders, - tool_results: list[dict[str, Any]], - conversation_messages: list[dict[str, Any]], -) -> None: - """ - Append tool results to conversation messages in provider-specific format. - - Args: - provider: The LLM provider - tool_results: List of tool result dicts with tool_id, tool_name, result, is_error keys - conversation_messages: The conversation to append to (modified in place) - """ - if provider == "anthropic": - # Anthropic requires tool results in specific content blocks - result_blocks: list[dict[str, Any]] = [] - for tr in tool_results: - result_blocks.append( - { - "type": "tool_result", - "tool_use_id": tr["tool_id"], - "content": str(tr["result"]), - "is_error": tr.get("is_error", False), - } - ) - - conversation_messages.append( - { - "role": "user", - "content": result_blocks, - } - ) - elif provider == "google": - # Google format: user role with function_response parts - response_parts: list[dict[str, Any]] = [] - for tr in tool_results: - response_parts.append( - { - "function_response": { - "name": tr["tool_name"], - "response": {"result": str(tr["result"])}, - } - } - ) - - conversation_messages.append( - { - "role": "user", - "parts": response_parts, - } - ) - else: - # OpenAI format - add each tool result as a separate message with role="tool" - for tr in tool_results: - conversation_messages.append( - { - "role": "tool", - "tool_call_id": tr["tool_id"], - "content": str(tr["result"]), - } - ) - - -@overload -async def honcho_llm_call( - llm_settings: LLMComponentSettings, - prompt: str, - max_tokens: int, - track_name: str | None = None, - *, - response_model: type[M], - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, - enable_retry: bool = True, - retry_attempts: int = 3, - stream: Literal[False] = False, - stream_final_only: bool = False, - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, - max_tool_iterations: int = 10, - messages: list[dict[str, Any]] | None = None, - max_input_tokens: int | None = None, - trace_name: str | None = None, - iteration_callback: IterationCallback | None = None, -) -> HonchoLLMCallResponse[M]: ... - - -@overload -async def honcho_llm_call( - llm_settings: LLMComponentSettings, - prompt: str, - max_tokens: int, - track_name: str | None = None, - response_model: None = None, - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, - enable_retry: bool = True, - retry_attempts: int = 3, - stream: Literal[False] = False, - stream_final_only: bool = False, - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, - max_tool_iterations: int = 10, - messages: list[dict[str, Any]] | None = None, - max_input_tokens: int | None = None, - trace_name: str | None = None, - iteration_callback: IterationCallback | None = None, -) -> HonchoLLMCallResponse[str]: ... - - -@overload -async def honcho_llm_call( - llm_settings: LLMComponentSettings, - prompt: str, - max_tokens: int, - track_name: str | None = None, - response_model: type[BaseModel] | None = None, - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, - enable_retry: bool = True, - retry_attempts: int = 3, - stream: Literal[True] = ..., - stream_final_only: bool = False, - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, - max_tool_iterations: int = 10, - messages: list[dict[str, Any]] | None = None, - max_input_tokens: int | None = None, - trace_name: str | None = None, - iteration_callback: IterationCallback | None = None, -) -> AsyncIterator[HonchoLLMCallStreamChunk] | StreamingResponseWithMetadata: ... - - -@conditional_observe(name="LLM Call") -async def honcho_llm_call( - llm_settings: LLMComponentSettings, - prompt: str, - max_tokens: int, - track_name: str | None = None, - response_model: type[BaseModel] | None = None, - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, - enable_retry: bool = True, - retry_attempts: int = 3, - stream: bool = False, - stream_final_only: bool = False, - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - tool_executor: Callable[[str, dict[str, Any]], Any] | None = None, - max_tool_iterations: int = 10, - messages: list[dict[str, Any]] | None = None, - max_input_tokens: int | None = None, - trace_name: str | None = None, - iteration_callback: IterationCallback | None = None, -) -> ( - HonchoLLMCallResponse[Any] - | AsyncIterator[HonchoLLMCallStreamChunk] - | StreamingResponseWithMetadata -): - """ - Make an LLM call with automatic backup provider failover. Backup provider/model - is used on the final retry attempt, which is 3 by default. - - Args: - llm_settings: Settings object containing PROVIDER, MODEL, - BACKUP_PROVIDER, and BACKUP_MODEL - prompt: The prompt to send to the LLM (used if messages is None) - max_tokens: Maximum tokens to generate - track_name: Optional name for AI tracking - response_model: Optional Pydantic model for structured output - json_mode: Whether to use JSON mode - temperature: Temperature for the LLM (default **none**, only some models support this) - stop_seqs: Stop sequences - reasoning_effort: OpenAI reasoning effort (GPT-5 only) - verbosity: OpenAI verbosity (GPT-5 only) - thinking_budget_tokens: Anthropic thinking budget - enable_retry: Whether to enable retry with exponential backoff - retry_attempts: Number of retry attempts - stream: Whether to stream the response - stream_final_only: If True with tools, run tool loop non-streaming then stream final answer - tools: Tool definitions for tool calling (Anthropic/OpenAI format) - tool_choice: Tool selection strategy (auto/required/specific tool) - tool_executor: Async callable to execute tools, receives (tool_name, tool_input) - max_tool_iterations: Maximum number of tool execution loops - messages: Optional message list for multi-turn conversations (overrides prompt) - iteration_callback: Optional callback invoked after each tool iteration with IterationData - - Returns: - HonchoLLMCallResponse or AsyncIterator depending on stream parameter - - Raises: - ValueError: If provider is not configured - """ - # Validate that streaming and tools are not used together - # (unless stream_final_only is set, which streams only the final response after tool calls) - if stream and tools and not stream_final_only: - raise ValueError( - "Streaming is not supported with tool calling. Set stream=False when using tools, " - + "or use stream_final_only=True to stream only the final response after tool calls." - ) - - # Set attempt counter to 1 for first call (tenacity uses 1-indexed attempts) - _current_attempt.set(1) - - def _get_provider_and_model() -> ( - tuple[SupportedProviders, str, int | None, ReasoningEffortType, VerbosityType] - ): - """ - Get the provider and model to use based on current attempt. - - Returns: - Tuple of (provider, model, thinking_budget, reasoning_effort, verbosity) - """ - attempt = _current_attempt.get() - - provider: SupportedProviders - model: str - thinking_budget: int | None - gpt5_reasoning_effort: ReasoningEffortType - gpt5_verbosity: VerbosityType - - # Use backup on final retry attempt (when attempt == retry_attempts) - if ( - attempt == retry_attempts - and llm_settings.BACKUP_PROVIDER is not None - and llm_settings.BACKUP_MODEL is not None - and llm_settings.BACKUP_PROVIDER in CLIENTS - ): - provider = llm_settings.BACKUP_PROVIDER - model = llm_settings.BACKUP_MODEL - thinking_budget = thinking_budget_tokens - gpt5_reasoning_effort = reasoning_effort - gpt5_verbosity = verbosity - - # Filter out incompatible parameters when using backup - if provider != "anthropic" and thinking_budget: - logger.warning( - f"thinking_budget_tokens not supported by {provider}, ignoring" - ) - thinking_budget = None - - if "gpt-5" not in model and (gpt5_reasoning_effort or gpt5_verbosity): - logger.warning( - "reasoning_effort/verbosity only supported by GPT-5 models, ignoring" - ) - gpt5_reasoning_effort = None - gpt5_verbosity = None - - logger.warning( - f"Final retry attempt {attempt}/{retry_attempts}: switching from " - + f"{llm_settings.PROVIDER}/{llm_settings.MODEL} to " - + f"backup {provider}/{model}" - ) - else: - provider = llm_settings.PROVIDER - model = llm_settings.MODEL - thinking_budget = thinking_budget_tokens - gpt5_reasoning_effort = reasoning_effort - gpt5_verbosity = verbosity - - return provider, model, thinking_budget, gpt5_reasoning_effort, gpt5_verbosity - - async def _call_with_provider_selection() -> ( - HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk] - ): - """ - Inner function that selects provider/model based on current attempt. - This function is retried, so provider selection happens on each attempt. - """ - provider, model, thinking_budget, gpt5_reasoning_effort, gpt5_verbosity = ( - _get_provider_and_model() - ) - - # Validate client exists - client = CLIENTS.get(provider) - if not client: - raise ValueError(f"Missing client for {provider}") - - # Convert tools to provider-specific format if provided - converted_tools = convert_tools_for_provider(tools, provider) if tools else None - - if stream: - return await honcho_llm_call_inner( - provider, - model, - prompt, - max_tokens, - response_model, - json_mode, - _get_effective_temperature(temperature), - stop_seqs, - gpt5_reasoning_effort, - gpt5_verbosity, - thinking_budget, - True, # type: ignore[arg-type] - converted_tools, - tool_choice, - ) - else: - return await honcho_llm_call_inner( - provider, - model, - prompt, - max_tokens, - response_model, - json_mode, - _get_effective_temperature(temperature), - stop_seqs, - gpt5_reasoning_effort, - gpt5_verbosity, - thinking_budget, - False, # type: ignore[arg-type] - converted_tools, - tool_choice, - ) - - decorated = _call_with_provider_selection - - # apply tracking - if track_name: - decorated = ai_track(track_name)(decorated) - - # Define retry callback for updating attempt counter and logging - def before_retry_callback(retry_state: Any) -> None: - """Update attempt counter before each retry. - - Note: before_sleep is called AFTER an attempt fails and BEFORE sleeping, - so we need to increment to the next attempt number. - """ - next_attempt = retry_state.attempt_number + 1 - _current_attempt.set(next_attempt) - exc = retry_state.outcome.exception() if retry_state.outcome else None - if exc: - logger.warning( - f"Error on attempt {retry_state.attempt_number}/{retry_attempts} with " - + f"{llm_settings.PROVIDER}/{llm_settings.MODEL}: {exc}" - ) - logger.info(f"Will retry with attempt {next_attempt}/{retry_attempts}") - - # apply retry logic - retries on ANY exception - if enable_retry: - decorated = retry( - stop=stop_after_attempt(retry_attempts), - wait=wait_exponential(multiplier=1, min=4, max=10), - before_sleep=before_retry_callback, - )(decorated) - - # If no tools or no tool_executor, just call once and return - if not tools or not tool_executor: - result: ( - HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk] - ) = await decorated() - if trace_name and isinstance(result, HonchoLLMCallResponse): - log_reasoning_trace( - task_type=trace_name, - llm_settings=llm_settings, - prompt=prompt, - response=result, - max_tokens=max_tokens, - thinking_budget_tokens=thinking_budget_tokens, - reasoning_effort=reasoning_effort, - json_mode=json_mode, - stop_seqs=stop_seqs, - messages=messages, - ) - return result - - # Validate and clamp max_tool_iterations - clamped_iterations = max( - MIN_TOOL_ITERATIONS, min(max_tool_iterations, MAX_TOOL_ITERATIONS) - ) - if clamped_iterations != max_tool_iterations: - logger.warning( - f"max_tool_iterations {max_tool_iterations} clamped to {clamped_iterations} " - + f"(valid range: {MIN_TOOL_ITERATIONS}-{MAX_TOOL_ITERATIONS})" - ) - - # Delegate to the tool execution loop - result = await _execute_tool_loop( - llm_settings=llm_settings, - prompt=prompt, - max_tokens=max_tokens, - messages=messages, - tools=tools, - tool_choice=tool_choice, - tool_executor=tool_executor, - max_tool_iterations=clamped_iterations, - response_model=response_model, - json_mode=json_mode, - temperature=temperature, - stop_seqs=stop_seqs, - reasoning_effort=reasoning_effort, - verbosity=verbosity, - thinking_budget_tokens=thinking_budget_tokens, - enable_retry=enable_retry, - retry_attempts=retry_attempts, - max_input_tokens=max_input_tokens, - get_provider_and_model=_get_provider_and_model, - before_retry_callback=before_retry_callback, - stream_final=stream_final_only, - iteration_callback=iteration_callback, - ) - if trace_name and isinstance(result, HonchoLLMCallResponse): - log_reasoning_trace( - task_type=trace_name, - llm_settings=llm_settings, - prompt=prompt, - response=result, - max_tokens=max_tokens, - thinking_budget_tokens=thinking_budget_tokens, - reasoning_effort=reasoning_effort, - json_mode=json_mode, - stop_seqs=stop_seqs, - messages=messages, - ) - return result - - -@overload -async def honcho_llm_call_inner( - provider: SupportedProviders, - model: str, - prompt: str, - max_tokens: int, - response_model: type[M], - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, # Anthropic only - stream: Literal[False] = False, - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - messages: list[dict[str, Any]] | None = None, -) -> HonchoLLMCallResponse[M]: ... - - -@overload -async def honcho_llm_call_inner( - provider: SupportedProviders, - model: str, - prompt: str, - max_tokens: int, - response_model: None = None, - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, # Anthropic only - stream: Literal[False] = False, - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - messages: list[dict[str, Any]] | None = None, -) -> HonchoLLMCallResponse[str]: ... - - -@overload -async def honcho_llm_call_inner( - provider: SupportedProviders, - model: str, - prompt: str, - max_tokens: int, - response_model: type[BaseModel] | None = None, - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, # Anthropic only - stream: Literal[True] = ..., - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - messages: list[dict[str, Any]] | None = None, -) -> AsyncIterator[HonchoLLMCallStreamChunk]: ... - - -async def honcho_llm_call_inner( - provider: SupportedProviders, - model: str, - prompt: str, - max_tokens: int, - response_model: type[BaseModel] | None = None, - json_mode: bool = False, - temperature: float | None = None, - stop_seqs: list[str] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] - | None = None, # OpenAI only - verbosity: Literal["low", "medium", "high"] | None = None, # OpenAI only - thinking_budget_tokens: int | None = None, # Anthropic only - stream: bool = False, - tools: list[dict[str, Any]] | None = None, - tool_choice: str | dict[str, Any] | None = None, - messages: list[dict[str, Any]] | None = None, -) -> HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk]: - # has already been validated by honcho_llm_call - client = CLIENTS[provider] - - # Use messages if provided, otherwise convert prompt to message - if messages is None: - messages = [{"role": "user", "content": prompt}] - - params: dict[str, Any] = { - "model": model, - "max_tokens": max_tokens, - "messages": messages, - "stream": stream, - } - - if temperature is not None: - params["temperature"] = temperature - - if stream: - # Return async generator for streaming responses - return handle_streaming_response( - client, - params, - json_mode, - thinking_budget_tokens, - response_model, - reasoning_effort, - verbosity, - ) - - # Remove stream parameter for non-streaming calls as some providers don't accept it - params.pop("stream", None) - - system_messages: list[str] = [] - non_system_messages: list[dict[str, Any]] = [] - - match client: - case AsyncAnthropic(): - # Anthropic requires system messages to be passed as a top-level parameter - # Extract system messages and non-system messages - for msg in params["messages"]: - if msg.get("role") == "system": - system_messages.append(msg["content"]) - else: - non_system_messages.append(msg) - - anthropic_params: dict[str, Any] = { - "model": params["model"], - "max_tokens": params["max_tokens"], - "messages": non_system_messages, - } - - if temperature is not None: - anthropic_params["temperature"] = temperature - - # Add system parameter if there are system messages - # Use cache_control for prompt caching - if system_messages: - anthropic_params["system"] = [ - { - "type": "text", - "text": "\n\n".join(system_messages), - "cache_control": {"type": "ephemeral"}, - } - ] - - # Add tools if provided - if tools: - anthropic_params["tools"] = tools - if tool_choice: - # Convert tool_choice to Anthropic format - if isinstance(tool_choice, str): - if tool_choice == "auto": - anthropic_params["tool_choice"] = {"type": "auto"} - elif tool_choice in ("any", "required"): - anthropic_params["tool_choice"] = {"type": "any"} - elif tool_choice == "none": - # Don't set tool_choice, let Anthropic default - pass - else: - # Assume it's a tool name - anthropic_params["tool_choice"] = { - "type": "tool", - "name": tool_choice, - } - else: - # Already in dict format, use as-is - anthropic_params["tool_choice"] = tool_choice - - # For response models, we need to request JSON and parse manually - # Note: tools and response_model should not be used together - if response_model or json_mode: - # Add JSON schema instructions to the prompt if using response_model - if response_model: - schema_json = json.dumps( - response_model.model_json_schema(), indent=2 - ) - anthropic_params["messages"][-1]["content"] += ( - f"\n\nRespond with valid JSON matching this schema:\n{schema_json}" - ) - anthropic_params["messages"].append( - {"role": "assistant", "content": "{"} - ) - - if thinking_budget_tokens: - anthropic_params["thinking"] = { - "type": "enabled", - "budget_tokens": thinking_budget_tokens, - } - - anthropic_response: AnthropicMessage = cast( - AnthropicMessage, await client.messages.create(**anthropic_params) - ) - - # Extract text content, thinking blocks, and tool use blocks from content blocks - text_blocks: list[str] = [] - thinking_text_blocks: list[str] = [] - thinking_full_blocks: list[dict[str, Any]] = [] - tool_calls: list[dict[str, Any]] = [] - for block in anthropic_response.content: - if isinstance(block, TextBlock): - text_blocks.append(block.text) - elif isinstance(block, ThinkingBlock): - thinking_text_blocks.append(block.thinking) - # Store full block with signature for multi-turn replay - thinking_full_blocks.append( - { - "type": "thinking", - "thinking": block.thinking, - "signature": block.signature, - } - ) - elif isinstance(block, ToolUseBlock): - tool_calls.append( - { - "id": block.id, - "name": block.name, - "input": block.input, - } - ) - - # Safely extract usage and stop_reason - usage: Any | Usage = anthropic_response.usage - stop_reason = anthropic_response.stop_reason - - text_content = "\n".join(text_blocks) - thinking_content = ( - "\n".join(thinking_text_blocks) if thinking_text_blocks else None - ) - - # Extract cache token counts from Anthropic usage - # Anthropic's input_tokens = uncached tokens only - # Total = input_tokens + cache_read + cache_creation - cache_creation_tokens = ( - getattr(usage, "cache_creation_input_tokens", 0) or 0 if usage else 0 - ) - cache_read_tokens = ( - getattr(usage, "cache_read_input_tokens", 0) or 0 if usage else 0 - ) - uncached_tokens = usage.input_tokens if usage else 0 - # Calculate total input tokens for consistent reporting - total_input_tokens = ( - uncached_tokens + cache_read_tokens + cache_creation_tokens - ) - - # If using response_model, parse the JSON response - if response_model: - try: - # Add back the opening brace that we prefilled - json_content = "{" + text_content - parsed_json = json.loads(json_content) - parsed_content = response_model.model_validate(parsed_json) - - return HonchoLLMCallResponse( - content=parsed_content, - input_tokens=total_input_tokens, - output_tokens=usage.output_tokens if usage else 0, - cache_creation_input_tokens=cache_creation_tokens, - cache_read_input_tokens=cache_read_tokens, - finish_reasons=[stop_reason] if stop_reason else [], - tool_calls_made=tool_calls, - thinking_content=thinking_content, - thinking_blocks=thinking_full_blocks, - ) - except (json.JSONDecodeError, ValidationError, ValueError) as e: - raise ValueError( - f"Failed to parse Anthropic response as {response_model}: {e}. Raw content: {text_content}" - ) from e - - return HonchoLLMCallResponse( - content=text_content, - input_tokens=total_input_tokens, - output_tokens=usage.output_tokens if usage else 0, - cache_creation_input_tokens=cache_creation_tokens, - cache_read_input_tokens=cache_read_tokens, - finish_reasons=[stop_reason] if stop_reason else [], - tool_calls_made=tool_calls, - thinking_content=thinking_content, - thinking_blocks=thinking_full_blocks, - ) - - case AsyncOpenAI(): - # For custom providers (e.g., OpenRouter), add cache_control to system messages - # This enables prompt caching for Anthropic models proxied via OpenAI-compatible APIs - processed_messages: list[dict[str, Any]] = params["messages"] - if provider == "custom": - processed_messages = [] - for msg in params["messages"]: - if msg.get("role") == "system" and isinstance( - msg.get("content"), str - ): - # Convert system message to content block format with cache_control - processed_messages.append( - { - "role": "system", - "content": [ - { - "type": "text", - "text": msg["content"], - "cache_control": {"type": "ephemeral"}, - } - ], - } - ) - else: - processed_messages.append(msg) - - openai_params: dict[str, Any] = { - "model": params["model"], - "messages": processed_messages, - } - - if temperature is not None and "gpt-5" not in model: - openai_params["temperature"] = temperature - - if "gpt-5" in model: - openai_params["max_completion_tokens"] = params["max_tokens"] - if reasoning_effort: - openai_params["reasoning_effort"] = reasoning_effort - if verbosity: - openai_params["verbosity"] = verbosity - else: - openai_params["max_tokens"] = params["max_tokens"] - - # Add tools if provided (not compatible with response_model for most cases) - if tools and not response_model: - openai_params["tools"] = tools - if tool_choice: - openai_params["tool_choice"] = tool_choice - - if json_mode and provider != "vllm": - openai_params["response_format"] = {"type": "json_object"} - - # custom shim for vLLM response model formatting - # NOTE: this is all specific to the Representation model. - # Do not call with any other response model. - if provider == "vllm" and response_model: - if response_model is not PromptRepresentation: - raise NotImplementedError( - "vLLM structured output currently supports only PromptRepresentation" - ) - openai_params["response_format"] = { - "type": "json_schema", - "json_schema": { - "name": response_model.__name__, - "schema": response_model.model_json_schema(), - }, - } - if stop_seqs: - openai_params["stop"] = stop_seqs - vllm_response: ChatCompletion = cast( - ChatCompletion, - await client.chat.completions.create(**openai_params), - ) - - usage = vllm_response.usage - finish_reason = vllm_response.choices[0].finish_reason - - try: - test_rep = "" - if vllm_response.choices[0].message.content is not None: - test_rep = vllm_response.choices[0].message.content - - final = validate_and_repair_json(test_rep) - - # Schema-aware repair: ensure deductive observations have required fields - - repaired_data = json.loads(final) - - # Fix deductive observations that might be missing conclusion - if "deductive" in repaired_data and isinstance( - repaired_data["deductive"], list - ): - for i, item in enumerate(repaired_data["deductive"]): - if isinstance(item, dict): - # If conclusion is missing but premises exist, create a placeholder - if "conclusion" not in item and "premises" in item: - logger.warning( - f"Deductive observation {i} missing conclusion, adding placeholder" - ) - # Try to generate a conclusion from premises if possible - if item["premises"]: - item["conclusion"] = ( - f"[Incomplete reasoning from premises: {item['premises'][0][:100]}...]" - ) - else: - item["conclusion"] = ( - "[Incomplete reasoning - conclusion missing]" - ) - # If premises is missing, add empty list (it's optional with default) - if "premises" not in item: - item["premises"] = [] - - final = json.dumps(repaired_data) - except (json.JSONDecodeError, KeyError, TypeError) as e: - final = "" - logger.warning(f"Could not perform schema-aware repair: {e}") - # Continue with original final value if repair fails - - try: - response_obj = PromptRepresentation.model_validate_json(final) - except ValidationError as e: - logger.error(f"Validation error after repair: {e}") - logger.debug(f"Problematic JSON: {final}") - - # Fallback: return empty response rather than failing - logger.warning( - "Using fallback empty Representation due to validation error" - ) - response_obj = PromptRepresentation(explicit=[]) # , deductive=[]) - - cache_creation, cache_read = extract_openai_cache_tokens(usage) - return HonchoLLMCallResponse( - content=response_obj, - input_tokens=usage.prompt_tokens if usage else 0, - output_tokens=usage.completion_tokens if usage else 0, - cache_creation_input_tokens=cache_creation, - cache_read_input_tokens=cache_read, - finish_reasons=[finish_reason] if finish_reason else [], - tool_calls_made=[], - thinking_content=extract_openai_reasoning_content(vllm_response), - ) - elif response_model: - openai_params["response_format"] = response_model - response: ChatCompletion = await client.chat.completions.parse( # pyright: ignore - **openai_params - ) - # Extract the parsed object for structured output - parsed_content = response.choices[0].message.parsed - if parsed_content is None: - raise ValueError("No parsed content in structured response") - - usage = response.usage - finish_reason = response.choices[0].finish_reason - - # Validate that parsed content matches the response model - if not isinstance(parsed_content, response_model): - raise ValueError( - f"Parsed content does not match the response model: {parsed_content} != {response_model}" - ) - - # Extract tool calls if present (though unlikely with structured output) - parsed_tool_calls: list[dict[str, Any]] = [] - if ( - hasattr(response.choices[0].message, "tool_calls") - and response.choices[0].message.tool_calls - ): - for tool_call in response.choices[0].message.tool_calls: - parsed_tool_calls.append( - { - "id": tool_call.id, - "name": tool_call.function.name, - "input": json.loads(tool_call.function.arguments) - if tool_call.function.arguments - else {}, - } - ) - - cache_creation, cache_read = extract_openai_cache_tokens(usage) - return HonchoLLMCallResponse( - content=parsed_content, - input_tokens=usage.prompt_tokens if usage else 0, - output_tokens=usage.completion_tokens if usage else 0, - cache_creation_input_tokens=cache_creation, - cache_read_input_tokens=cache_read, - finish_reasons=[finish_reason] if finish_reason else [], - tool_calls_made=parsed_tool_calls, - thinking_content=extract_openai_reasoning_content(response), - ) - else: - response: ChatCompletion = await client.chat.completions.create( # pyright: ignore - **openai_params - ) - - usage = response.usage # pyright: ignore - finish_reason = response.choices[0].finish_reason # pyright: ignore - - # Extract tool calls if present - tool_calls_list: list[dict[str, Any]] = [] - if response.choices[0].message.tool_calls: # pyright: ignore - for tool_call in response.choices[0].message.tool_calls: # pyright: ignore - tool_calls_list.append( - { - "id": tool_call.id, # pyright: ignore - "name": tool_call.function.name, # pyright: ignore - "input": json.loads(tool_call.function.arguments) # pyright: ignore - if tool_call.function.arguments # pyright: ignore - else {}, - } - ) - - cache_creation, cache_read = extract_openai_cache_tokens(usage) - return HonchoLLMCallResponse( - content=response.choices[0].message.content or "", # pyright: ignore - input_tokens=usage.prompt_tokens if usage else 0, # pyright: ignore - output_tokens=usage.completion_tokens if usage else 0, # pyright: ignore - cache_creation_input_tokens=cache_creation, - cache_read_input_tokens=cache_read, - finish_reasons=[finish_reason] if finish_reason else [], - tool_calls_made=tool_calls_list, - thinking_content=extract_openai_reasoning_content(response), - reasoning_details=extract_openai_reasoning_details(response), - ) - - case genai.Client(): - # Build config for Gemini - gemini_config: dict[str, Any] = {} - - # Gemini uses max_output_tokens, not max_tokens. - gemini_config["max_output_tokens"] = params["max_tokens"] - - if temperature is not None: - gemini_config["temperature"] = temperature - - # Add tools if provided - if tools: - gemini_config["tools"] = tools - # Handle tool_choice - if tool_choice: - if tool_choice == "auto": - gemini_config["tool_config"] = { - "function_calling_config": {"mode": "AUTO"} - } - elif tool_choice == "any" or tool_choice == "required": - gemini_config["tool_config"] = { - "function_calling_config": {"mode": "ANY"} - } - elif tool_choice == "none": - gemini_config["tool_config"] = { - "function_calling_config": {"mode": "NONE"} - } - elif isinstance(tool_choice, dict) and "name" in tool_choice: - # Specific tool selection - gemini_config["tool_config"] = { - "function_calling_config": { - "mode": "ANY", - "allowed_function_names": [tool_choice["name"]], - } - } - - if response_model is None: - if json_mode and not tools: - gemini_config["response_mime_type"] = "application/json" - - # Use messages if provided, otherwise use prompt - if messages: - # Extract system messages for system_instruction parameter - # Gemini doesn't support system role in contents - it causes - # consecutive user messages which results in empty responses - for msg in messages: - if msg.get("role") == "system": - if isinstance(msg.get("content"), str): - system_messages.append(msg["content"]) - else: - non_system_messages.append(msg) - - # Add system instruction if present - if system_messages: - gemini_config["system_instruction"] = "\n\n".join( - system_messages - ) - - # Convert non-system messages to Google format - gemini_contents: list[dict[str, Any]] = [] - for msg in non_system_messages: - # Map roles to Google's expected values (user, model) - role = msg.get("role", "user") - if role == "assistant": - role = "model" - - # Handle different content formats - if isinstance(msg.get("content"), str): - # Simple string content - gemini_contents.append( - {"role": role, "parts": [{"text": msg["content"]}]} - ) - elif isinstance(msg.get("parts"), list): - # Already in Google format (from tool calling loop) - # But still need to ensure role is correct - msg_copy = msg.copy() - msg_copy["role"] = role - gemini_contents.append(msg_copy) - elif isinstance(msg.get("content"), list): - # Content is a list of parts (Anthropic format) - skip for now - # This shouldn't happen with Google provider in tool loop - continue - else: - # Empty or unknown format, skip - continue - contents: ContentListUnionDict = cast( - ContentListUnionDict, gemini_contents - ) - else: - contents = prompt - - gemini_response: GenerateContentResponse = ( - await client.aio.models.generate_content( - model=model, - contents=contents, - config=cast(GenerateContentConfigDict, gemini_config) # pyright: ignore[reportInvalidCast] - if gemini_config - else None, - ) - ) - - # Extract text content and function calls from response - text_parts: list[str] = [] - gemini_tool_calls: list[dict[str, Any]] = [] - - if gemini_response.candidates and gemini_response.candidates[0].content: - for part in gemini_response.candidates[0].content.parts or []: - if hasattr(part, "text") and part.text: - text_parts.append(part.text) - if hasattr(part, "function_call") and part.function_call: - fc = part.function_call - tool_call_data: dict[str, Any] = { - "id": f"call_{fc.name}_{len(gemini_tool_calls)}", - "name": fc.name, - "input": dict(fc.args) if fc.args else {}, - } - # Preserve thought_signature if present (required by Gemini) - if ( - hasattr(part, "thought_signature") - and part.thought_signature - ): - tool_call_data["thought_signature"] = ( - part.thought_signature - ) - gemini_tool_calls.append(tool_call_data) - - text_content = "\n".join(text_parts) if text_parts else "" - input_token_count = ( - gemini_response.usage_metadata.prompt_token_count or 0 - if gemini_response.usage_metadata - else 0 - ) - output_token_count = ( - gemini_response.usage_metadata.candidates_token_count or 0 - if gemini_response.usage_metadata - else 0 - ) - finish_reason = ( - gemini_response.candidates[0].finish_reason.name - if gemini_response.candidates - and gemini_response.candidates[0].finish_reason - else "stop" - ) - - # Raise on blocked responses so retry/backup-provider logic kicks in - if ( - not text_content - and not gemini_tool_calls - and finish_reason in GEMINI_BLOCKED_FINISH_REASONS - ): - raise LLMError( - f"Gemini response blocked (finish_reason={finish_reason})", - provider="google", - model=model, - finish_reason=finish_reason, - ) - - return HonchoLLMCallResponse( - content=text_content, - input_tokens=input_token_count, - output_tokens=output_token_count, - finish_reasons=[finish_reason], - tool_calls_made=gemini_tool_calls, - ) - - else: - gemini_config["response_mime_type"] = "application/json" - gemini_config["response_schema"] = response_model - - gemini_response = await client.aio.models.generate_content( - model=model, - contents=prompt, - config=cast(GenerateContentConfigDict, gemini_config), # pyright: ignore[reportInvalidCast] - ) - - input_token_count = ( - gemini_response.usage_metadata.prompt_token_count or 0 - if gemini_response.usage_metadata - else 0 - ) - output_token_count = ( - gemini_response.usage_metadata.candidates_token_count or 0 - if gemini_response.usage_metadata - else 0 - ) - finish_reason = ( - gemini_response.candidates[0].finish_reason.name - if gemini_response.candidates - and gemini_response.candidates[0].finish_reason - else "stop" - ) - - # Raise on blocked responses before checking parsed content - if ( - not gemini_response.parsed - and finish_reason in GEMINI_BLOCKED_FINISH_REASONS - ): - raise LLMError( - f"Gemini response blocked (finish_reason={finish_reason})", - provider="google", - model=model, - finish_reason=finish_reason, - ) - - # Validate that parsed content matches the response model - if not isinstance(gemini_response.parsed, response_model): - raise ValueError( - f"Parsed content does not match the response model: {gemini_response.parsed} != {response_model}" - ) - - return HonchoLLMCallResponse( - content=gemini_response.parsed, - input_tokens=input_token_count, - output_tokens=output_token_count, - finish_reasons=[finish_reason], - tool_calls_made=[], - ) - - case AsyncGroq(): - groq_params: dict[str, Any] = { - "model": params["model"], - "max_tokens": params["max_tokens"], - "messages": params["messages"], - } - - if temperature is not None: - groq_params["temperature"] = temperature - - if response_model: - groq_params["response_format"] = response_model - elif json_mode: - groq_params["response_format"] = {"type": "json_object"} - - # TODO: figure out why groq returns unknown type and fix it - response: ChatCompletion = await client.chat.completions.create( # pyright: ignore - **groq_params - ) - if response.choices[0].message.content is None: # pyright: ignore - raise ValueError("No content in response") - - # Safely extract usage and finish_reason - usage = response.usage # pyright: ignore - finish_reason = response.choices[0].finish_reason # pyright: ignore - - # Handle response model parsing for Groq - cache_creation, cache_read = extract_openai_cache_tokens(usage) - if response_model: - try: - json_content = json.loads(response.choices[0].message.content) # pyright: ignore - parsed_content = response_model.model_validate(json_content) - - return HonchoLLMCallResponse( - content=parsed_content, - input_tokens=usage.prompt_tokens if usage else 0, # pyright: ignore - output_tokens=usage.completion_tokens if usage else 0, # pyright: ignore - cache_creation_input_tokens=cache_creation, - cache_read_input_tokens=cache_read, - finish_reasons=[finish_reason] if finish_reason else [], - tool_calls_made=[], - ) - except (json.JSONDecodeError, ValidationError, ValueError) as e: - raise ValueError( - f"Failed to parse Groq response as {response_model}: {e}. Raw content: {response.choices[0].message.content}" # pyright: ignore - ) from e - else: - return HonchoLLMCallResponse( - content=response.choices[0].message.content, # pyright: ignore - input_tokens=usage.prompt_tokens if usage else 0, # pyright: ignore - output_tokens=usage.completion_tokens if usage else 0, # pyright: ignore - cache_creation_input_tokens=cache_creation, - cache_read_input_tokens=cache_read, - finish_reasons=[finish_reason] if finish_reason else [], - tool_calls_made=[], - ) - - -async def handle_streaming_response( - client: AsyncAnthropic | AsyncOpenAI | genai.Client | AsyncGroq, - params: dict[str, Any], - json_mode: bool, - thinking_budget_tokens: int | None, - response_model: type[BaseModel] | None = None, - reasoning_effort: Literal["low", "medium", "high", "minimal"] | None = None, - verbosity: Literal["low", "medium", "high"] | None = None, -) -> AsyncIterator[HonchoLLMCallStreamChunk]: - """ - Handle streaming responses for all supported providers. - - Args: - client: The LLM client instance - params: Request parameters including stream=True - json_mode: Whether to use JSON mode - thinking_budget_tokens: Anthropic thinking budget tokens - response_model: Pydantic model for structured output - reasoning_effort: OpenAI reasoning effort level (GPT-5 only) - verbosity: OpenAI verbosity level (GPT-5 only) - - Yields: - HonchoLLMCallStreamChunk: Individual chunks of the streaming response - """ - match client: - case AsyncAnthropic(): - # Anthropic requires system messages as a top-level parameter - messages = params["messages"] - system_content = "\n\n".join( - m["content"] for m in messages if m.get("role") == "system" - ) - anthropic_params: dict[str, Any] = { - "model": params["model"], - "max_tokens": params["max_tokens"], - "messages": [m for m in messages if m.get("role") != "system"], - } - if system_content: - anthropic_params["system"] = [ - { - "type": "text", - "text": system_content, - "cache_control": {"type": "ephemeral"}, - } - ] - - # For response models, we need to request JSON and parse manually - # Note: Streaming with response_model is not ideal but we'll accumulate and parse at the end - if response_model or json_mode: - # Add JSON schema instructions to the prompt if using response_model - if response_model: - schema_json = json.dumps( - response_model.model_json_schema(), indent=2 - ) - anthropic_params["messages"][-1]["content"] += ( - f"\n\nRespond with valid JSON matching this schema:\n{schema_json}" - ) - anthropic_params["messages"].append( - {"role": "assistant", "content": "{"} - ) - - if thinking_budget_tokens: - anthropic_params["thinking"] = { - "type": "enabled", - "budget_tokens": thinking_budget_tokens, - } - - async with client.messages.stream(**anthropic_params) as anthropic_stream: - async for chunk in anthropic_stream: - if ( - chunk.type == "content_block_delta" - and hasattr(chunk, "delta") - and hasattr(chunk.delta, "text") - ): - text_content = getattr(chunk.delta, "text", "") - yield HonchoLLMCallStreamChunk(content=text_content) - final_message = await anthropic_stream.get_final_message() - usage = final_message.usage - output_tokens = usage.output_tokens if usage else None - yield HonchoLLMCallStreamChunk( - content="", - is_done=True, - finish_reasons=[final_message.stop_reason] - if final_message.stop_reason - else [], - output_tokens=output_tokens, - ) - - case AsyncOpenAI(): - openai_params: dict[str, Any] = { - "model": params["model"], - "messages": params["messages"], - "stream": True, - "stream_options": {"include_usage": True}, - } - - model_name = params["model"] - if "gpt-5" in model_name: - openai_params["max_completion_tokens"] = params["max_tokens"] - if reasoning_effort: - openai_params["reasoning_effort"] = reasoning_effort - if verbosity: - openai_params["verbosity"] = verbosity - else: - openai_params["max_tokens"] = params["max_tokens"] - - if response_model: - openai_params["response_format"] = response_model - elif json_mode: - openai_params["response_format"] = {"type": "json_object"} - - openai_stream = await client.chat.completions.create(**openai_params) # pyright: ignore - finish_reason: str | None = None - usage_chunk_received = False - async for chunk in openai_stream: # pyright: ignore - chunk = cast(ChatCompletionChunk, chunk) - if chunk.choices and chunk.choices[0].delta.content: - content = chunk.choices[0].delta.content - yield HonchoLLMCallStreamChunk(content=content) - # Track finish_reason when it appears (before usage chunk) - if chunk.choices and chunk.choices[0].finish_reason: - finish_reason = chunk.choices[0].finish_reason - # Check for usage info in chunk (with include_usage, this is a separate chunk with empty choices) - if hasattr(chunk, "usage") and chunk.usage: - yield HonchoLLMCallStreamChunk( - content="", - is_done=True, - finish_reasons=[finish_reason] if finish_reason else [], - output_tokens=chunk.usage.completion_tokens, - ) - usage_chunk_received = True - - # If stream ended without usage chunk (interrupted), still yield final chunk - if not usage_chunk_received and finish_reason: - logger.warning("OpenAI stream ended without usage chunk (interrupted)") - yield HonchoLLMCallStreamChunk( - content="", - is_done=True, - finish_reasons=[finish_reason], - output_tokens=None, - ) - - case genai.Client(): - prompt_text = params["messages"][0]["content"] if params["messages"] else "" - stream_config: GenerateContentConfigDict = { - "max_output_tokens": cast(int, params["max_tokens"]), - } - - if response_model is not None: - stream_config["response_mime_type"] = "application/json" - stream_config["response_schema"] = response_model - response_stream = await client.aio.models.generate_content_stream( - model=params["model"], - contents=prompt_text, - config=stream_config, - ) - else: - if json_mode: - stream_config["response_mime_type"] = "application/json" - response_stream = await client.aio.models.generate_content_stream( - model=params["model"], - contents=prompt_text, - config=stream_config, - ) - - final_chunk = None - async for chunk in response_stream: - if chunk.text: - yield HonchoLLMCallStreamChunk(content=chunk.text) - final_chunk = chunk - - # NOTE: Blocked-response check is intentionally omitted for streaming. - # Exceptions mid-iteration in an async generator won't be caught by - # the tenacity retry wrapper in honcho_llm_call. - finish_reason = "stop" # Default fallback - gemini_output_tokens: int | None = None - if ( - final_chunk - and hasattr(final_chunk, "candidates") - and final_chunk.candidates - and hasattr(final_chunk.candidates[0], "finish_reason") - and final_chunk.candidates[0].finish_reason - ): - finish_reason = final_chunk.candidates[0].finish_reason.name - - # Extract output tokens from usage_metadata if available - if ( - final_chunk - and hasattr(final_chunk, "usage_metadata") - and final_chunk.usage_metadata - and hasattr(final_chunk.usage_metadata, "candidates_token_count") - ): - gemini_output_tokens = ( - final_chunk.usage_metadata.candidates_token_count or None - ) - - yield HonchoLLMCallStreamChunk( - content="", - is_done=True, - finish_reasons=[finish_reason], - output_tokens=gemini_output_tokens, - ) - - case AsyncGroq(): - groq_params: dict[str, Any] = { - "model": params["model"], - "max_tokens": params["max_tokens"], - "messages": params["messages"], - "stream": True, - } - - if response_model: - groq_params["response_format"] = response_model - elif json_mode: - groq_params["response_format"] = {"type": "json_object"} - - groq_stream = await client.chat.completions.create(**groq_params) # pyright: ignore - async for chunk in groq_stream: # pyright: ignore - chunk = cast(ChatCompletionChunk, chunk) - if chunk.choices and chunk.choices[0].delta.content: - yield HonchoLLMCallStreamChunk( - content=chunk.choices[0].delta.content - ) - if chunk.choices and chunk.choices[0].finish_reason: - yield HonchoLLMCallStreamChunk( - content="", - is_done=True, - finish_reasons=[chunk.choices[0].finish_reason], - ) diff --git a/src/utils/search.py b/src/utils/search.py index 67a0d355..59c8f137 100644 --- a/src/utils/search.py +++ b/src/utils/search.py @@ -382,7 +382,7 @@ async def search( query_embedding = await embedding_client.embed(query) except ValueError as e: raise ValidationException( - f"Query exceeds maximum token limit of {settings.MAX_EMBEDDING_TOKENS}." + f"Query exceeds maximum token limit of {settings.EMBEDDING.MAX_INPUT_TOKENS}." ) from e if not _uses_pgvector_message_search(): diff --git a/src/utils/summarizer.py b/src/utils/summarizer.py index ca1965b9..d964402c 100644 --- a/src/utils/summarizer.py +++ b/src/utils/summarizer.py @@ -11,10 +11,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from src import schemas from src.cache.client import cache as cache_client -from src.config import settings +from src.config import ConfiguredModelSettings, settings from src.crud.session import session_cache_key from src.dependencies import tracked_db from src.exceptions import ResourceNotFoundException +from src.llm import HonchoLLMCallResponse, honcho_llm_call from src.models import Message from src.telemetry import prometheus_metrics from src.telemetry.events import AgentToolSummaryCreatedEvent, emit @@ -24,7 +25,6 @@ from src.telemetry.prometheus.metrics import ( DeriverTaskTypes, TokenTypes, ) -from src.utils.clients import HonchoLLMCallResponse, honcho_llm_call from src.utils.formatting import utc_now_iso from src.utils.tokens import estimate_tokens, track_deriver_input_tokens @@ -78,6 +78,10 @@ __all__ = [ ] +def _get_summary_model_config() -> ConfiguredModelSettings: + return settings.SUMMARY.MODEL_CONFIG + + # Configuration constants for summaries MESSAGES_PER_SHORT_SUMMARY = settings.SUMMARY.MESSAGES_PER_SHORT_SUMMARY MESSAGES_PER_LONG_SUMMARY = settings.SUMMARY.MESSAGES_PER_LONG_SUMMARY @@ -212,7 +216,7 @@ async def create_short_summary( ) return await honcho_llm_call( - llm_settings=settings.SUMMARY, + model_config=_get_summary_model_config(), prompt=prompt, max_tokens=settings.SUMMARY.MAX_TOKENS_SHORT, ) @@ -237,7 +241,7 @@ async def create_long_summary( ) return await honcho_llm_call( - llm_settings=settings.SUMMARY, + model_config=_get_summary_model_config(), prompt=prompt, max_tokens=settings.SUMMARY.MAX_TOKENS_LONG, ) diff --git a/src/utils/types.py b/src/utils/types.py index 0654ed7a..dd66dccf 100644 --- a/src/utils/types.py +++ b/src/utils/types.py @@ -34,7 +34,6 @@ class GetOrCreateResult(Generic[T]): await self.on_commit() -SupportedProviders = Literal["anthropic", "openai", "google", "groq", "custom", "vllm"] TaskType = Literal[ "webhook", "summary", "representation", "dream", "deletion", "reconciler" ] diff --git a/src/vector_store/lancedb.py b/src/vector_store/lancedb.py index 0853a98d..77c3c6cd 100644 --- a/src/vector_store/lancedb.py +++ b/src/vector_store/lancedb.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) _VALID_IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") # Schema for LanceDB tables -# id: string, vector: fixed_size_list of float32 (1536 dimensions for OpenAI embeddings) +# id: string, vector: fixed_size_list of float32 (dimension from embedding settings) # Additional metadata columns are added dynamically # pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownParameterType=false @@ -93,7 +93,7 @@ class LanceDBVectorStore(VectorStore): fields: list[pa.Field] = [ pa.field("id", pa.string()), pa.field( - "vector", pa.list_(pa.float32(), settings.VECTOR_STORE.DIMENSIONS) + "vector", pa.list_(pa.float32(), settings.EMBEDDING.VECTOR_DIMENSIONS) ), ] fields.extend(self._metadata_fields_for_namespace(namespace)) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..7468bd26 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Test package marker for shared helper imports. diff --git a/tests/bench/harness.py b/tests/bench/harness.py index 6f725bf7..90f32e44 100755 --- a/tests/bench/harness.py +++ b/tests/bench/harness.py @@ -648,29 +648,69 @@ sys.path.insert(0, str(project_root)) # and will be inherited by this subprocess try: + from pydantic import BaseModel from src.config import settings - # Function to recursively print settings + SENSITIVE_TOKENS = ('password', 'secret', 'key', 'uri') + + def _mask(full_key, value): + if isinstance(full_key, str) and any(t in full_key.lower() for t in SENSITIVE_TOKENS): + return '*' * len(value) if value else 'None' + return value + + def _compact(model): + # Render a pydantic BaseModel as `field=value` pairs, skipping + # None / empty-dict fields and recursing into nested models. + parts = [] + for field_name in type(model).model_fields: + val = getattr(model, field_name) + if val is None: + continue + if isinstance(val, BaseModel): + inner = _compact(val) + if inner: + parts.append(f"{{field_name}}=({{inner}})") + continue + if isinstance(val, dict) and not val: + continue + parts.append(f"{{field_name}}={{val!r}}") + return " ".join(parts) + def print_settings(obj, prefix="", max_depth=3, current_depth=0): if current_depth >= max_depth: return - if hasattr(obj, '__dict__'): - for key, value in obj.__dict__.items(): - if not key.startswith('_'): - full_key = f"{{prefix}}.{{key}}" if prefix else key - # Handle nested settings objects - if hasattr(value, '__dict__') and not isinstance(value, (str, int, float, bool, type(None))): - print(f"\\n📋 {{full_key}}:") - print_settings(value, full_key, max_depth, current_depth + 1) - else: - # Mask sensitive information - if isinstance(full_key, str) and any(sensitive in full_key.lower() for sensitive in ['password', 'secret', 'key', 'uri']): - masked_value = '*' * len(value) if value else 'None' - else: - masked_value = value - print(f" {{key}}: {{masked_value}}") + if not hasattr(obj, '__dict__'): + return + for key, value in obj.__dict__.items(): + if key.startswith('_'): + continue + full_key = f"{{prefix}}.{{key}}" if prefix else key + + # dict-of-BaseModel → print each entry on its own line compactly + if ( + isinstance(value, dict) and value + and all(isinstance(v, BaseModel) for v in value.values()) + ): + print(f"\\n📋 {{full_key}}:") + for k, v in value.items(): + rendered = _compact(v) + print(f" {{k}}: {{rendered}}") + continue + + if isinstance(value, BaseModel): + print(f"\\n📋 {{full_key}}:") + rendered = _compact(value) + if rendered: + print(f" {{rendered}}") + continue + + if hasattr(value, '__dict__') and not isinstance(value, (str, int, float, bool, type(None))): + print(f"\\n📋 {{full_key}}:") + print_settings(value, full_key, max_depth, current_depth + 1) + continue + + print(f" {{key}}: {{_mask(full_key, value)}}") - # Print all settings print_settings(settings) except Exception as e: diff --git a/tests/conftest.py b/tests/conftest.py index 2ef7086b..cd739b7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,8 +72,17 @@ _RUNTIME_MOCK_TEST_BLOCKLIST_PREFIXES = ( "tests/bench/", "tests/alembic/", "tests/unified/", + "tests/live_llm/", + # Pure llm unit tests should stay isolated from the broader app/runtime fixtures. + "tests/llm/", + # LLM transport tests mock providers directly and don't need database/runtime setup. + "tests/utils/test_length_finish_reason.py", + "tests/utils/test_clients.py", ) +_LIVE_LLM_MARKER = "live_llm" +_LIVE_LLM_SKIP_REASON = "live LLM tests are disabled; pass --live-llm to run them" + def _requires_runtime_mocks(nodeid: str) -> bool: return not any( @@ -87,6 +96,28 @@ def _get_nodeid(request: pytest.FixtureRequest) -> str: return nodeid if isinstance(nodeid, str) else "" +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--live-llm", + action="store_true", + default=False, + help="Run opt-in live LLM integration tests that call provider APIs.", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, + items: list[pytest.Item], +) -> None: + if config.getoption("--live-llm"): + return + + skip_live = pytest.mark.skip(reason=_LIVE_LLM_SKIP_REASON) + for item in items: + if _LIVE_LLM_MARKER in item.keywords: + item.add_marker(skip_live) + + def _get_test_db_url(worker_id: str) -> URL: """Get a worker-specific test database URL for pytest-xdist parallelism.""" @@ -412,9 +443,10 @@ def _content_to_embedding(content: str) -> list[float]: # Hash the content to get a deterministic seed content_hash = hashlib.sha256(content.encode()).digest() - # Use hash bytes to generate 1536 floats between -1 and 1 + vector_dimensions = settings.EMBEDDING.VECTOR_DIMENSIONS + # Use hash bytes to generate deterministic floats between -1 and 1 embedding: list[float] = [] - for i in range(1536): + for i in range(vector_dimensions): # Use different bytes from hash (cycling through) byte_val = content_hash[i % len(content_hash)] # Normalize to [-1, 1] range @@ -431,6 +463,9 @@ def mock_openai_embeddings(request: pytest.FixtureRequest): with ( patch("src.embedding_client.embedding_client.embed") as mock_embed, + patch( + "src.embedding_client.embedding_client.simple_batch_embed" + ) as mock_simple_batch_embed, patch("src.embedding_client.embedding_client.batch_embed") as mock_batch_embed, ): # Mock the embed method to return content-dependent embedding @@ -439,6 +474,11 @@ def mock_openai_embeddings(request: pytest.FixtureRequest): mock_embed.side_effect = embed_side_effect + async def mock_simple_batch_embed_func(texts: list[str]) -> list[list[float]]: + return [_content_to_embedding(text) for text in texts] + + mock_simple_batch_embed.side_effect = mock_simple_batch_embed_func + # Mock the batch_embed method to return content-dependent embeddings async def mock_batch_embed_func( id_resource_dict: dict[str, tuple[str, list[int]]], @@ -450,7 +490,11 @@ def mock_openai_embeddings(request: pytest.FixtureRequest): mock_batch_embed.side_effect = mock_batch_embed_func - yield {"embed": mock_embed, "batch_embed": mock_batch_embed} + yield { + "embed": mock_embed, + "simple_batch_embed": mock_simple_batch_embed, + "batch_embed": mock_batch_embed, + } @pytest.fixture(autouse=True) @@ -670,10 +714,10 @@ def mock_honcho_llm_call(request: pytest.FixtureRequest): # Patch the honcho_llm_call decorator to prevent actual LLM calls at module level original_decorator = None try: - import src.utils.clients + import src.llm - original_decorator = src.utils.clients.honcho_llm_call - src.utils.clients.honcho_llm_call = lambda *args, **kwargs: lambda func: func # pyright: ignore[reportUnknownLambdaType] + original_decorator = src.llm.honcho_llm_call + src.llm.honcho_llm_call = lambda *args, **kwargs: lambda func: func # pyright: ignore[reportUnknownLambdaType] except ImportError: pass @@ -707,21 +751,21 @@ def mock_honcho_llm_call(request: pytest.FixtureRequest): return mock_llm_decorator - with patch("src.utils.clients.honcho_llm_call", side_effect=decorator_factory): + with patch("src.llm.honcho_llm_call", side_effect=decorator_factory): yield decorator_factory # Restore the original decorator if original_decorator: try: - import src.utils.clients + import src.llm - src.utils.clients.honcho_llm_call = original_decorator + src.llm.honcho_llm_call = original_decorator except ImportError: pass @pytest.fixture(autouse=True) -def mock_tracked_db(db_engine: AsyncEngine, request: pytest.FixtureRequest): +def mock_tracked_db(request: pytest.FixtureRequest): """Mock tracked_db to create fresh sessions per call. Using a session factory instead of a shared session avoids asyncio lock @@ -733,6 +777,7 @@ def mock_tracked_db(db_engine: AsyncEngine, request: pytest.FixtureRequest): from contextlib import asynccontextmanager + db_engine = request.getfixturevalue("db_engine") session_factory = async_sessionmaker(bind=db_engine, expire_on_commit=False) @asynccontextmanager diff --git a/tests/deriver/test_deriver_processing.py b/tests/deriver/test_deriver_processing.py index 0cde8a68..5822f4d5 100644 --- a/tests/deriver/test_deriver_processing.py +++ b/tests/deriver/test_deriver_processing.py @@ -1,10 +1,15 @@ import signal +from datetime import datetime, timezone from typing import Any +from unittest.mock import AsyncMock, Mock, patch import pytest from src import models -from src.utils.representation import Representation +from src.config import settings +from src.deriver.deriver import process_representation_tasks_batch +from src.llm import HonchoLLMCallResponse +from src.utils.representation import PromptRepresentation, Representation from src.utils.work_unit import construct_work_unit_key, parse_work_unit_key @@ -12,6 +17,59 @@ from src.utils.work_unit import construct_work_unit_key, parse_work_unit_key class TestDeriverProcessing: """Test suite for deriver processing using the conftest fixtures""" + async def test_process_representation_tasks_batch_uses_model_config(self): + message = Mock( + id=1, + public_id="msg_1", + session_name="session-1", + workspace_name="workspace-1", + peer_name="alice", + content="hello", + token_count=5, + created_at=datetime.now(timezone.utc), + ) + configuration = Mock() + configuration.reasoning.enabled = True + + mock_response = HonchoLLMCallResponse( + content=PromptRepresentation(explicit=[]), + input_tokens=10, + output_tokens=5, + finish_reasons=["STOP"], + ) + + with patch( + "src.deriver.deriver.honcho_llm_call", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_llm_call: + await process_representation_tasks_batch( + messages=[message], + message_level_configuration=configuration, + observers=["bob"], + observed="alice", + queue_item_message_ids=[1], + ) + + await_args = mock_llm_call.await_args + if await_args is None: + raise AssertionError("Expected deriver LLM call") + kwargs = await_args.kwargs + expected_config = settings.DERIVER.MODEL_CONFIG.model_copy( + update={ + "stop_sequences": [" \n", "\n\n\n\n"], + } + ) + assert "model_config" in kwargs + assert kwargs["model_config"].model == expected_config.model + assert kwargs["model_config"].thinking_effort == expected_config.thinking_effort + assert ( + kwargs["model_config"].thinking_budget_tokens + == expected_config.thinking_budget_tokens + ) + assert kwargs["model_config"].stop_sequences == expected_config.stop_sequences + assert "llm_settings" not in kwargs + async def test_work_unit_key_generation( self, sample_session_with_peers: tuple[models.Session, list[models.Peer]], diff --git a/tests/deriver/test_queue_processing.py b/tests/deriver/test_queue_processing.py index 9538c871..540dfab5 100644 --- a/tests/deriver/test_queue_processing.py +++ b/tests/deriver/test_queue_processing.py @@ -1088,8 +1088,16 @@ class TestQueueProcessing: db_session: AsyncSession, sample_session_with_peers: tuple[models.Session, list[models.Peer]], create_queue_payload: Callable[..., Any], + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Test that representation work units below token threshold are not claimed""" + """Test that representation work units below token threshold are not claimed. + + The token-threshold gate in QueueManager.get_and_claim_work_units is + skipped entirely when DERIVER_FLUSH_ENABLED is True, so this test + forces it False regardless of what the process env has set (benches + commonly enable flush mode for immediate processing). + """ + monkeypatch.setattr(settings.DERIVER, "FLUSH_ENABLED", False) session, peers = sample_session_with_peers peer = peers[0] diff --git a/tests/dialectic/test_model_config_usage.py b/tests/dialectic/test_model_config_usage.py new file mode 100644 index 00000000..03b6e9c9 --- /dev/null +++ b/tests/dialectic/test_model_config_usage.py @@ -0,0 +1,111 @@ +import time +from unittest.mock import AsyncMock, patch + +import pytest + +from src.config import settings +from src.dialectic.core import DialecticAgent +from src.llm import ( + HonchoLLMCallResponse, + HonchoLLMCallStreamChunk, + StreamingResponseWithMetadata, +) + + +async def _stream_chunks() -> StreamingResponseWithMetadata: + async def _stream(): + yield HonchoLLMCallStreamChunk(content="streamed") + yield HonchoLLMCallStreamChunk(content="", is_done=True) + + return StreamingResponseWithMetadata( + _stream(), + tool_calls_made=[], + input_tokens=10, + output_tokens=5, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + iterations=1, + ) + + +@pytest.mark.asyncio +async def test_dialectic_answer_uses_level_model_config() -> None: + agent = DialecticAgent( + workspace_name="workspace", + session_name="session", + observer="observer", + observed="observed", + reasoning_level="medium", + ) + + mock_response = HonchoLLMCallResponse( + content="answer", + input_tokens=10, + output_tokens=5, + finish_reasons=["stop"], + ) + + with ( + patch.object( + DialecticAgent, + "_prepare_query", + new=AsyncMock( + return_value=(AsyncMock(), "task", "run", time.perf_counter()) + ), + ), + patch.object(DialecticAgent, "_log_response_metrics"), + patch( + "src.dialectic.core.honcho_llm_call", + new=AsyncMock(return_value=mock_response), + ) as mock_llm_call, + ): + result = await agent.answer("What do you know?") + + await_args = mock_llm_call.await_args + if await_args is None: + raise AssertionError("Expected dialectic LLM call") + kwargs = await_args.kwargs + expected_config = settings.DIALECTIC.LEVELS["medium"].MODEL_CONFIG + + assert result == "answer" + assert kwargs["model_config"] == expected_config + assert "llm_settings" not in kwargs + assert "thinking_budget_tokens" not in kwargs + + +@pytest.mark.asyncio +async def test_dialectic_answer_stream_uses_level_model_config() -> None: + agent = DialecticAgent( + workspace_name="workspace", + session_name="session", + observer="observer", + observed="observed", + reasoning_level="medium", + ) + + with ( + patch.object( + DialecticAgent, + "_prepare_query", + new=AsyncMock( + return_value=(AsyncMock(), "task", "run", time.perf_counter()) + ), + ), + patch.object(DialecticAgent, "_log_response_metrics"), + patch( + "src.dialectic.core.honcho_llm_call", + new=AsyncMock(return_value=await _stream_chunks()), + ) as mock_llm_call, + ): + chunks = [chunk async for chunk in agent.answer_stream("What do you know?")] + + await_args = mock_llm_call.await_args + if await_args is None: + raise AssertionError("Expected dialectic streaming LLM call") + kwargs = await_args.kwargs + expected_config = settings.DIALECTIC.LEVELS["medium"].MODEL_CONFIG + + assert chunks == ["streamed"] + assert kwargs["model_config"] == expected_config + assert "llm_settings" not in kwargs + assert "thinking_budget_tokens" not in kwargs diff --git a/tests/dreamer/test_model_config_usage.py b/tests/dreamer/test_model_config_usage.py new file mode 100644 index 00000000..91d1d141 --- /dev/null +++ b/tests/dreamer/test_model_config_usage.py @@ -0,0 +1,56 @@ +from unittest.mock import AsyncMock, patch + +import pytest + +from src.config import settings +from src.dreamer.specialists import DeductionSpecialist +from src.llm import HonchoLLMCallResponse + + +@pytest.mark.asyncio +async def test_deduction_specialist_uses_nested_model_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings.METRICS, "ENABLED", False) + specialist = DeductionSpecialist() + mock_response = HonchoLLMCallResponse( + content="done", + input_tokens=10, + output_tokens=5, + finish_reasons=["stop"], + ) + + with ( + patch( + "src.dreamer.specialists.crud.get_peer", + new=AsyncMock(), + ), + patch( + "src.dreamer.specialists.crud.get_peer_card", + new=AsyncMock(return_value=None), + ), + patch( + "src.dreamer.specialists.create_tool_executor", + new=AsyncMock(return_value=AsyncMock()), + ), + patch( + "src.dreamer.specialists.honcho_llm_call", + new=AsyncMock(return_value=mock_response), + ) as mock_llm_call, + ): + result = await specialist.run( + workspace_name="workspace", + observer="alice", + observed="alice", + session_name="session", + ) + + await_args = mock_llm_call.await_args + if await_args is None: + raise AssertionError("Expected dreamer LLM call") + kwargs = await_args.kwargs + expected_config = settings.DREAM.DEDUCTION_MODEL_CONFIG + + assert result.content == "done" + assert kwargs["model_config"] == expected_config + assert "llm_settings" not in kwargs diff --git a/tests/integration/test_enqueue.py b/tests/integration/test_enqueue.py index 09bc9eda..0f3d50ad 100644 --- a/tests/integration/test_enqueue.py +++ b/tests/integration/test_enqueue.py @@ -641,68 +641,6 @@ class TestEnqueueFunction: assert observer_who_stayed.name in observers assert sender_peer.name in observers - @pytest.mark.asyncio - async def test_sender_not_in_peer_configuration_uses_defaults( - self, - db_session: AsyncSession, - sample_data: tuple[Workspace, Peer], - ): - """Test get_effective_observe_me handles missing sender configuration gracefully""" - test_workspace, existing_peer = sample_data - - # Create observer peer - observer_peer = models.Peer( - workspace_name=test_workspace.name, name=str(generate_nanoid()) - ) - db_session.add(observer_peer) - - # Create session with only observer (sender not in peers_with_configuration) - test_session = ( - await crud.get_or_create_session( - db_session, - schemas.SessionCreate( - name=str(generate_nanoid()), - peers={ - observer_peer.name: schemas.SessionPeerConfig( - observe_others=True - ), - }, - ), - test_workspace.name, - ) - ).resource - await db_session.commit() - - # Create message from peer NOT in the session configuration - # This simulates the race condition where a peer left after sending - payload = await self.create_sample_payload( - db_session, - workspace_name=test_workspace.name, - session_name=test_session.name, - peer_name=existing_peer.name, - ) - - initial_count = await self.count_queue_items(db_session) - await enqueue(payload) - final_count = await self.count_queue_items(db_session) - - # With deduplication: 1 queue item per message with all observers - assert final_count - initial_count == 1 - - result = await db_session.execute( - select(QueueItem).where(QueueItem.session_id == test_session.id) - ) - queue_items = result.scalars().all() - - assert len(queue_items) == 1 - item = queue_items[0] - assert item.payload.get("task_type") == "representation" - assert item.payload.get("observed") == existing_peer.name - observers = item.payload.get("observers") - assert observers is not None - assert existing_peer.name in observers # self-observation (default) - assert observer_peer.name in observers # observer (observing others) - @pytest.mark.asyncio async def test_mixed_active_inactive_peers_complex_scenario( self, diff --git a/tests/integration/test_message_embeddings.py b/tests/integration/test_message_embeddings.py index ef045049..091e8cf3 100644 --- a/tests/integration/test_message_embeddings.py +++ b/tests/integration/test_message_embeddings.py @@ -478,7 +478,7 @@ async def test_message_chunking_creates_multiple_embeddings( monkeypatch.setattr("src.config.settings.EMBED_MESSAGES", True) # Mock a low token limit to force chunking - monkeypatch.setattr("src.config.settings.MAX_EMBEDDING_TOKENS", 10) + monkeypatch.setattr("src.config.settings.EMBEDDING.MAX_INPUT_TOKENS", 10) test_workspace, test_peer = sample_data diff --git a/tests/integration/test_token_metrics.py b/tests/integration/test_token_metrics.py index 7c24223a..4e6c1652 100644 --- a/tests/integration/test_token_metrics.py +++ b/tests/integration/test_token_metrics.py @@ -19,6 +19,7 @@ from prometheus_client import Counter from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models, schemas +from src.llm import HonchoLLMCallResponse from src.models import Peer, Workspace from src.schemas import ( ResolvedConfiguration, @@ -31,7 +32,6 @@ from src.telemetry.prometheus.metrics import ( deriver_tokens_processed_counter, dialectic_tokens_processed_counter, ) -from src.utils.clients import HonchoLLMCallResponse from src.utils.representation import ExplicitObservationBase, PromptRepresentation from src.utils.summarizer import ( SummaryType, diff --git a/tests/live_llm/README.md b/tests/live_llm/README.md new file mode 100644 index 00000000..cef10802 --- /dev/null +++ b/tests/live_llm/README.md @@ -0,0 +1,59 @@ +# Live LLM Tests + +These tests call real provider APIs and are disabled by default. + +Run them with: + +```bash +uv run pytest tests/live_llm -n 0 --live-llm --no-header -q +``` + +Required API key env vars: + +- `LLM_ANTHROPIC_API_KEY` +- `LLM_OPENAI_API_KEY` +- `LLM_GEMINI_API_KEY` + +Model-family env vars: + +- `LIVE_LLM_ANTHROPIC_45_PLUS_MODELS` +- `LIVE_LLM_OPENAI_GPT4_MODELS` +- `LIVE_LLM_OPENAI_GPT5_MODELS` +- `LIVE_LLM_OPENAI_OPENROUTER_NON_REASONING_MODELS` (OpenAI-transport → OpenRouter-served non-reasoning models) +- `LIVE_LLM_GEMINI_25_MODELS` +- `LIVE_LLM_GEMINI_30_MODELS` +- `LIVE_LLM_GEMINI_31_MODELS` + +Each model env var accepts a comma-separated list of bare model ids or provider-qualified ids. + +Examples: + +```bash +export LIVE_LLM_ANTHROPIC_45_PLUS_MODELS="claude-sonnet-4-5,claude-sonnet-4-6" +export LIVE_LLM_OPENAI_GPT4_MODELS="gpt-4.1" +export LIVE_LLM_OPENAI_GPT5_MODELS="gpt-5,gpt-5.4,gpt-5.4-mini" +export LIVE_LLM_OPENAI_OPENROUTER_NON_REASONING_MODELS="inception/mercury-2" +export LIVE_LLM_GEMINI_25_MODELS="gemini-2.5-flash,gemini-2.5-pro" +export LIVE_LLM_GEMINI_30_MODELS="gemini-3-flash-preview" +export LIVE_LLM_GEMINI_31_MODELS="gemini-3.1-pro-preview" +``` + +OpenRouter-routed models require additional env for the proxy endpoint: + +```bash +export OPENROUTER_API_KEY="sk-or-v1-..." +# Per-feature config example: +# DERIVER_MODEL_CONFIG__TRANSPORT=openai +# DERIVER_MODEL_CONFIG__MODEL=inception/mercury-2 +# DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 +# DERIVER_MODEL_CONFIG__OVERRIDES__API_KEY_ENV=OPENROUTER_API_KEY +``` + +Coverage by provider: + +- Anthropic: structured output path, prompt caching metrics, thinking blocks, multi-turn tool replay +- OpenAI GPT-4 class: structured outputs, prompt caching +- OpenAI GPT-5 class (incl. gpt-5.x point-releases): structured outputs, prompt caching, `reasoning_effort`, `max_completion_tokens` routing +- OpenAI transport → OpenRouter non-reasoning models (e.g. `inception/mercury-2`): non-chat / diffusion architectures must stay on `max_tokens`, no `reasoning_effort`, tool-calling parameter-schema compatibility is the canary for exotic OR-served providers +- Gemini 2.5/3.0 classes: structured outputs, cached-content reuse, thought signatures, multi-turn tool replay +- Gemini 3.1 class: thinking and tool replay coverage by default; structured-output/caching coverage should only be added once Google documents support for that path diff --git a/tests/live_llm/__init__.py b/tests/live_llm/__init__.py new file mode 100644 index 00000000..2c935503 --- /dev/null +++ b/tests/live_llm/__init__.py @@ -0,0 +1 @@ +# Live LLM integration test package. diff --git a/tests/live_llm/conftest.py b/tests/live_llm/conftest.py new file mode 100644 index 00000000..d9646383 --- /dev/null +++ b/tests/live_llm/conftest.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +import pytest +from pydantic import BaseModel + +from src.config import ModelConfig, settings +from src.llm import get_backend +from src.llm.caching import gemini_cache_store + +from .model_matrix import LiveModelSpec, selected_model_summary_lines + + +class StructuredLiveResponse(BaseModel): + provider: str + family: str + answer: str + + +def pytest_report_header(config: pytest.Config) -> list[str] | None: + if not config.getoption("--live-llm"): + return None + return ["live llm model matrix:"] + [ + f" {line}" for line in selected_model_summary_lines() + ] + + +@pytest.fixture(autouse=True) +def clear_live_gemini_cache_store() -> Iterator[None]: + # The live Gemini cache store is process-local and should not leak state between tests. + gemini_cache_store._handles.clear() # pyright: ignore[reportPrivateUsage] + yield + gemini_cache_store._handles.clear() # pyright: ignore[reportPrivateUsage] + + +def require_provider_key(model_spec: LiveModelSpec) -> None: + key_present = { + "anthropic": bool(settings.LLM.ANTHROPIC_API_KEY), + "openai": bool(settings.LLM.OPENAI_API_KEY), + "gemini": bool(settings.LLM.GEMINI_API_KEY), + }[model_spec.provider] + if not key_present: + pytest.skip(f"Missing API key for live provider {model_spec.provider}") + + +def make_model_config(model_spec: LiveModelSpec, **overrides: Any) -> ModelConfig: + return ModelConfig( + model=model_spec.model, + transport=model_spec.provider, + **overrides, + ) + + +def make_backend( + model_spec: LiveModelSpec, **config_overrides: Any +) -> tuple[Any, ModelConfig]: + config = make_model_config(model_spec, **config_overrides) + return get_backend(config), config + + +def make_large_system_prompt(*, label: str) -> str: + repeated_prefix = " ".join([f"{label}-token-{index % 37}" for index in range(2400)]) + return ( + f"{label} system prompt. Reuse this prefix exactly for prompt-caching validation. " + f"{repeated_prefix}" + ) + + +def favorite_prime_tools() -> list[dict[str, Any]]: + return [ + { + "name": "get_favorite_prime", + "description": "Return the favorite prime number for the current test run.", + "input_schema": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "description": "Why the caller wants the prime number.", + } + }, + "required": ["topic"], + }, + } + ] + + +def execute_local_tool(tool_name: str, tool_input: dict[str, Any]) -> str: + assert tool_name == "get_favorite_prime" + assert isinstance(tool_input, dict) + return "13" + + +def wrap_async_method( + monkeypatch: pytest.MonkeyPatch, + target: Any, + attribute: str, +) -> list[dict[str, Any]]: + original = getattr(target, attribute) + calls: list[dict[str, Any]] = [] + + async def wrapped(*args: Any, **kwargs: Any) -> Any: + calls.append({"args": args, "kwargs": kwargs}) + return await original(*args, **kwargs) + + monkeypatch.setattr(target, attribute, wrapped) + return calls + + +def extract_openai_reasoning_tokens(raw_response: Any) -> int | None: + usage = getattr(raw_response, "usage", None) + if usage is None: + return None + details = getattr(usage, "completion_tokens_details", None) + if details is None: + return None + reasoning_tokens = getattr(details, "reasoning_tokens", None) + return int(reasoning_tokens) if reasoning_tokens is not None else None diff --git a/tests/live_llm/model_matrix.py b/tests/live_llm/model_matrix.py new file mode 100644 index 00000000..a9abf0f9 --- /dev/null +++ b/tests/live_llm/model_matrix.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Literal + +ProviderName = Literal["anthropic", "openai", "gemini"] +FeatureName = Literal["thinking", "structured_output", "caching", "reasoning"] + + +@dataclass(frozen=True) +class LiveModelFamily: + provider: ProviderName + family: str + env_var: str + default_models: tuple[str, ...] = () + supports_thinking: bool = False + supports_structured_output: bool = False + supports_caching: bool = False + supports_reasoning: bool = False + supports_tool_replay: bool = False + docs_url: str | None = None + + +@dataclass(frozen=True) +class LiveModelSpec: + provider: ProviderName + family: str + model: str + env_var: str + supports_thinking: bool + supports_structured_output: bool + supports_caching: bool + supports_reasoning: bool + supports_tool_replay: bool + docs_url: str | None = None + + @property + def id(self) -> str: + return f"{self.provider}:{self.family}:{self.model}" + + +MODEL_FAMILIES: tuple[LiveModelFamily, ...] = ( + LiveModelFamily( + provider="anthropic", + family="claude_4_5_plus", + env_var="LIVE_LLM_ANTHROPIC_45_PLUS_MODELS", + supports_thinking=True, + supports_structured_output=True, + supports_caching=True, + supports_tool_replay=True, + docs_url="https://docs.anthropic.com/en/docs/about-claude/models/all-models", + ), + LiveModelFamily( + provider="openai", + family="gpt_4_class", + env_var="LIVE_LLM_OPENAI_GPT4_MODELS", + default_models=("gpt-4.1",), + supports_structured_output=True, + supports_caching=True, + docs_url="https://platform.openai.com/docs/models/gpt-4.1", + ), + LiveModelFamily( + provider="openai", + family="gpt_5_class", + env_var="LIVE_LLM_OPENAI_GPT5_MODELS", + default_models=("gpt-5", "gpt-5.4", "gpt-5.4-mini"), + supports_structured_output=True, + supports_caching=True, + supports_reasoning=True, + docs_url="https://platform.openai.com/docs/models/gpt-5", + ), + # OpenAI-compatible transport → OpenRouter-served non-reasoning models. + # Best canary for operators routing exotic providers through OpenRouter: + # if honcho works here, it works for most OR-served models. Currently + # anchored on Inception Labs' Mercury-2 diffusion model (non-chat + # architecture, must stay on max_tokens, no reasoning_effort). + LiveModelFamily( + provider="openai", + family="openrouter_non_reasoning", + env_var="LIVE_LLM_OPENAI_OPENROUTER_NON_REASONING_MODELS", + default_models=("inception/mercury-2",), + supports_structured_output=False, + supports_caching=False, + docs_url="https://openrouter.ai/models", + ), + LiveModelFamily( + provider="gemini", + family="gemini_2_5_class", + env_var="LIVE_LLM_GEMINI_25_MODELS", + default_models=("gemini-2.5-flash",), + supports_thinking=True, + supports_structured_output=True, + supports_caching=True, + supports_tool_replay=True, + docs_url="https://ai.google.dev/gemini-api/docs/models/gemini", + ), + LiveModelFamily( + provider="gemini", + family="gemini_3_0_class", + env_var="LIVE_LLM_GEMINI_30_MODELS", + supports_thinking=True, + supports_structured_output=True, + supports_caching=True, + supports_tool_replay=True, + docs_url="https://ai.google.dev/gemini-api/docs/models/gemini", + ), + LiveModelFamily( + provider="gemini", + family="gemini_3_1_class", + env_var="LIVE_LLM_GEMINI_31_MODELS", + supports_thinking=True, + supports_structured_output=False, + supports_caching=False, + supports_tool_replay=True, + docs_url="https://ai.google.dev/gemini-api/docs/models/gemini", + ), +) + + +def _parse_env_models(value: str | None) -> tuple[str, ...]: + if value is None: + return () + models = [model.strip() for model in value.split(",")] + return tuple(model for model in models if model) + + +def iter_live_model_specs() -> tuple[LiveModelSpec, ...]: + specs: list[LiveModelSpec] = [] + for family in MODEL_FAMILIES: + configured_models = _parse_env_models(os.getenv(family.env_var)) + models = configured_models or family.default_models + for model in models: + specs.append( + LiveModelSpec( + provider=family.provider, + family=family.family, + model=model, + env_var=family.env_var, + supports_thinking=family.supports_thinking, + supports_structured_output=family.supports_structured_output, + supports_caching=family.supports_caching, + supports_reasoning=family.supports_reasoning, + supports_tool_replay=family.supports_tool_replay, + docs_url=family.docs_url, + ) + ) + return tuple(specs) + + +def get_live_model_specs( + *, + provider: ProviderName | None = None, + feature: FeatureName | None = None, +) -> tuple[LiveModelSpec, ...]: + specs = iter_live_model_specs() + filtered: list[LiveModelSpec] = [] + + for spec in specs: + if provider is not None and spec.provider != provider: + continue + if feature == "thinking" and not spec.supports_thinking: + continue + if feature == "structured_output" and not spec.supports_structured_output: + continue + if feature == "caching" and not spec.supports_caching: + continue + if feature == "reasoning" and not spec.supports_reasoning: + continue + filtered.append(spec) + + return tuple(filtered) + + +def selected_model_summary_lines() -> list[str]: + lines: list[str] = [] + for family in MODEL_FAMILIES: + configured_models = _parse_env_models(os.getenv(family.env_var)) + models = configured_models or family.default_models + joined_models = ", ".join(models) if models else "(none configured)" + lines.append( + f"{family.env_var} [{family.provider}/{family.family}]: {joined_models}" + ) + return lines diff --git a/tests/live_llm/test_live_anthropic.py b/tests/live_llm/test_live_anthropic.py new file mode 100644 index 00000000..e10d2b10 --- /dev/null +++ b/tests/live_llm/test_live_anthropic.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import pytest + +from src.llm.backend import CompletionResult +from src.llm.history_adapters import AnthropicHistoryAdapter +from src.llm.request_builder import execute_completion + +from .conftest import ( + StructuredLiveResponse, + execute_local_tool, + favorite_prime_tools, + make_backend, + make_large_system_prompt, + require_provider_key, + wrap_async_method, +) +from .model_matrix import LiveModelSpec, get_live_model_specs + +pytestmark = [pytest.mark.live_llm, pytest.mark.requires_anthropic] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_spec", + get_live_model_specs(provider="anthropic"), + ids=lambda spec: spec.id, +) +async def test_live_anthropic_structured_output_and_prefix_caching( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec) + create_calls = wrap_async_method(monkeypatch, backend._client.messages, "create") + + messages = [ + { + "role": "system", + "content": make_large_system_prompt(label=f"anthropic-{model_spec.family}"), + }, + { + "role": "user", + "content": ( + "Return valid JSON with provider='anthropic', " + f"family='{model_spec.family}', and answer='cache-ok'." + ), + }, + ] + + results: list[CompletionResult] = [] + for _ in range(3): + results.append( + await execute_completion( + backend, + config, + messages=messages, + max_tokens=256, + response_format=StructuredLiveResponse, + ) + ) + if len(results) >= 2 and results[-1].cache_read_input_tokens > 0: + break + + first = results[0] + later_results = results[1:] + + assert isinstance(first.content, StructuredLiveResponse) + assert first.content.provider == "anthropic" + assert first.content.family == model_spec.family + assert later_results, "Anthropic caching validation requires at least two calls" + for result in later_results: + assert isinstance(result.content, StructuredLiveResponse) + assert any( + result.cache_read_input_tokens > 0 for result in later_results + ), "Anthropic prompt caching did not report a cache hit after repeated identical requests" + + assert len(create_calls) == len(results) + for call in create_calls: + assert call["kwargs"]["system"][0]["cache_control"] == {"type": "ephemeral"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_spec", + get_live_model_specs(provider="anthropic"), + ids=lambda spec: spec.id, +) +async def test_live_anthropic_thinking_and_tool_replay( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec, thinking_budget_tokens=1024) + create_calls = wrap_async_method(monkeypatch, backend._client.messages, "create") + tools = favorite_prime_tools() + adapter = AnthropicHistoryAdapter() + + initial_messages = [ + { + "role": "user", + "content": ( + "Before answering, call the get_favorite_prime tool exactly once. " + "After you receive the tool result, answer in one sentence that includes " + "the number and the word 'prime'." + ), + } + ] + + first = await execute_completion( + backend, + config, + messages=initial_messages, + max_tokens=2048, + tools=tools, + ) + + assert create_calls[0]["kwargs"]["thinking"] == { + "type": "enabled", + "budget_tokens": 1024, + } + assert first.tool_calls, "Anthropic should issue a tool call in the first turn" + assert first.thinking_blocks, "Anthropic thinking blocks should be preserved" + + tool_call = first.tool_calls[0] + tool_result = execute_local_tool(tool_call.name, tool_call.input) + replay_messages = initial_messages + [ + adapter.format_assistant_tool_message(first), + *adapter.format_tool_results( + [ + { + "tool_id": tool_call.id, + "tool_name": tool_call.name, + "result": tool_result, + } + ] + ), + ] + + second = await execute_completion( + backend, + config, + messages=replay_messages, + max_tokens=2048, + tools=tools, + ) + + assert create_calls[1]["kwargs"]["thinking"] == { + "type": "enabled", + "budget_tokens": 1024, + } + assert isinstance(second.content, str) + assert "13" in second.content + assert "prime" in second.content.lower() diff --git a/tests/live_llm/test_live_gemini.py b/tests/live_llm/test_live_gemini.py new file mode 100644 index 00000000..022b6cf0 --- /dev/null +++ b/tests/live_llm/test_live_gemini.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import pytest + +from src.llm.caching import PromptCachePolicy +from src.llm.history_adapters import GeminiHistoryAdapter +from src.llm.request_builder import execute_completion + +from .conftest import ( + StructuredLiveResponse, + execute_local_tool, + favorite_prime_tools, + make_backend, + make_large_system_prompt, + require_provider_key, + wrap_async_method, +) +from .model_matrix import LiveModelSpec, get_live_model_specs + +pytestmark = [pytest.mark.live_llm, pytest.mark.requires_gemini] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_spec", + get_live_model_specs(provider="gemini", feature="structured_output"), + ids=lambda spec: spec.id, +) +async def test_live_gemini_structured_output_and_explicit_cache_reuse( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec, temperature=0) + cache_create_calls = wrap_async_method( + monkeypatch, + backend._client.aio.caches, + "create", + ) + generate_calls = wrap_async_method( + monkeypatch, + backend._client.aio.models, + "generate_content", + ) + cache_policy = PromptCachePolicy(mode="gemini_cached_content", ttl_seconds=300) + + messages = [ + { + "role": "system", + "content": make_large_system_prompt(label=f"gemini-{model_spec.family}"), + }, + { + "role": "user", + "content": ( + "Return valid JSON with provider='gemini', " + f"family='{model_spec.family}', and answer='cache-ok'. " + "Return JSON only, with no prose or markdown." + ), + }, + ] + + first = await execute_completion( + backend, + config, + messages=messages, + max_tokens=512, + response_format=StructuredLiveResponse, + cache_policy=cache_policy, + ) + second = await execute_completion( + backend, + config, + messages=messages, + max_tokens=512, + response_format=StructuredLiveResponse, + cache_policy=cache_policy, + ) + + assert isinstance(first.content, StructuredLiveResponse) + assert first.content.provider == "gemini" + assert first.content.family == model_spec.family + assert isinstance(second.content, StructuredLiveResponse) + + assert len(cache_create_calls) == 1 + assert len(generate_calls) == 2 + first_cached_content = generate_calls[0]["kwargs"]["config"]["cached_content"] + second_cached_content = generate_calls[1]["kwargs"]["config"]["cached_content"] + assert first_cached_content == second_cached_content + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_spec", + get_live_model_specs(provider="gemini", feature="thinking"), + ids=lambda spec: spec.id, +) +async def test_live_gemini_thinking_and_tool_replay( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend( + model_spec, + thinking_budget_tokens=512, + temperature=0, + ) + generate_calls = wrap_async_method( + monkeypatch, + backend._client.aio.models, + "generate_content", + ) + tools = favorite_prime_tools() + adapter = GeminiHistoryAdapter() + + initial_messages = [ + { + "role": "user", + "content": ( + "Before answering, call the get_favorite_prime tool exactly once. " + "Do not answer with plain text on this turn. " + "After the tool result arrives, answer with the exact text " + "'13 is prime.'" + ), + } + ] + + first = await execute_completion( + backend, + config, + messages=initial_messages, + max_tokens=512, + tools=tools, + tool_choice="required", + ) + + assert generate_calls[0]["kwargs"]["config"]["thinking_config"] == { + "thinking_budget": 512, + } + assert first.tool_calls, "Gemini should issue a tool call in the first turn" + assert any( + tool_call.thought_signature for tool_call in first.tool_calls + ), "Gemini tool replay should preserve thought signatures" + + tool_call = first.tool_calls[0] + tool_result = execute_local_tool(tool_call.name, tool_call.input) + replay_messages = initial_messages + [ + adapter.format_assistant_tool_message(first), + *adapter.format_tool_results( + [ + { + "tool_id": tool_call.id, + "tool_name": tool_call.name, + "result": tool_result, + } + ] + ), + ] + + second = await execute_completion( + backend, + config, + messages=replay_messages, + max_tokens=512, + tools=tools, + tool_choice="none", + ) + + assert generate_calls[1]["kwargs"]["config"]["thinking_config"] == { + "thinking_budget": 512, + } + assert isinstance(second.content, str) + assert "13" in second.content + assert "prime" in second.content.lower() diff --git a/tests/live_llm/test_live_openai.py b/tests/live_llm/test_live_openai.py new file mode 100644 index 00000000..60d89161 --- /dev/null +++ b/tests/live_llm/test_live_openai.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import pytest + +from src.llm.request_builder import execute_completion + +from .conftest import ( + StructuredLiveResponse, + make_backend, + make_large_system_prompt, + require_provider_key, + wrap_async_method, +) +from .model_matrix import LiveModelSpec, get_live_model_specs + +pytestmark = [pytest.mark.live_llm, pytest.mark.requires_openai] + +_GPT4_SPECS = tuple( + spec + for spec in get_live_model_specs(provider="openai") + if spec.family == "gpt_4_class" +) +_GPT5_SPECS = tuple( + spec + for spec in get_live_model_specs(provider="openai") + if spec.family == "gpt_5_class" +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_spec", _GPT4_SPECS, ids=lambda spec: spec.id) +async def test_live_openai_gpt4_structured_output_and_prefix_caching( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec) + parse_calls = wrap_async_method( + monkeypatch, + backend._client.chat.completions, + "parse", + ) + + messages = [ + { + "role": "system", + "content": make_large_system_prompt(label=f"openai-{model_spec.family}"), + }, + { + "role": "user", + "content": ( + "Return valid JSON with provider='openai', " + f"family='{model_spec.family}', and answer='cache-ok'." + ), + }, + ] + + first = await execute_completion( + backend, + config, + messages=messages, + max_tokens=256, + response_format=StructuredLiveResponse, + ) + second = await execute_completion( + backend, + config, + messages=messages, + max_tokens=256, + response_format=StructuredLiveResponse, + ) + + assert isinstance(first.content, StructuredLiveResponse) + assert first.content.provider == "openai" + assert first.content.family == model_spec.family + assert isinstance(second.content, StructuredLiveResponse) + assert second.cache_read_input_tokens > 0 + + assert parse_calls[0]["kwargs"]["response_format"] is StructuredLiveResponse + assert "max_tokens" in parse_calls[0]["kwargs"] + assert "max_completion_tokens" not in parse_calls[0]["kwargs"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_spec", _GPT5_SPECS, ids=lambda spec: spec.id) +async def test_live_openai_gpt5_reasoning_structured_output_and_prefix_caching( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec, reasoning_effort="minimal") + parse_calls = wrap_async_method( + monkeypatch, + backend._client.chat.completions, + "parse", + ) + + messages = [ + { + "role": "system", + "content": make_large_system_prompt(label=f"openai-{model_spec.family}"), + }, + { + "role": "user", + "content": ( + "Return valid JSON with provider='openai', " + f"family='{model_spec.family}', and answer='reasoning-ok'." + ), + }, + ] + + first = await execute_completion( + backend, + config, + messages=messages, + max_tokens=1024, + response_format=StructuredLiveResponse, + ) + second = await execute_completion( + backend, + config, + messages=messages, + max_tokens=1024, + response_format=StructuredLiveResponse, + ) + + assert isinstance(first.content, StructuredLiveResponse) + assert first.content.provider == "openai" + assert first.content.family == model_spec.family + assert isinstance(second.content, StructuredLiveResponse) + assert second.cache_read_input_tokens > 0 + + assert parse_calls[0]["kwargs"]["response_format"] is StructuredLiveResponse + assert parse_calls[0]["kwargs"]["reasoning_effort"] == "minimal" + assert "max_completion_tokens" in parse_calls[0]["kwargs"] + assert "max_tokens" not in parse_calls[0]["kwargs"] diff --git a/tests/llm/conftest.py b/tests/llm/conftest.py new file mode 100644 index 00000000..fccccf2d --- /dev/null +++ b/tests/llm/conftest.py @@ -0,0 +1,30 @@ +from collections.abc import AsyncIterator, Iterator +from typing import Any + +import pytest + +from src.llm.backend import CompletionResult, ProviderBackend, StreamChunk + + +class FakeBackend(ProviderBackend): + """Simple backend for request-builder and orchestration tests.""" + + def __init__(self, responses: list[CompletionResult] | None = None) -> None: + self.calls: list[dict[str, Any]] = [] + self._responses: Iterator[CompletionResult] = iter( + responses or [CompletionResult(content="ok")] + ) + + async def complete(self, **kwargs: Any) -> CompletionResult: + self.calls.append(kwargs) + return next(self._responses) + + async def stream(self, **kwargs: Any) -> AsyncIterator[StreamChunk]: + self.calls.append(kwargs) + result = next(self._responses) + yield StreamChunk(content=result.content, is_done=True) + + +@pytest.fixture +def fake_backend() -> FakeBackend: + return FakeBackend() diff --git a/tests/llm/test_agent_tool_schemas.py b/tests/llm/test_agent_tool_schemas.py new file mode 100644 index 00000000..12c12c11 --- /dev/null +++ b/tests/llm/test_agent_tool_schemas.py @@ -0,0 +1,72 @@ +from typing import Any, cast + +from src.utils.agent_tools import ( + DEDUCTION_SPECIALIST_TOOLS, + INDUCTION_SPECIALIST_TOOLS, + TOOLS, +) + + +def _observation_items_schema(tool_key: str) -> dict[str, Any]: + return cast( + dict[str, Any], + TOOLS[tool_key]["input_schema"]["properties"]["observations"]["items"], + ) + + +def test_generic_create_observations_schema_has_level_specific_requirements() -> None: + items = _observation_items_schema("create_observations") + + assert items["additionalProperties"] is False + + level_requirements = { + condition["if"]["properties"]["level"]["const"]: condition["then"]["required"] + for condition in cast(list[dict[str, Any]], items["allOf"]) + } + + assert level_requirements["deductive"] == ["source_ids", "premises"] + assert level_requirements["inductive"] == [ + "source_ids", + "sources", + "pattern_type", + "confidence", + ] + assert level_requirements["contradiction"] == ["source_ids", "sources"] + + +def test_deductive_specialist_tool_requires_evidence_fields() -> None: + items = _observation_items_schema("create_observations_deductive") + + assert TOOLS["create_observations_deductive"]["name"] == ( + "create_observations_deductive" + ) + assert items["required"] == ["content", "source_ids", "premises"] + assert items["properties"]["source_ids"]["minItems"] == 1 + assert items["properties"]["premises"]["minItems"] == 1 + + +def test_inductive_specialist_tool_requires_pattern_fields() -> None: + items = _observation_items_schema("create_observations_inductive") + + assert TOOLS["create_observations_inductive"]["name"] == ( + "create_observations_inductive" + ) + assert items["required"] == [ + "content", + "source_ids", + "sources", + "pattern_type", + "confidence", + ] + assert items["properties"]["source_ids"]["minItems"] == 2 + assert items["properties"]["sources"]["minItems"] == 2 + + +def test_dreamer_specialists_use_level_specific_creation_tools() -> None: + deduction_tool_names = {tool["name"] for tool in DEDUCTION_SPECIALIST_TOOLS} + induction_tool_names = {tool["name"] for tool in INDUCTION_SPECIALIST_TOOLS} + + assert "create_observations_deductive" in deduction_tool_names + assert "create_observations_inductive" in induction_tool_names + assert "create_observations" not in deduction_tool_names + assert "create_observations" not in induction_tool_names diff --git a/tests/llm/test_backends/test_anthropic.py b/tests/llm/test_backends/test_anthropic.py new file mode 100644 index 00000000..c8f2bbdf --- /dev/null +++ b/tests/llm/test_backends/test_anthropic.py @@ -0,0 +1,135 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock +from pydantic import BaseModel + +from src.llm.backends.anthropic import AnthropicBackend + + +@pytest.mark.asyncio +async def test_anthropic_backend_extracts_text_thinking_and_tool_calls() -> None: + client = Mock() + client.messages.create = AsyncMock( + return_value=SimpleNamespace( + content=[ + ThinkingBlock( + type="thinking", + thinking="internal reasoning", + signature="sig_123", + ), + TextBlock(type="text", text="Hello from Anthropic"), + ToolUseBlock( + type="tool_use", + id="tool_1", + name="search", + input={"query": "honcho"}, + ), + ], + usage=SimpleNamespace( + input_tokens=10, + output_tokens=5, + cache_creation_input_tokens=3, + cache_read_input_tokens=2, + ), + stop_reason="tool_use", + ) + ) + + backend = AnthropicBackend(client) + result = await backend.complete( + model="claude-haiku-4-5", + messages=[ + {"role": "system", "content": "System prompt"}, + {"role": "user", "content": "Hello"}, + ], + max_tokens=100, + tools=[ + { + "name": "search", + "description": "Search for information", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + } + ], + thinking_budget_tokens=2048, + tool_choice="required", + ) + + assert result.content == "Hello from Anthropic" + assert result.thinking_content == "internal reasoning" + assert result.thinking_blocks == [ + { + "type": "thinking", + "thinking": "internal reasoning", + "signature": "sig_123", + } + ] + assert result.tool_calls[0].name == "search" + assert result.input_tokens == 15 + assert result.output_tokens == 5 + assert result.finish_reason == "tool_use" + + await_args = client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic client call") + call = await_args.kwargs + assert call["model"] == "claude-haiku-4-5" + assert call["system"][0]["text"] == "System prompt" + assert call["thinking"] == {"type": "enabled", "budget_tokens": 2048} + assert call["tool_choice"] == {"type": "any"} + + +class StructuredResponse(BaseModel): + answer: str + + +@pytest.mark.asyncio +async def test_anthropic_backend_skips_assistant_prefill_for_claude_4_models() -> None: + client = Mock() + client.messages.create = AsyncMock( + return_value=SimpleNamespace( + content=[TextBlock(type="text", text='{"answer":"ok"}')], + usage=SimpleNamespace( + input_tokens=10, + output_tokens=5, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ), + stop_reason="end_turn", + ) + ) + + backend = AnthropicBackend(client) + result = await backend.complete( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=StructuredResponse, + ) + + assert isinstance(result.content, StructuredResponse) + assert result.content.answer == "ok" + await_args = client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic client call") + call = await_args.kwargs + assert len(call["messages"]) == 1 + assert call["messages"][0]["role"] == "user" + assert call["messages"][0]["content"].startswith("Hello\n\nRespond with valid JSON") + + +@pytest.mark.asyncio +async def test_anthropic_backend_rejects_thinking_effort() -> None: + backend = AnthropicBackend(Mock()) + + with pytest.raises(ValueError, match="does not support thinking_effort"): + await backend.complete( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + thinking_effort="high", + ) diff --git a/tests/llm/test_backends/test_gemini.py b/tests/llm/test_backends/test_gemini.py new file mode 100644 index 00000000..b327c8e4 --- /dev/null +++ b/tests/llm/test_backends/test_gemini.py @@ -0,0 +1,391 @@ +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from pydantic import BaseModel + +from src.exceptions import LLMError, ValidationException +from src.llm.backends.gemini import GeminiBackend +from src.llm.caching import PromptCachePolicy, gemini_cache_store + + +@pytest.mark.asyncio +async def test_gemini_backend_preserves_thought_signature() -> None: + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace( + parts=[ + SimpleNamespace(text="Hello from Gemini"), + SimpleNamespace( + function_call=SimpleNamespace( + name="search", + args={"query": "honcho"}, + ), + thought_signature="sig_gemini", + ), + ] + ), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + result = await backend.complete( + model="gemini-2.5-flash", + messages=[ + {"role": "system", "content": "System prompt"}, + {"role": "user", "content": "Hello"}, + ], + max_tokens=100, + thinking_budget_tokens=256, + ) + + assert result.content == "Hello from Gemini" + assert result.tool_calls[0].name == "search" + assert result.tool_calls[0].thought_signature == "sig_gemini" + + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + call = await_args.kwargs + assert call["model"] == "gemini-2.5-flash" + assert call["config"]["system_instruction"] == "System prompt" + assert call["config"]["thinking_config"] == {"thinking_budget": 256} + + +@pytest.mark.asyncio +async def test_gemini_backend_maps_thinking_effort_to_thinking_level() -> None: + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=[SimpleNamespace(text="ok")]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + await backend.complete( + model="gemini-3-pro-preview", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + thinking_effort="low", + ) + + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + call = await_args.kwargs + assert call["config"]["thinking_config"] == {"thinking_level": "low"} + + +@pytest.mark.asyncio +async def test_gemini_backend_rejects_budget_and_effort_together() -> None: + backend = GeminiBackend(Mock()) + + with pytest.raises( + ValidationException, + match="does not support sending both thinking_budget_tokens and thinking_effort", + ): + await backend.complete( + model="gemini-3-pro-preview", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + thinking_budget_tokens=256, + thinking_effort="low", + ) + + +@pytest.mark.asyncio +async def test_gemini_backend_raises_on_blocked_response() -> None: + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="SAFETY"), + content=SimpleNamespace(parts=[]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=0, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + + with pytest.raises(LLMError, match="Gemini response blocked"): + await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + +class StructuredResponse(BaseModel): + answer: str + + +@pytest.mark.asyncio +async def test_gemini_backend_validates_dict_parsed_payload() -> None: + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=[]), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed={"answer": "ok"}, + text=None, + function_calls=None, + ) + ) + + backend = GeminiBackend(client) + result = await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=StructuredResponse, + ) + + assert isinstance(result.content, StructuredResponse) + assert result.content.answer == "ok" + + +@pytest.mark.asyncio +async def test_gemini_backend_falls_back_to_response_text_and_function_calls() -> None: + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace(parts=None), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + text="13 is prime.", + function_calls=[ + SimpleNamespace(name="get_favorite_prime", args={"topic": "test"}) + ], + ) + ) + + backend = GeminiBackend(client) + result = await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + assert result.content == "13 is prime." + assert result.tool_calls[0].name == "get_favorite_prime" + + +@pytest.mark.asyncio +async def test_gemini_backend_ignores_mock_text_and_function_call_placeholders() -> ( + None +): + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=Mock( + candidates=[ + Mock( + finish_reason=SimpleNamespace(name="STOP"), + content=None, + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=0, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + result = await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + assert result.content == "" + assert result.tool_calls == [] + + +@pytest.mark.asyncio +async def test_gemini_backend_strips_system_and_tools_when_using_cached_content() -> ( + None +): + gemini_cache_store._handles.clear() # pyright: ignore[reportPrivateUsage] + client = Mock() + client.aio.caches.create = AsyncMock( + return_value=SimpleNamespace( + name="cachedContents/abc123", + expire_time=datetime.now(timezone.utc) + timedelta(minutes=5), + ) + ) + client.aio.models.generate_content = AsyncMock( + return_value=SimpleNamespace( + candidates=[ + SimpleNamespace( + finish_reason=SimpleNamespace(name="STOP"), + content=SimpleNamespace( + parts=[SimpleNamespace(text="cached result")] + ), + ) + ], + usage_metadata=SimpleNamespace( + prompt_token_count=12, + candidates_token_count=6, + ), + parsed=None, + ) + ) + + backend = GeminiBackend(client) + result = await backend.complete( + model="gemini-2.5-flash", + messages=[ + {"role": "system", "content": "System prompt"}, + {"role": "user", "content": "Hello"}, + ], + max_tokens=100, + tools=[ + { + "name": "search", + "description": "Search for information", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + } + ], + tool_choice="required", + extra_params={ + "cache_policy": PromptCachePolicy( + mode="gemini_cached_content", + ttl_seconds=300, + ) + }, + ) + + assert result.content == "cached result" + await_args = client.aio.models.generate_content.await_args + if await_args is None: + raise AssertionError("Expected Gemini generate_content call") + call = await_args.kwargs + assert call["config"]["cached_content"] == "cachedContents/abc123" + assert "system_instruction" not in call["config"] + assert "tools" not in call["config"] + assert "tool_config" not in call["config"] + + +def test_gemini_sanitize_schema_strips_unsupported_keywords() -> None: + """Gemini's function-declarations validator rejects JSON-Schema keywords + outside its narrow allowlist (additionalProperties, allOf, if/then, $ref, + anyOf, oneOf, patternProperties, ...). _sanitize_schema must strip them + recursively so tool schemas authored for OpenAI/Anthropic don't 400 here. + """ + raw = { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "content": {"type": "string"}, + "level": {"type": "string", "enum": ["a", "b"]}, + }, + "required": ["content"], + "additionalProperties": False, + "allOf": [ + { + "if": {"properties": {"level": {"const": "a"}}}, + "then": {"required": ["aux"]}, + } + ], + }, + }, + }, + "required": ["items"], + "$defs": {"Foo": {"type": "string"}}, + } + cleaned = GeminiBackend._sanitize_schema(raw) # pyright: ignore[reportPrivateUsage] + + # Top-level + assert "additionalProperties" not in cleaned + assert "$defs" not in cleaned + assert cleaned["type"] == "object" + assert cleaned["required"] == ["items"] + + # Nested under items + item_schema = cleaned["properties"]["items"]["items"] + assert "additionalProperties" not in item_schema + assert "allOf" not in item_schema + assert item_schema["properties"]["level"]["enum"] == ["a", "b"] + + +def test_gemini_convert_tools_sanitizes_parameters_schema() -> None: + """End-to-end: feeding a Pydantic/OpenAI-style schema through _convert_tools + must produce a Gemini-safe function_declarations payload.""" + tools = [ + { + "name": "create_observations", + "description": "Create observations.", + "input_schema": { + "type": "object", + "properties": { + "observations": { + "type": "array", + "items": { + "type": "object", + "properties": {"content": {"type": "string"}}, + "additionalProperties": False, + }, + } + }, + "required": ["observations"], + "additionalProperties": False, + }, + } + ] + converted = GeminiBackend._convert_tools(tools) # pyright: ignore[reportPrivateUsage] + params = converted[0]["function_declarations"][0]["parameters"] + assert "additionalProperties" not in params + assert "additionalProperties" not in params["properties"]["observations"]["items"] diff --git a/tests/llm/test_backends/test_openai.py b/tests/llm/test_backends/test_openai.py new file mode 100644 index 00000000..81838202 --- /dev/null +++ b/tests/llm/test_backends/test_openai.py @@ -0,0 +1,276 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from src.exceptions import ValidationException +from src.llm.backends.openai import OpenAIBackend + + +@pytest.mark.asyncio +async def test_openai_backend_uses_gpt5_params_and_extracts_reasoning() -> None: + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="Hello from GPT-5", + tool_calls=[], + reasoning_details=[ + SimpleNamespace( + content="reasoning summary", + model_dump=lambda: { + "type": "reasoning", + "content": "reasoning summary", + }, + ) + ], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=SimpleNamespace(cached_tokens=4), + ), + ) + ) + + backend = OpenAIBackend(client) + result = await backend.complete( + model="gpt-5-mini", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + thinking_effort="high", + ) + + assert result.content == "Hello from GPT-5" + assert result.thinking_content == "reasoning summary" + assert result.reasoning_details == [ + {"type": "reasoning", "content": "reasoning summary"} + ] + assert result.cache_read_input_tokens == 4 + + await_args = client.chat.completions.create.await_args + if await_args is None: + raise AssertionError("Expected OpenAI create call") + call = await_args.kwargs + assert call["model"] == "gpt-5-mini" + assert call["max_completion_tokens"] == 100 + assert call["reasoning_effort"] == "high" + assert "max_tokens" not in call + + +@pytest.mark.asyncio +async def test_openai_backend_passes_thinking_effort_through_for_non_gpt5_models() -> ( + None +): + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="Hello from GPT-4.1", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + thinking_effort="low", + ) + + await_args = client.chat.completions.create.await_args + if await_args is None: + raise AssertionError("Expected OpenAI create call") + call = await_args.kwargs + assert call["model"] == "gpt-4.1" + assert call["max_tokens"] == 100 + assert call["reasoning_effort"] == "low" + + +@pytest.mark.asyncio +async def test_openai_backend_does_not_treat_proxy_models_with_gpt5_substring_as_gpt5() -> ( + None +): + """Regression: proxy/deployment names containing 'gpt-5' must use `max_tokens`. + + Flexible OpenAI-compatible configuration means operators commonly route through + proxies/Azure deployments with IDs like `azure-gpt-5-deployment` or + `my-gpt-5-proxy`. A naive substring check would incorrectly send + `max_completion_tokens` (a GPT-5-only parameter) to those endpoints. + """ + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="ok", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="my-gpt-5-proxy", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + await_args = client.chat.completions.create.await_args + if await_args is None: + raise AssertionError("Expected OpenAI create call") + call = await_args.kwargs + assert call["max_tokens"] == 100 + assert "max_completion_tokens" not in call + + +@pytest.mark.asyncio +async def test_openai_backend_rejects_thinking_budget_tokens() -> None: + backend = OpenAIBackend(Mock()) + + with pytest.raises( + ValidationException, match="does not support thinking_budget_tokens" + ): + await backend.complete( + model="gpt-5-mini", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + thinking_budget_tokens=256, + ) + + +@pytest.mark.asyncio +async def test_openai_backend_converts_anthropic_style_tools() -> None: + client = Mock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace( + content="Used tools", + tool_calls=[], + reasoning_details=[], + ), + ) + ], + usage=SimpleNamespace( + prompt_tokens=10, + completion_tokens=5, + prompt_tokens_details=None, + ), + ) + ) + + backend = OpenAIBackend(client) + await backend.complete( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + tools=[ + { + "name": "get_weather", + "description": "Lookup weather", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], + tool_choice="required", + ) + + await_args = client.chat.completions.create.await_args + if await_args is None: + raise AssertionError("Expected OpenAI create call") + call = await_args.kwargs + assert call["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Lookup weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + assert call["tool_choice"] == "required" + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5", + "gpt-5-turbo", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.5-preview", + "o1", + "o1-mini", + "o3", + "o3-mini", + "o4-preview", + ], +) +def test_openai_reasoning_models_use_max_completion_tokens(model: str) -> None: + """Reasoning model families (gpt-5 incl. x.y versions, o1/o3/o4) must send + max_completion_tokens, not max_tokens — OpenAI rejects max_tokens for them + with 400 unsupported_parameter.""" + from src.llm.backends.openai import ( + _uses_max_completion_tokens, # pyright: ignore[reportPrivateUsage] + ) + + assert _uses_max_completion_tokens(model) is True + + +@pytest.mark.parametrize( + "model", + [ + "gpt-4.1", + "gpt-4o", + "gpt-4o-mini", + "gpt-3.5-turbo", + "some-proxy-model", + ], +) +def test_openai_classic_models_use_max_tokens(model: str) -> None: + """Non-reasoning OpenAI and OpenAI-compatible proxy models stay on + the classic max_tokens parameter.""" + from src.llm.backends.openai import ( + _uses_max_completion_tokens, # pyright: ignore[reportPrivateUsage] + ) + + assert _uses_max_completion_tokens(model) is False diff --git a/tests/llm/test_conversation.py b/tests/llm/test_conversation.py new file mode 100644 index 00000000..324803ce --- /dev/null +++ b/tests/llm/test_conversation.py @@ -0,0 +1,103 @@ +from typing import Any + +from src.llm.conversation import ( + _is_tool_result_message, # pyright: ignore[reportPrivateUsage] + _is_tool_use_message, # pyright: ignore[reportPrivateUsage] + truncate_messages_to_fit, +) + + +def test_truncate_messages_to_fit_keeps_last_unit_when_over_limit() -> None: + messages = [ + {"role": "user", "content": "x " * 2000}, + ] + + truncated = truncate_messages_to_fit(messages, max_tokens=1) + + assert truncated == messages + + +def test_truncate_messages_to_fit_preserves_tool_result_pair() -> None: + messages = [ + {"role": "user", "content": "old context " * 1000}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + ] + + truncated = truncate_messages_to_fit(messages, max_tokens=5) + + assert truncated == messages[1:] + + +def test_is_tool_use_message_detects_gemini_function_call_in_parts() -> None: + msg: dict[str, Any] = { + "role": "model", + "parts": [ + {"function_call": {"name": "search", "args": {"q": "honcho"}}}, + ], + } + assert _is_tool_use_message(msg) is True + + +def test_is_tool_result_message_detects_gemini_function_response_in_parts() -> None: + msg: dict[str, Any] = { + "role": "user", + "parts": [ + {"function_response": {"name": "search", "response": {"result": "ok"}}}, + ], + } + assert _is_tool_result_message(msg) is True + + +def test_is_tool_use_message_detects_anthropic_tool_use_block() -> None: + msg: dict[str, Any] = { + "role": "assistant", + "content": [ + {"type": "text", "text": "calling lookup"}, + {"type": "tool_use", "id": "t_1", "name": "lookup", "input": {}}, + ], + } + assert _is_tool_use_message(msg) is True + + +def test_truncate_messages_to_fit_preserves_gemini_tool_pair() -> None: + """A Gemini-shaped function_call / function_response pair must stay + grouped when older units get dropped. Regression: before adding the + parts-based detection, neither message would be recognized as a tool + unit, and truncation could split or drop them individually.""" + messages: list[dict[str, Any]] = [ + {"role": "user", "parts": [{"text": "old context " * 1000}]}, + { + "role": "model", + "parts": [ + {"function_call": {"name": "lookup", "args": {}}}, + ], + }, + { + "role": "user", + "parts": [ + { + "function_response": { + "name": "lookup", + "response": {"result": "found"}, + } + } + ], + }, + ] + + truncated = truncate_messages_to_fit(messages, max_tokens=20) + + # The oldest (bulk-text) message should be dropped; the function_call + + # function_response pair stays intact together. + assert truncated == messages[1:] diff --git a/tests/llm/test_credentials.py b/tests/llm/test_credentials.py new file mode 100644 index 00000000..fae9a8cf --- /dev/null +++ b/tests/llm/test_credentials.py @@ -0,0 +1,50 @@ +import pytest + +from src.config import ModelConfig, settings +from src.llm.credentials import resolve_credentials + + +def test_transport_credentials_use_global_settings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings.LLM, "ANTHROPIC_API_KEY", "anthropic-test-key") + + credentials = resolve_credentials( + ModelConfig(model="claude-haiku-4-5", transport="anthropic") + ) + + assert credentials == {"api_key": "anthropic-test-key", "api_base": None} + + +def test_openai_transport_credentials_use_per_model_config() -> None: + credentials = resolve_credentials( + ModelConfig( + model="my-local-model", + transport="openai", + api_key="local-key", + base_url="http://localhost:8000/v1", + ) + ) + + assert credentials == { + "api_key": "local-key", + "api_base": "http://localhost:8000/v1", + } + + +def test_openai_transport_credentials_fall_back_to_global_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings.LLM, "OPENAI_API_KEY", "openai-test-key") + + credentials = resolve_credentials( + ModelConfig( + model="my-local-model", + transport="openai", + ) + ) + + assert credentials == { + "api_key": "openai-test-key", + "api_base": None, + } diff --git a/tests/llm/test_embedding_client.py b/tests/llm/test_embedding_client.py new file mode 100644 index 00000000..463b3dd6 --- /dev/null +++ b/tests/llm/test_embedding_client.py @@ -0,0 +1,139 @@ +from types import SimpleNamespace +from typing import Any + +import pytest + +from src.config import EmbeddingModelConfig +from src.embedding_client import _EmbeddingClient # pyright: ignore[reportPrivateUsage] + + +class FakeOpenAIEmbeddingsAPI: + def __init__(self, embedding: list[float]) -> None: + self.embedding: list[float] = embedding + self.calls: list[dict[str, Any]] = [] + + async def create(self, *, model: str, input: str | list[str]) -> SimpleNamespace: + self.calls.append({"model": model, "input": input}) + if isinstance(input, list): + data = [SimpleNamespace(embedding=self.embedding) for _ in input] + else: + data = [SimpleNamespace(embedding=self.embedding)] + return SimpleNamespace(data=data) + + +@pytest.mark.asyncio +async def test_openai_embedding_client_uses_configured_model_and_dimensions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 8) + + class FakeOpenAIClient: + def __init__(self, *, api_key: str | None, base_url: str | None) -> None: + self.api_key: str | None = api_key + self.base_url: str | None = base_url + self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings + + monkeypatch.setattr("src.embedding_client.AsyncOpenAI", FakeOpenAIClient) + + client = _EmbeddingClient( + EmbeddingModelConfig( + transport="openai", + model="text-embedding-3-small", + api_key="test-key", + base_url="http://localhost:8000/v1", + ), + vector_dimensions=8, + max_input_tokens=8192, + max_tokens_per_request=300_000, + ) + + embedding = await client.embed("hello world") + + assert embedding == [0.1] * 8 + assert fake_embeddings.calls == [ + {"model": "text-embedding-3-small", "input": "hello world"} + ] + + +@pytest.mark.asyncio +async def test_openai_embedding_client_rejects_dimension_mismatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 7) + + class FakeOpenAIClient: + def __init__(self, *, api_key: str | None, base_url: str | None) -> None: + self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings + + monkeypatch.setattr("src.embedding_client.AsyncOpenAI", FakeOpenAIClient) + + client = _EmbeddingClient( + EmbeddingModelConfig( + transport="openai", + model="text-embedding-3-small", + api_key="test-key", + ), + vector_dimensions=8, + max_input_tokens=8192, + max_tokens_per_request=300_000, + ) + + with pytest.raises(ValueError, match="Embedding dimension mismatch"): + await client.embed("hello world") + + +@pytest.mark.asyncio +async def test_gemini_embedding_client_uses_output_dimensionality( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + + class FakeGeminiModels: + async def embed_content( + self, + *, + model: str, + contents: str | list[str], + config: dict[str, Any], + ) -> SimpleNamespace: + calls.append( + { + "model": model, + "contents": contents, + "config": config, + } + ) + return SimpleNamespace( + embeddings=[SimpleNamespace(values=[0.2] * 12)], + ) + + class FakeGeminiClient: + def __init__(self, *, api_key: str | None, http_options: Any) -> None: + self.api_key: str | None = api_key + self.http_options: Any = http_options + self.aio: Any = SimpleNamespace(models=FakeGeminiModels()) + + monkeypatch.setattr("src.embedding_client.genai.Client", FakeGeminiClient) + + client = _EmbeddingClient( + EmbeddingModelConfig( + transport="gemini", + model="gemini-embedding-001", + api_key="gemini-key", + base_url="https://gemini-proxy.example/v1beta", + ), + vector_dimensions=12, + max_input_tokens=4096, + max_tokens_per_request=300_000, + ) + + embedding = await client.embed("hello world") + + assert embedding == [0.2] * 12 + assert calls == [ + { + "model": "gemini-embedding-001", + "contents": "hello world", + "config": {"output_dimensionality": 12}, + } + ] diff --git a/tests/llm/test_history_adapters.py b/tests/llm/test_history_adapters.py new file mode 100644 index 00000000..6881df6a --- /dev/null +++ b/tests/llm/test_history_adapters.py @@ -0,0 +1,67 @@ +from src.llm.backend import CompletionResult, ToolCallResult +from src.llm.history_adapters import ( + AnthropicHistoryAdapter, + GeminiHistoryAdapter, + OpenAIHistoryAdapter, +) + + +def test_anthropic_history_adapter_preserves_thinking_blocks() -> None: + adapter = AnthropicHistoryAdapter() + result = CompletionResult( + content="Done", + thinking_blocks=[ + { + "type": "thinking", + "thinking": "private reasoning", + "signature": "sig_123", + } + ], + tool_calls=[ + ToolCallResult(id="tool_1", name="search", input={"query": "honcho"}) + ], + ) + + message = adapter.format_assistant_tool_message(result) + + assert message["role"] == "assistant" + assert message["content"][0]["type"] == "thinking" + assert message["content"][1] == {"type": "text", "text": "Done"} + assert message["content"][2]["type"] == "tool_use" + + +def test_gemini_history_adapter_preserves_thought_signature() -> None: + adapter = GeminiHistoryAdapter() + result = CompletionResult( + content="Calling a tool", + tool_calls=[ + ToolCallResult( + id="tool_1", + name="search", + input={"query": "honcho"}, + thought_signature="sig_abc", + ) + ], + ) + + message = adapter.format_assistant_tool_message(result) + + assert message["role"] == "model" + assert message["parts"][1]["thought_signature"] == "sig_abc" + + +def test_openai_history_adapter_preserves_reasoning_details() -> None: + adapter = OpenAIHistoryAdapter() + result = CompletionResult( + content="Calling a tool", + reasoning_details=[{"type": "reasoning", "content": "step 1"}], + tool_calls=[ + ToolCallResult(id="tool_1", name="search", input={"query": "honcho"}) + ], + ) + + message = adapter.format_assistant_tool_message(result) + + assert message["role"] == "assistant" + assert message["reasoning_details"] == [{"type": "reasoning", "content": "step 1"}] + assert message["tool_calls"][0]["function"]["name"] == "search" diff --git a/tests/llm/test_model_config.py b/tests/llm/test_model_config.py new file mode 100644 index 00000000..20d4aa7c --- /dev/null +++ b/tests/llm/test_model_config.py @@ -0,0 +1,510 @@ +import os +import re +from pathlib import Path +from typing import Any, cast + +import pytest + +from src.config import ( + AppSettings, + ConfiguredEmbeddingModelSettings, + ConfiguredModelSettings, + DialecticLevelSettings, + DreamSettings, + EmbeddingSettings, + ModelConfig, + ModelOverrideSettings, + SummarySettings, + VectorStoreSettings, + load_toml_config, + resolve_embedding_model_config, + resolve_model_config, +) + + +def test_fallback_config_is_independent() -> None: + """Fallback config has its own transport and reasoning params.""" + from src.config import ResolvedFallbackConfig + + config = ModelConfig( + model="claude-haiku-4-5", + transport="anthropic", + thinking_budget_tokens=1024, + fallback=ResolvedFallbackConfig( + model="gpt-4.1-mini", + transport="openai", + base_url="https://example.com/v1", + ), + ) + assert config.fallback is not None + assert config.fallback.transport == "openai" + assert config.fallback.thinking_budget_tokens is None + assert config.fallback.base_url == "https://example.com/v1" + + +def test_base_url_is_allowed_for_any_transport() -> None: + config = ModelConfig( + model="claude-haiku-4-5", + transport="anthropic", + base_url="https://anthropic-proxy.example/v1", + ) + + assert config.base_url == "https://anthropic-proxy.example/v1" + + +def test_anthropic_thinking_budget_has_minimum() -> None: + with pytest.raises(ValueError, match="thinking_budget_tokens must be >= 1024"): + ModelConfig( + model="claude-haiku-4-5", + transport="anthropic", + thinking_budget_tokens=512, + ) + + +def test_reasoning_effort_alias_populates_generic_thinking_effort() -> None: + config = ModelConfig.model_validate( + { + "model": "gpt-5", + "transport": "openai", + "reasoning_effort": "minimal", + } + ) + + assert config.thinking_effort == "minimal" + assert config.reasoning_effort == "minimal" + + +def test_for_model_overrides_model_and_transport() -> None: + config = ModelConfig( + model="claude-haiku-4-5", + transport="anthropic", + ) + + updated = config.for_model( + "gpt-5-mini", + transport_override="openai", + ) + + assert updated.model == "gpt-5-mini" + assert updated.transport == "openai" + assert config.transport == "anthropic" + + +def test_configured_model_settings_validate_like_runtime_model_config() -> None: + with pytest.raises(ValueError, match="thinking_budget_tokens must be >= 1024"): + ConfiguredModelSettings( + model="claude-haiku-4-5", + transport="anthropic", + thinking_budget_tokens=512, + ) + + +def test_summary_settings_accept_nested_model_config() -> None: + from src.config import FallbackModelSettings + + settings = SummarySettings( + MODEL_CONFIG=ConfiguredModelSettings( + model="claude-haiku-4-5", + transport="anthropic", + fallback=FallbackModelSettings( + model="gemini-2.5-pro", + transport="gemini", + ), + thinking_budget_tokens=1024, + ), + ) + + assert settings.MODEL_CONFIG.model == "claude-haiku-4-5" + assert settings.MODEL_CONFIG.transport == "anthropic" + assert settings.MODEL_CONFIG.fallback is not None + assert settings.MODEL_CONFIG.fallback.model == "gemini-2.5-pro" + assert settings.MODEL_CONFIG.fallback.transport == "gemini" + assert settings.MODEL_CONFIG.thinking_budget_tokens == 1024 + + +def test_resolve_model_config_reads_override_env_and_provider_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SUMMARY_LOCAL_API_KEY", "test-key") + + configured = ConfiguredModelSettings( + model="my-local-model", + transport="openai", + overrides=ModelOverrideSettings( + api_key_env="SUMMARY_LOCAL_API_KEY", + base_url="http://localhost:8000/v1", + provider_params={"verbosity": "low"}, + ), + ) + + resolved = resolve_model_config(configured) + + assert resolved.api_key == "test-key" + assert resolved.base_url == "http://localhost:8000/v1" + assert resolved.provider_params == {"verbosity": "low"} + + +def test_resolve_embedding_model_config_reads_override_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("EMBEDDING_LOCAL_API_KEY", "embed-key") + + configured = ConfiguredEmbeddingModelSettings( + transport="openai", + model="text-embedding-3-small", + overrides=ModelOverrideSettings( + api_key_env="EMBEDDING_LOCAL_API_KEY", + base_url="http://localhost:8000/v1", + ), + ) + + resolved = resolve_embedding_model_config(configured) + + assert resolved.api_key == "embed-key" + assert resolved.base_url == "http://localhost:8000/v1" + + +def test_dialectic_level_settings_accepts_nested_model_config() -> None: + from src.config import FallbackModelSettings + + settings = DialecticLevelSettings( + MODEL_CONFIG=ConfiguredModelSettings( + model="claude-haiku-4-5", + transport="anthropic", + fallback=FallbackModelSettings( + model="gemini-2.5-pro", + transport="gemini", + ), + thinking_budget_tokens=1024, + ), + MAX_TOOL_ITERATIONS=2, + ) + + resolved = resolve_model_config(settings.MODEL_CONFIG) + assert resolved.model == "claude-haiku-4-5" + assert resolved.transport == "anthropic" + assert resolved.fallback is not None + assert resolved.fallback.model == "gemini-2.5-pro" + assert resolved.fallback.transport == "gemini" + + +def test_dialectic_level_settings_require_nested_model_config() -> None: + with pytest.raises(ValueError, match="Field required"): + DialecticLevelSettings.model_validate({"MAX_TOOL_ITERATIONS": 2}) + + +def test_dialectic_level_settings_reject_legacy_flat_model_shape() -> None: + with pytest.raises(ValueError, match="Field required"): + DialecticLevelSettings.model_validate( + { + "MODEL": "claude-haiku-4-5", + "THINKING_BUDGET_TOKENS": 1024, + "MAX_TOOL_ITERATIONS": 2, + } + ) + + +def test_legacy_prefixed_model_strings_are_normalized() -> None: + config = ModelConfig.model_validate({"model": "gemini/gemini-2.5-flash"}) + configured = ConfiguredModelSettings.model_validate( + {"model": "anthropic/claude-haiku-4-5"} + ) + + assert config.transport == "gemini" + assert config.model == "gemini-2.5-flash" + assert configured.transport == "anthropic" + assert configured.model == "claude-haiku-4-5" + + +def test_dream_specialist_model_configs_are_independent() -> None: + """Specialist configs carry their own defaults and don't inherit from a parent.""" + + dream = DreamSettings( + DEDUCTION_MODEL_CONFIG=ConfiguredModelSettings( + model="claude-haiku-4-5", + transport="anthropic", + thinking_budget_tokens=2048, + ), + INDUCTION_MODEL_CONFIG=ConfiguredModelSettings( + model="claude-opus-4-1", + transport="anthropic", + max_output_tokens=8000, + ), + ) + + assert dream.DEDUCTION_MODEL_CONFIG.model == "claude-haiku-4-5" + assert dream.DEDUCTION_MODEL_CONFIG.thinking_budget_tokens == 2048 + assert dream.DEDUCTION_MODEL_CONFIG.max_output_tokens is None + + assert dream.INDUCTION_MODEL_CONFIG.model == "claude-opus-4-1" + assert dream.INDUCTION_MODEL_CONFIG.max_output_tokens == 8000 + assert dream.INDUCTION_MODEL_CONFIG.thinking_budget_tokens is None + + +def test_app_settings_propagate_embedding_dimensions_to_vector_store() -> None: + settings = AppSettings( + EMBEDDING=EmbeddingSettings(VECTOR_DIMENSIONS=2048), + VECTOR_STORE=VectorStoreSettings(TYPE="lancedb", MIGRATED=True), + ) + + assert settings.EMBEDDING.VECTOR_DIMENSIONS == 2048 + assert settings.VECTOR_STORE.DIMENSIONS == 2048 + + +def test_app_settings_require_matching_embedding_and_vector_store_dimensions() -> None: + with pytest.raises( + ValueError, + match=re.escape( + "VECTOR_STORE.DIMENSIONS must match EMBEDDING.VECTOR_DIMENSIONS" + ), + ): + AppSettings( + EMBEDDING=EmbeddingSettings(VECTOR_DIMENSIONS=2048), + VECTOR_STORE=VectorStoreSettings( + TYPE="lancedb", + MIGRATED=True, + DIMENSIONS=1536, + ), + ) + + +def test_app_settings_reject_non_1536_dimensions_while_pgvector_or_dual_write_active() -> ( + None +): + with pytest.raises( + ValueError, + match=re.escape("EMBEDDING.VECTOR_DIMENSIONS must remain 1536"), + ): + AppSettings( + EMBEDDING=EmbeddingSettings(VECTOR_DIMENSIONS=2048), + VECTOR_STORE=VectorStoreSettings(TYPE="pgvector", MIGRATED=True), + ) + + with pytest.raises( + ValueError, + match=re.escape("EMBEDDING.VECTOR_DIMENSIONS must remain 1536"), + ): + AppSettings( + EMBEDDING=EmbeddingSettings(VECTOR_DIMENSIONS=2048), + VECTOR_STORE=VectorStoreSettings(TYPE="lancedb", MIGRATED=False), + ) + + +def test_config_toml_example_uses_nested_model_config_sections() -> None: + config_path = Path(__file__).resolve().parents[2] / "config.toml.example" + config_data = load_toml_config(str(config_path)) + + deriver_config = ConfiguredModelSettings.model_validate( + config_data["deriver"]["model_config"] + ) + minimal_level = DialecticLevelSettings.model_validate( + config_data["dialectic"]["levels"]["minimal"] + ) + max_level = DialecticLevelSettings.model_validate( + config_data["dialectic"]["levels"]["max"] + ) + embedding_config = ConfiguredEmbeddingModelSettings.model_validate( + config_data["embedding"]["model_config"] + ) + summary_config = ConfiguredModelSettings.model_validate( + config_data["summary"]["model_config"] + ) + deduction_model_config = ConfiguredModelSettings.model_validate( + config_data["dream"]["deduction_model_config"] + ) + induction_model_config = ConfiguredModelSettings.model_validate( + config_data["dream"]["induction_model_config"] + ) + dream = DreamSettings.model_validate( + { + "DEDUCTION_MODEL_CONFIG": deduction_model_config, + "INDUCTION_MODEL_CONFIG": induction_model_config, + } + ) + + # config.toml.example ships the same minimal defaults the app uses: + # transport=openai, model=gpt-5.4-mini across every text-generation + # feature, with embeddings on openai/text-embedding-3-small. Asserting + # these keeps the example file and the in-code defaults in lockstep. + assert deriver_config.transport == "openai" + assert deriver_config.model == "gpt-5.4-mini" + assert deriver_config.thinking_budget_tokens is None + assert minimal_level.MODEL_CONFIG.model == "gpt-5.4-mini" + assert minimal_level.MODEL_CONFIG.transport == "openai" + assert max_level.MODEL_CONFIG.model == "gpt-5.4-mini" + assert max_level.MODEL_CONFIG.transport == "openai" + assert max_level.MODEL_CONFIG.thinking_budget_tokens is None + assert embedding_config.transport == "openai" + assert embedding_config.model == "text-embedding-3-small" + assert summary_config.model == "gpt-5.4-mini" + assert summary_config.transport == "openai" + assert dream.DEDUCTION_MODEL_CONFIG.model == "gpt-5.4-mini" + assert dream.INDUCTION_MODEL_CONFIG.model == "gpt-5.4-mini" + + +def test_env_template_uses_nested_model_config_keys() -> None: + env_template_path = Path(__file__).resolve().parents[2] / ".env.template" + env_template = env_template_path.read_text() + + assert "EMBEDDING_MODEL_CONFIG__MODEL" in env_template + assert "EMBEDDING_VECTOR_DIMENSIONS" in env_template + assert "DERIVER_MODEL_CONFIG__MODEL" in env_template + assert "DIALECTIC_LEVELS__minimal__MODEL_CONFIG__MODEL" in env_template + assert "SUMMARY_MODEL_CONFIG__MODEL" in env_template + assert "DREAM_DEDUCTION_MODEL_CONFIG__MODEL" in env_template + + assert "DERIVER_PROVIDER=" not in env_template + assert "SUMMARY_PROVIDER=" not in env_template + assert "DIALECTIC_LEVELS__minimal__PROVIDER=" not in env_template + assert "DREAM_PROVIDER=" not in env_template + assert "DREAM_DEDUCTION_MODEL=" not in env_template + + +def _clear_deriver_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Strip any DERIVER_MODEL_CONFIG__* env that would interfere with + direct-construction tests.""" + for name in list(os.environ): + if name.startswith("DERIVER_MODEL_CONFIG__") or name == "DERIVER_MODEL_CONFIG": + monkeypatch.delenv(name, raising=False) + + +def test_partial_env_override_of_transport_drops_default_thinking_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A partial env override of transport must not leak the default's thinking + params into a transport that rejects them. + + Regression: setting DERIVER_MODEL_CONFIG__TRANSPORT=openai + + DERIVER_MODEL_CONFIG__MODEL=gpt-4.1-mini (without clearing the default + thinking_budget_tokens=1024 carried over from the gemini default) used to + produce a merged ConfiguredModelSettings with thinking_budget_tokens=1024, + which the OpenAI backend then rejected at call time. + """ + from src.config import DeriverSettings + + _clear_deriver_env(monkeypatch) + # Exercise the @model_validator(mode="before") merge path with a raw dict + # — pyright can't see through the pre-validator that accepts dict input. + settings = DeriverSettings( + MODEL_CONFIG={"transport": "openai", "model": "gpt-4.1-mini"}, # pyright: ignore[reportArgumentType] + ) + + assert settings.MODEL_CONFIG.transport == "openai" + assert settings.MODEL_CONFIG.model == "gpt-4.1-mini" + assert settings.MODEL_CONFIG.thinking_budget_tokens is None + assert settings.MODEL_CONFIG.thinking_effort is None + + +def test_partial_env_override_same_transport_keeps_default_thinking_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """When env preserves the default transport, default thinking params still + apply — we only strip on actual transport change. + + The app-level defaults are intentionally minimal (transport + model only) + to avoid clobbering operator config, so this test patches in a deliberately + rich default to exercise the merge-preservation behavior. + """ + from src.config import ConfiguredModelSettings, DeriverSettings + + _clear_deriver_env(monkeypatch) + + def _rich_default() -> ConfiguredModelSettings: + return ConfiguredModelSettings( + transport="gemini", + model="gemini-2.5-flash-lite", + thinking_budget_tokens=1024, + max_output_tokens=4096, + ) + + monkeypatch.setattr(DeriverSettings, "_MODEL_CONFIG_DEFAULT", _rich_default) + + settings = DeriverSettings( + MODEL_CONFIG={"model": "gemini-2.5-pro"}, # pyright: ignore[reportArgumentType] + ) + + assert settings.MODEL_CONFIG.transport == "gemini" + assert settings.MODEL_CONFIG.model == "gemini-2.5-pro" + assert settings.MODEL_CONFIG.thinking_budget_tokens == 1024 + assert settings.MODEL_CONFIG.max_output_tokens == 4096 + + +def test_explicit_thinking_effort_survives_transport_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """User-set thinking params in the override are always preserved.""" + from src.config import DeriverSettings + + _clear_deriver_env(monkeypatch) + settings = DeriverSettings( + MODEL_CONFIG={ # pyright: ignore[reportArgumentType] + "transport": "openai", + "model": "gpt-5", + "thinking_effort": "high", + }, + ) + + assert settings.MODEL_CONFIG.transport == "openai" + assert settings.MODEL_CONFIG.thinking_effort == "high" + assert settings.MODEL_CONFIG.thinking_budget_tokens is None + + +def test_dialectic_level_transport_override_drops_default_thinking_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Same leak existed in DialecticSettings._merge_level_defaults. + Regression: when a level default has thinking_budget_tokens=0 under a + gemini transport and env flips the override to openai, the 0 used to leak + through and trip the OpenAI backend's thinking-param rejection. + + The app-level defaults are intentionally minimal (transport + model only) + to avoid clobbering operator config, so this test patches in a rich + level default to exercise the strip-on-transport-change behavior. + + Exercises the before-validator directly to avoid DialecticSettings' + "all 5 levels required" constraint. + """ + from src.config import ( + ConfiguredModelSettings, + DialecticLevelSettings, + DialecticSettings, + ) + + def _rich_levels() -> dict[str, DialecticLevelSettings]: + return { + "minimal": DialecticLevelSettings( + MODEL_CONFIG=ConfiguredModelSettings( + transport="gemini", + model="gemini-2.5-flash-lite", + thinking_budget_tokens=0, + ), + MAX_TOOL_ITERATIONS=1, + MAX_OUTPUT_TOKENS=250, + TOOL_CHOICE="any", + ), + } + + monkeypatch.setattr("src.config._default_dialectic_levels", _rich_levels) + + data: dict[str, object] = { + "LEVELS": { + "minimal": { + "MODEL_CONFIG": { + "transport": "openai", + "model": "gpt-4.1-mini", + } + } + } + } + # The @model_validator decorator wraps the classmethod in a descriptor proxy + # that pyright can't see as callable; at runtime pydantic routes it correctly. + merged = cast( + dict[str, Any], + DialecticSettings._merge_level_defaults(data), # pyright: ignore[reportPrivateUsage, reportCallIssue] + ) + levels = cast(dict[str, dict[str, Any]], merged["LEVELS"]) + minimal_mc = cast(dict[str, Any], levels["minimal"]["MODEL_CONFIG"]) + assert minimal_mc["transport"] == "openai" + assert minimal_mc["model"] == "gpt-4.1-mini" + assert "thinking_budget_tokens" not in minimal_mc + assert "thinking_effort" not in minimal_mc diff --git a/tests/llm/test_request_builder.py b/tests/llm/test_request_builder.py new file mode 100644 index 00000000..c8ed7dfd --- /dev/null +++ b/tests/llm/test_request_builder.py @@ -0,0 +1,97 @@ +from pydantic import BaseModel + +from src.config import ModelConfig +from src.llm.caching import PromptCachePolicy +from src.llm.request_builder import execute_completion +from tests.llm.conftest import FakeBackend + + +class SampleResponse(BaseModel): + answer: str + + +async def test_gemini_explicit_budget_passes_tokens_through_without_adjustment( + fake_backend: FakeBackend, +) -> None: + config = ModelConfig( + model="gemini-2.5-flash", + transport="gemini", + thinking_budget_tokens=256, + ) + + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + call = fake_backend.calls[0] + # No auto-adjustment — operators set explicit values + assert call["max_output_tokens"] == 100 + assert call["max_tokens"] == 100 + assert call["thinking_budget_tokens"] == 256 + + +async def test_thinking_params_are_passed_through_without_capability_dropping( + fake_backend: FakeBackend, +) -> None: + config = ModelConfig( + model="claude-haiku-4-5", + transport="anthropic", + thinking_effort="high", + thinking_budget_tokens=1024, + ) + + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + call = fake_backend.calls[0] + assert call["thinking_effort"] == "high" + assert call["thinking_budget_tokens"] == 1024 + + +async def test_cache_policy_is_passed_through_extra_params( + fake_backend: FakeBackend, +) -> None: + config = ModelConfig(model="gpt-4.1-mini", transport="openai") + cache_policy = PromptCachePolicy(mode="prefix", ttl_seconds=300) + + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=SampleResponse, + cache_policy=cache_policy, + ) + + call = fake_backend.calls[0] + assert call["response_format"] is SampleResponse + assert call["extra_params"]["cache_policy"] == cache_policy + + +async def test_provider_params_are_merged_into_extra_params( + fake_backend: FakeBackend, +) -> None: + config = ModelConfig( + model="gpt-4.1-mini", + transport="openai", + top_p=0.9, + provider_params={"custom_flag": True}, + ) + + await execute_completion( + fake_backend, + config, + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + ) + + call = fake_backend.calls[0] + assert call["extra_params"]["top_p"] == 0.9 + assert call["extra_params"]["custom_flag"] is True diff --git a/tests/routes/test_peers.py b/tests/routes/test_peers.py index c8047f02..9626ea10 100644 --- a/tests/routes/test_peers.py +++ b/tests/routes/test_peers.py @@ -576,25 +576,6 @@ def test_get_peer_representation_with_all_parameters( assert isinstance(data["representation"], str) -def test_get_peer_representation_structure( - client: TestClient, sample_data: tuple[Workspace, Peer] -): - """Test that peer representation response has correct structure""" - test_workspace, test_peer = sample_data - - # Get representation and validate structure - response = client.post( - f"/v3/workspaces/{test_workspace.name}/peers/{test_peer.name}/representation", - json={}, - ) - assert response.status_code == 200 - data = response.json() - - # Validate response structure - assert "representation" in data - assert isinstance(data["representation"], str) - - def test_get_peer_representation_boundary_values( client: TestClient, sample_data: tuple[Workspace, Peer] ): diff --git a/tests/routes/test_queue_status.py b/tests/routes/test_queue_status.py index af25077b..279ad913 100644 --- a/tests/routes/test_queue_status.py +++ b/tests/routes/test_queue_status.py @@ -74,20 +74,6 @@ class TestDeriverStatusEndpoint: assert response.status_code == 200 assert response.json()["total_work_units"] == 0 - async def test_get_deriver_status_with_include_sender_false( - self, - client: TestClient, - sample_data: tuple[models.Workspace, models.Peer], - ): - """Test getting deriver status with include_sender=False (default)""" - workspace, peer = sample_data - response = client.get( - f"/v3/workspaces/{workspace.name}/queue/status", - params={"observer_id": peer.name}, - ) - assert response.status_code == 200 - assert response.json()["total_work_units"] == 0 - async def test_get_deriver_status_no_parameters( self, client: TestClient, sample_data: tuple[models.Workspace, models.Peer] ): diff --git a/tests/routes/test_scoped_api.py b/tests/routes/test_scoped_api.py index c2d7d2e6..a4d4677a 100644 --- a/tests/routes/test_scoped_api.py +++ b/tests/routes/test_scoped_api.py @@ -20,45 +20,6 @@ def test_create_workspace_with_auth(auth_client: AuthClient): assert response.status_code in [200, 201] -def test_auth_response_time(auth_client: AuthClient): - name = str(generate_nanoid()) - - import time - - start_time = time.time() - - response = auth_client.post( - "/v3/workspaces", json={"name": name, "metadata": {"key": "value"}} - ) - - end_time = time.time() - response_time = end_time - start_time - print( - f"Server response time for client {auth_client.auth_type}: {response_time:.6f} seconds" - ) - - # Check expected behavior based on auth type - if auth_client.auth_type != "admin": - assert response.status_code == 401 - return - - assert response.status_code in [200, 201] - - -def test_get_or_create_workspace_with_auth(auth_client: AuthClient): - name = str(generate_nanoid()) - - response = auth_client.post( - "/v3/workspaces", json={"name": name, "metadata": {"key": "value"}} - ) - - if auth_client.auth_type != "admin": - assert response.status_code == 401 - return - - assert response.status_code in [200, 201] - - def test_get_workspace_with_auth( auth_client: AuthClient, sample_data: tuple[Workspace, Peer] ): diff --git a/tests/utils/test_clients.py b/tests/utils/test_clients.py index f76cb27c..213a370c 100644 --- a/tests/utils/test_clients.py +++ b/tests/utils/test_clients.py @@ -1,14 +1,12 @@ """ -Comprehensive tests for src/utils/clients.py +Comprehensive tests for the public src.llm orchestration surface. Tests cover: -- All supported LLM providers (Anthropic, OpenAI, Google/Gemini, Groq) +- All supported LLM providers (Anthropic, OpenAI, Google/Gemini) - Streaming and non-streaming responses - Response models (structured output) - Error handling and retries - Provider-specific features -- Client initialization -- Langfuse integration """ from typing import Any @@ -25,13 +23,12 @@ from openai.types.chat.chat_completion_message import ChatCompletionMessage from openai.types.completion_usage import CompletionUsage from pydantic import BaseModel, Field -from src.config import settings -from src.exceptions import LLMError -from src.utils.clients import ( +from src.config import ConfiguredModelSettings, ModelConfig, ResolvedFallbackConfig +from src.exceptions import LLMError, ValidationException +from src.llm import ( CLIENTS, HonchoLLMCallResponse, HonchoLLMCallStreamChunk, - handle_streaming_response, honcho_llm_call, honcho_llm_call_inner, ) @@ -185,45 +182,14 @@ class TestAnthropicClient: model="claude-3-sonnet", prompt="Think about this", max_tokens=100, - thinking_budget_tokens=1000, + thinking_budget_tokens=1024, ) # Verify thinking parameter was passed mock_client.messages.create.assert_called_once() call_args = mock_client.messages.create.call_args thinking_config = call_args.kwargs["thinking"] - assert thinking_config == {"type": "enabled", "budget_tokens": 1000} - - async def test_anthropic_response_model_with_json_parsing(self): - """Test that Anthropic supports response models via JSON schema in prompt""" - from anthropic.types import TextBlock - - # Create an actual Anthropic client mock that passes isinstance checks - mock_messages = AsyncMock() - mock_response = Mock() - # Create an actual TextBlock instance that will pass isinstance checks - text_block = TextBlock(type="text", text='"name": "Alice", "age": 30}') - mock_response.content = [text_block] - mock_response.usage = Mock(output_tokens=10) - mock_response.stop_reason = "end_turn" - mock_messages.create.return_value = mock_response - - # Instead of mocking the CLIENTS dict, we mock the entire AsyncAnthropic class - # to return our configured mock when instantiated - with patch("src.utils.clients.AsyncAnthropic") as mock_anthropic_class: - mock_client_instance = Mock() - mock_client_instance.messages = mock_messages - mock_anthropic_class.return_value = mock_client_instance - - # Also need to patch the CLIENTS dict with an instance that passes isinstance - # Since this is complex, let's verify the simpler behavior - that response_model - # is supported and the prompt is modified (no NotImplementedError) - - # Note: Full integration testing of response_model parsing would require - # a more complex setup with actual Anthropic client mocking. - # This test verifies that the code path for response_model exists and - # modifies the prompt appropriately. - pass # Test simplified - behavior is now supported + assert thinking_config == {"type": "enabled", "budget_tokens": 1024} async def test_anthropic_streaming(self): """Test Anthropic streaming response""" @@ -253,16 +219,16 @@ class TestAnthropicClient: with patch.dict(CLIENTS, {"anthropic": mock_client}): chunks: list[HonchoLLMCallStreamChunk] = [] - async for chunk in handle_streaming_response( - client=mock_client, - params={ - "model": "claude-3-sonnet", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hello"}], - }, - json_mode=False, - thinking_budget_tokens=None, - ): + stream = await honcho_llm_call_inner( + provider="anthropic", + model="claude-3-sonnet", + prompt="Hello", + max_tokens=100, + stream=True, + client_override=mock_client, + messages=[{"role": "user", "content": "Hello"}], + ) + async for chunk in stream: chunks.append(chunk) assert len(chunks) == 3 # 2 content chunks + 1 final chunk @@ -501,16 +467,16 @@ class TestOpenAIClient: with patch.dict(CLIENTS, {"openai": mock_client}): chunks: list[HonchoLLMCallStreamChunk] = [] - async for chunk in handle_streaming_response( - client=mock_client, - params={ - "model": "gpt-4", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hello"}], - }, - json_mode=False, - thinking_budget_tokens=None, - ): + stream = await honcho_llm_call_inner( + provider="openai", + model="gpt-4", + prompt="Hello", + max_tokens=100, + stream=True, + client_override=mock_client, + messages=[{"role": "user", "content": "Hello"}], + ) + async for chunk in stream: chunks.append(chunk) assert len(chunks) == 3 @@ -553,9 +519,9 @@ class TestGoogleClient: mock_aio.models.generate_content = AsyncMock(return_value=mock_response) mock_client.aio = mock_aio - with patch.dict(CLIENTS, {"google": mock_client}): + with patch.dict(CLIENTS, {"gemini": mock_client}): response = await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-1.5-pro", prompt="Hello", max_tokens=100, @@ -600,9 +566,9 @@ class TestGoogleClient: mock_aio.models.generate_content = AsyncMock(return_value=mock_response) mock_client.aio = mock_aio - with patch.dict(CLIENTS, {"google": mock_client}): + with patch.dict(CLIENTS, {"gemini": mock_client}): _response = await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-1.5-pro", prompt="Generate JSON", max_tokens=100, @@ -637,9 +603,9 @@ class TestGoogleClient: mock_aio.models.generate_content = AsyncMock(return_value=mock_response) mock_client.aio = mock_aio - with patch.dict(CLIENTS, {"google": mock_client}): + with patch.dict(CLIENTS, {"gemini": mock_client}): response = await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-1.5-pro", prompt="Generate a person", max_tokens=100, @@ -691,18 +657,18 @@ class TestGoogleClient: ) mock_client.aio = mock_aio - with patch.dict(CLIENTS, {"google": mock_client}): + with patch.dict(CLIENTS, {"gemini": mock_client}): chunks: list[HonchoLLMCallStreamChunk] = [] - async for chunk in handle_streaming_response( - client=mock_client, - params={ - "model": "gemini-1.5-pro", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hello"}], - }, - json_mode=False, - thinking_budget_tokens=None, - ): + stream = await honcho_llm_call_inner( + provider="gemini", + model="gemini-1.5-pro", + prompt="Hello", + max_tokens=100, + stream=True, + client_override=mock_client, + messages=[{"role": "user", "content": "Hello"}], + ) + async for chunk in stream: chunks.append(chunk) assert len(chunks) == 3 @@ -731,9 +697,9 @@ class TestGoogleClient: mock_aio.models.generate_content = AsyncMock(return_value=mock_response) mock_client.aio = mock_aio - with patch.dict(CLIENTS, {"google": mock_client}): + with patch.dict(CLIENTS, {"gemini": mock_client}): response = await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-1.5-pro", prompt="Hello", max_tokens=100, @@ -769,11 +735,11 @@ class TestGoogleClient: mock_client.aio = mock_aio with ( - patch.dict(CLIENTS, {"google": mock_client}), + patch.dict(CLIENTS, {"gemini": mock_client}), pytest.raises(LLMError, match=f"finish_reason={finish_reason}"), ): await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-2.5-flash", prompt="Summarize this", max_tokens=1000, @@ -799,9 +765,9 @@ class TestGoogleClient: mock_aio.models.generate_content = AsyncMock(return_value=mock_response) mock_client.aio = mock_aio - with patch.dict(CLIENTS, {"google": mock_client}): + with patch.dict(CLIENTS, {"gemini": mock_client}): response = await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-2.5-flash", prompt="Hello", max_tokens=100, @@ -829,11 +795,11 @@ class TestGoogleClient: mock_client.aio = mock_aio with ( - patch.dict(CLIENTS, {"google": mock_client}), + patch.dict(CLIENTS, {"gemini": mock_client}), pytest.raises(LLMError, match="finish_reason=SAFETY"), ): await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-2.5-flash", prompt="Generate a person", max_tokens=100, @@ -858,9 +824,9 @@ class TestGoogleClient: mock_aio.models.generate_content = AsyncMock(return_value=mock_response) mock_client.aio = mock_aio - with patch.dict(CLIENTS, {"google": mock_client}): + with patch.dict(CLIENTS, {"gemini": mock_client}): response = await honcho_llm_call_inner( - provider="google", + provider="gemini", model="gemini-2.5-flash", prompt="Generate a person", max_tokens=100, @@ -872,244 +838,6 @@ class TestGoogleClient: assert response.finish_reasons == ["SAFETY"] -@pytest.mark.asyncio -class TestGroqClient: - """Tests for Groq client functionality""" - - async def test_groq_basic_call(self): - """Test basic Groq API call""" - from groq import AsyncGroq - - mock_client = AsyncMock(spec=AsyncGroq) - mock_response = ChatCompletion( - id="test-id", - object="chat.completion", - created=1234567890, - model="llama-3.1-70b", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage( - role="assistant", content="Hello from Groq" - ), - finish_reason="stop", - ) - ], - usage=CompletionUsage( - prompt_tokens=10, completion_tokens=8, total_tokens=18 - ), - ) - mock_client.chat.completions.create = AsyncMock(return_value=mock_response) - - with patch.dict(CLIENTS, {"groq": mock_client}): - response = await honcho_llm_call_inner( - provider="groq", model="llama-3.1-70b", prompt="Hello", max_tokens=100 - ) - - assert isinstance(response, HonchoLLMCallResponse) - assert response.content == "Hello from Groq" - assert response.output_tokens == 8 - assert response.finish_reasons == ["stop"] - - async def test_groq_json_mode(self): - """Test Groq with JSON mode""" - from groq import AsyncGroq - - mock_client = AsyncMock(spec=AsyncGroq) - mock_response = ChatCompletion( - id="test-id", - object="chat.completion", - created=1234567890, - model="llama-3.1-70b", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage( - role="assistant", content='{"success": true}' - ), - finish_reason="stop", - ) - ], - usage=CompletionUsage( - prompt_tokens=10, completion_tokens=5, total_tokens=15 - ), - ) - mock_client.chat.completions.create = AsyncMock(return_value=mock_response) - - with patch.dict(CLIENTS, {"groq": mock_client}): - _response = await honcho_llm_call_inner( - provider="groq", - model="llama-3.1-70b", - prompt="Generate JSON", - max_tokens=100, - json_mode=True, - ) - - # Verify JSON mode was set - mock_client.chat.completions.create.assert_called_once() - call_args = mock_client.chat.completions.create.call_args - assert call_args.kwargs["response_format"] == {"type": "json_object"} - - async def test_groq_response_model(self): - """Test Groq with response model (structured output)""" - from groq import AsyncGroq - - mock_client = AsyncMock(spec=AsyncGroq) - # Mock JSON response that matches SampleTestModel structure - json_content = '{"name": "Bob", "age": 30, "active": true}' - mock_response = ChatCompletion( - id="test-id", - object="chat.completion", - created=1234567890, - model="llama-3.1-70b", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage( - role="assistant", content=json_content - ), - finish_reason="stop", - ) - ], - usage=CompletionUsage( - prompt_tokens=10, completion_tokens=12, total_tokens=22 - ), - ) - mock_client.chat.completions.create = AsyncMock(return_value=mock_response) - - with patch.dict(CLIENTS, {"groq": mock_client}): - response = await honcho_llm_call_inner( - provider="groq", - model="llama-3.1-70b", - prompt="Generate a person", - max_tokens=100, - response_model=SampleTestModel, - ) - - # Verify the response contains the parsed model - assert isinstance(response.content, SampleTestModel) - assert response.content.name == "Bob" - assert response.content.age == 30 - assert response.content.active is True - assert response.output_tokens == 12 - assert response.finish_reasons == ["stop"] - - # Verify the response format was set to the model - mock_client.chat.completions.create.assert_called_once() - call_args = mock_client.chat.completions.create.call_args - assert call_args.kwargs["response_format"] == SampleTestModel - - async def test_groq_no_content_error(self): - """Test Groq error handling when no content in response""" - from groq import AsyncGroq - - mock_client = AsyncMock(spec=AsyncGroq) - mock_response = ChatCompletion( - id="test-id", - object="chat.completion", - created=1234567890, - model="llama-3.1-70b", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content=None), - finish_reason="stop", - ) - ], - usage=CompletionUsage( - prompt_tokens=10, completion_tokens=0, total_tokens=10 - ), - ) - mock_client.chat.completions.create = AsyncMock(return_value=mock_response) - - with ( - patch.dict(CLIENTS, {"groq": mock_client}), - pytest.raises(ValueError, match="No content in response"), - ): - await honcho_llm_call_inner( - provider="groq", - model="llama-3.1-70b", - prompt="Hello", - max_tokens=100, - ) - - async def test_groq_streaming(self): - """Test Groq streaming response""" - from groq import AsyncGroq - - mock_client = AsyncMock(spec=AsyncGroq) - - # Create mock streaming chunks - mock_chunks = [ - ChatCompletionChunk( - id="test-id", - object="chat.completion.chunk", - created=1234567890, - model="llama-3.1-70b", - choices=[ - ChunkChoice( - index=0, delta=ChoiceDelta(content="Hello"), finish_reason=None - ) - ], - ), - ChatCompletionChunk( - id="test-id", - object="chat.completion.chunk", - created=1234567890, - model="llama-3.1-70b", - choices=[ - ChunkChoice( - index=0, - delta=ChoiceDelta(content=" from Groq"), - finish_reason=None, - ) - ], - ), - ChatCompletionChunk( - id="test-id", - object="chat.completion.chunk", - created=1234567890, - model="llama-3.1-70b", - choices=[ - ChunkChoice( - index=0, delta=ChoiceDelta(content=None), finish_reason="stop" - ) - ], - ), - ] - - # Create async iterator - async def async_chunk_iterator(): - for chunk in mock_chunks: - yield chunk - - # Mock the create method to return the async generator when awaited - mock_client.chat.completions.create = AsyncMock( - return_value=async_chunk_iterator() - ) - - with patch.dict(CLIENTS, {"groq": mock_client}): - chunks: list[HonchoLLMCallStreamChunk] = [] - async for chunk in handle_streaming_response( - client=mock_client, - params={ - "model": "llama-3.1-70b", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hello"}], - }, - json_mode=False, - thinking_budget_tokens=None, - ): - chunks.append(chunk) - - assert len(chunks) == 3 - assert chunks[0].content == "Hello" - assert chunks[1].content == " from Groq" - assert chunks[2].content == "" - assert chunks[2].is_done is True - assert chunks[2].finish_reasons == ["stop"] - - @pytest.mark.asyncio class TestMainLLMCallFunction: """Tests for the main honcho_llm_call function""" @@ -1136,11 +864,12 @@ class TestMainLLMCallFunction: mock_client.messages.stream.return_value = mock_stream with patch.dict(CLIENTS, {"anthropic": mock_client}): - settings.DIALECTIC.LEVELS["medium"].PROVIDER = "anthropic" - settings.DIALECTIC.LEVELS["medium"].MODEL = "claude-4-sonnet" chunks: list[HonchoLLMCallStreamChunk] = [] async for chunk in await honcho_llm_call( - llm_settings=settings.DIALECTIC.LEVELS["medium"], + model_config=ConfiguredModelSettings( + model="claude-4-sonnet", + transport="anthropic", + ), prompt="Hello", max_tokens=100, stream=True, @@ -1164,10 +893,11 @@ class TestMainLLMCallFunction: mock_client.messages.create = AsyncMock(return_value=mock_response) with patch.dict(CLIENTS, {"anthropic": mock_client}): - settings.DIALECTIC.LEVELS["medium"].PROVIDER = "anthropic" - settings.DIALECTIC.LEVELS["medium"].MODEL = "claude-4-sonnet" response = await honcho_llm_call( - llm_settings=settings.DIALECTIC.LEVELS["medium"], + model_config=ConfiguredModelSettings( + model="claude-4-sonnet", + transport="anthropic", + ), prompt="Hello", max_tokens=100, enable_retry=False, @@ -1191,44 +921,399 @@ class TestEdgeCases: assert new_chunk.finish_reasons == [] # Should still be empty -# Test fixtures and utilities -@pytest.fixture -def sample_test_model(): - """Fixture providing a sample SampleTestModel instance""" - return SampleTestModel(name="Test User", age=25, active=True) +@pytest.mark.asyncio +class TestModelConfigCalls: + async def test_honcho_llm_call_accepts_model_config(self): + mock_client = AsyncMock(spec=AsyncAnthropic) + mock_response = Mock() + mock_response.content = [TextBlock(text="ModelConfig response", type="text")] + mock_response.usage = Usage(input_tokens=8, output_tokens=4) + mock_response.stop_reason = "stop" + mock_client.messages.create = AsyncMock(return_value=mock_response) - -@pytest.fixture -def mock_anthropic_client(): - """Fixture providing a mocked Anthropic client""" - mock_client = AsyncMock() - mock_response = Mock() - mock_response.content = [TextBlock(text="Mocked Anthropic response", type="text")] - mock_response.usage = Usage(input_tokens=10, output_tokens=5) - mock_response.stop_reason = "stop" - mock_client.messages.create.return_value = mock_response - return mock_client - - -@pytest.fixture -def mock_openai_client(): - """Fixture providing a mocked OpenAI client""" - mock_client = AsyncMock() - mock_response = ChatCompletion( - id="test-id", - object="chat.completion", - created=1234567890, - model="gpt-4", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage( - role="assistant", content="Mocked OpenAI response" + with patch.dict(CLIENTS, {"anthropic": mock_client}): + response = await honcho_llm_call( + model_config=ModelConfig( + model="claude-haiku-4-5", + transport="anthropic", ), - finish_reason="stop", + prompt="Hello", + max_tokens=100, + enable_retry=False, + ) + + assert response.content == "ModelConfig response" + await_args = mock_client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic create call") + call_args = await_args.kwargs + assert call_args["model"] == "claude-haiku-4-5" + + async def test_honcho_llm_call_accepts_configured_model_settings(self): + mock_client = AsyncMock(spec=AsyncAnthropic) + mock_response = Mock() + mock_response.content = [ + TextBlock(text="ConfiguredModelSettings response", type="text") + ] + mock_response.usage = Usage(input_tokens=8, output_tokens=4) + mock_response.stop_reason = "stop" + mock_client.messages.create = AsyncMock(return_value=mock_response) + + with patch.dict(CLIENTS, {"anthropic": mock_client}): + response = await honcho_llm_call( + model_config=ConfiguredModelSettings( + model="claude-haiku-4-5", + transport="anthropic", + thinking_budget_tokens=1024, + ), + prompt="Hello", + max_tokens=100, + enable_retry=False, + ) + + assert response.content == "ConfiguredModelSettings response" + await_args = mock_client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic create call") + call_args = await_args.kwargs + assert call_args["model"] == "claude-haiku-4-5" + assert call_args["thinking"] == { + "type": "enabled", + "budget_tokens": 1024, + } + + +@pytest.mark.asyncio +class TestModelConfigExtraParamsPropagation: + """Regression tests — config knobs must reach the backend. + + Prior to the fix, honcho_llm_call_inner built extra_params from only + {json_mode, verbosity}, silently dropping top_p/top_k/frequency_penalty/ + presence_penalty/seed/provider_params off the ModelConfig. These tests + lock in that each backend now receives them. + """ + + async def test_openai_propagates_top_p_frequency_seed(self): + from openai import AsyncOpenAI + + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_response = ChatCompletion( + id="test-id", + object="chat.completion", + created=1234567890, + model="gpt-4.1", + choices=[ + Choice( + index=0, + message=ChatCompletionMessage(role="assistant", content="ok"), + finish_reason="stop", + ) + ], + usage=CompletionUsage( + prompt_tokens=10, completion_tokens=5, total_tokens=15 + ), + ) + mock_client.chat.completions.create = AsyncMock(return_value=mock_response) + + with patch.dict(CLIENTS, {"openai": mock_client}): + await honcho_llm_call( + model_config=ModelConfig( + model="gpt-4.1", + transport="openai", + top_p=0.92, + frequency_penalty=0.5, + presence_penalty=0.1, + seed=42, + ), + prompt="Hello", + max_tokens=100, + enable_retry=False, + ) + + mock_client.chat.completions.create.assert_called_once() + kwargs = mock_client.chat.completions.create.call_args.kwargs + assert kwargs["top_p"] == 0.92 + assert kwargs["frequency_penalty"] == 0.5 + assert kwargs["presence_penalty"] == 0.1 + assert kwargs["seed"] == 42 + + async def test_anthropic_propagates_top_p_top_k(self): + mock_client = AsyncMock(spec=AsyncAnthropic) + mock_response = Mock() + mock_response.content = [TextBlock(text="ok", type="text")] + mock_response.usage = Usage(input_tokens=8, output_tokens=4) + mock_response.stop_reason = "stop" + mock_client.messages.create = AsyncMock(return_value=mock_response) + + with patch.dict(CLIENTS, {"anthropic": mock_client}): + await honcho_llm_call( + model_config=ModelConfig( + model="claude-haiku-4-5", + transport="anthropic", + top_p=0.85, + top_k=40, + ), + prompt="Hello", + max_tokens=100, + enable_retry=False, + ) + + await_args = mock_client.messages.create.await_args + if await_args is None: + raise AssertionError("Expected Anthropic create call") + kwargs = await_args.kwargs + assert kwargs["top_p"] == 0.85 + assert kwargs["top_k"] == 40 + + async def test_provider_params_passthrough(self): + """Operator-supplied provider_params must reach the backend's extra_params. + + Scope: verifies the ModelConfig.provider_params → backend.extra_params + boundary inside honcho_llm_call_inner. This is NOT a guarantee that + arbitrary keys reach the provider SDK — each backend's _build_params + forwards only an allowlist (top_p, top_k, frequency_penalty, seed, + etc.). We assert only that the sentinel key arrives in extra_params + at the backend boundary, which is the internal contract this test + exists to protect. + """ + from openai import AsyncOpenAI + + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_response = ChatCompletion( + id="test-id", + object="chat.completion", + created=1234567890, + model="gpt-4.1", + choices=[ + Choice( + index=0, + message=ChatCompletionMessage(role="assistant", content="ok"), + finish_reason="stop", + ) + ], + usage=CompletionUsage( + prompt_tokens=10, completion_tokens=5, total_tokens=15 + ), + ) + mock_client.chat.completions.create = AsyncMock(return_value=mock_response) + + captured_extra: dict[str, Any] = {} + + from src.llm.backends.openai import OpenAIBackend + + original_complete = OpenAIBackend.complete + + async def capture_extra(self: Any, **kwargs: Any) -> Any: + captured_extra.update(kwargs.get("extra_params") or {}) + return await original_complete(self, **kwargs) + + with ( + patch.dict(CLIENTS, {"openai": mock_client}), + patch.object(OpenAIBackend, "complete", capture_extra), + ): + await honcho_llm_call( + model_config=ModelConfig( + model="gpt-4.1", + transport="openai", + provider_params={"honcho_sentinel": "zap"}, + ), + prompt="Hello", + max_tokens=100, + enable_retry=False, + ) + + assert captured_extra.get("honcho_sentinel") == "zap" + + async def test_cache_policy_reaches_gemini_backend(self): + """PromptCachePolicy set on ModelConfig must reach the Gemini backend's + extra_params as a typed object (so gemini_cached_content reuse fires).""" + from google import genai + + from src.config import PromptCachePolicy + from src.llm.backends.gemini import GeminiBackend + + mock_client = Mock(spec=genai.Client) + mock_client.__class__ = genai.Client # pyright: ignore[reportAttributeAccessIssue] + + import contextlib + + captured_extra: dict[str, Any] = {} + + async def capture_extra(_self: Any, **kwargs: Any) -> Any: + captured_extra.update(kwargs.get("extra_params") or {}) + return None + + policy = PromptCachePolicy(mode="gemini_cached_content", ttl_seconds=300) + + with ( + patch.dict(CLIENTS, {"gemini": mock_client}), + patch.object(GeminiBackend, "complete", capture_extra), + # capture_extra returns None, so downstream normalization will raise; + # we only care that extra_params was observed pre-raise. + contextlib.suppress(Exception), + ): + await honcho_llm_call( + model_config=ModelConfig( + model="gemini-2.5-flash", + transport="gemini", + cache_policy=policy, + ), + prompt="Hello", + max_tokens=100, + enable_retry=False, + ) + + assert captured_extra.get("cache_policy") is policy + + async def test_per_call_kwargs_override_provider_params(self): + """json_mode/verbosity from honcho_llm_call must win over provider_params defaults.""" + from openai import AsyncOpenAI + + from src.llm.backends.openai import OpenAIBackend + + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_response = ChatCompletion( + id="test-id", + object="chat.completion", + created=1234567890, + model="gpt-4.1", + choices=[ + Choice( + index=0, + message=ChatCompletionMessage(role="assistant", content="{}"), + finish_reason="stop", + ) + ], + usage=CompletionUsage( + prompt_tokens=10, completion_tokens=5, total_tokens=15 + ), + ) + mock_client.chat.completions.create = AsyncMock(return_value=mock_response) + + captured_extra: dict[str, Any] = {} + original_complete = OpenAIBackend.complete + + async def capture_extra(self: Any, **kwargs: Any) -> Any: + captured_extra.update(kwargs.get("extra_params") or {}) + return await original_complete(self, **kwargs) + + with ( + patch.dict(CLIENTS, {"openai": mock_client}), + patch.object(OpenAIBackend, "complete", capture_extra), + ): + await honcho_llm_call( + model_config=ModelConfig( + model="gpt-4.1", + transport="openai", + provider_params={"json_mode": False, "verbosity": "low"}, + ), + prompt="Hello", + max_tokens=100, + json_mode=True, + verbosity="high", + enable_retry=False, + ) + + assert captured_extra["json_mode"] is True + assert captured_extra["verbosity"] == "high" + + async def test_fallback_config_thinking_params_applied_on_final_retry( + self, + ) -> None: + """When primary fails, the FALLBACK ModelConfig's own temperature and + thinking_budget_tokens must reach the backend on the final retry — + not the primary's values, and not whatever the caller never set. + + Regression for the 'default caller kwargs from runtime_model_config too + early' bug: if honcho_llm_call pre-populated temperature from + runtime_model_config (the primary) before attempt selection, those + primary values would clobber the fallback's own thinking params via + effective_config_for_call(update={...}). + """ + mock_client = AsyncMock(spec=AsyncAnthropic) + mock_response = Mock() + mock_response.content = [TextBlock(text="from fallback", type="text")] + mock_response.usage = Usage(input_tokens=5, output_tokens=3) + mock_response.stop_reason = "stop" + + # Primary fails twice, then fallback succeeds on attempt 3. + mock_client.messages.create = AsyncMock( + side_effect=[ + RuntimeError("primary attempt 1"), + RuntimeError("primary attempt 2"), + mock_response, + ] + ) + + fallback = ResolvedFallbackConfig( + model="claude-haiku-4-5", + transport="anthropic", + temperature=0.9, + thinking_budget_tokens=2048, + ) + + with patch.dict(CLIENTS, {"anthropic": mock_client}): + await honcho_llm_call( + model_config=ModelConfig( + model="claude-sonnet-4-5", + transport="anthropic", + temperature=0.1, + thinking_budget_tokens=1024, + fallback=fallback, + ), + prompt="Hello", + max_tokens=100, + enable_retry=True, + retry_attempts=3, + ) + + # Final call should carry the FALLBACK's values, not primary's. + final_call = mock_client.messages.create.await_args_list[-1] + kwargs = final_call.kwargs + assert kwargs["model"] == "claude-haiku-4-5" + assert kwargs["temperature"] == 0.9 + assert kwargs["thinking"] == { + "type": "enabled", + "budget_tokens": 2048, + } + + +@pytest.mark.asyncio +class TestToolLoopValidation: + """Lock in the fail-fast behavior on max_tool_iterations out of range.""" + + @pytest.mark.parametrize("bad_value", [0, -1, 101, 1_000]) + async def test_invalid_max_tool_iterations_raises(self, bad_value: int) -> None: + from src.llm.tool_loop import execute_tool_loop + + def _noop_plan() -> Any: # pragma: no cover - never called + raise AssertionError("plan should not be invoked for invalid input") + + def _noop_executor( + _name: str, _input: dict[str, Any] + ) -> str: # pragma: no cover + return "ok" + + def _noop_retry_callback(_state: Any) -> None: # pragma: no cover + return None + + with pytest.raises(ValidationException, match="max_tool_iterations"): + await execute_tool_loop( + prompt="x", + max_tokens=10, + messages=None, + tools=[{"name": "t", "description": "d", "input_schema": {}}], + tool_choice=None, + tool_executor=_noop_executor, + max_tool_iterations=bad_value, + response_model=None, + json_mode=False, + temperature=None, + stop_seqs=None, + verbosity=None, + enable_retry=False, + retry_attempts=3, + max_input_tokens=None, + get_attempt_plan=_noop_plan, + before_retry_callback=_noop_retry_callback, ) - ], - usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ) - mock_client.chat.completions.create = AsyncMock(return_value=mock_response) - return mock_client diff --git a/tests/utils/test_length_finish_reason.py b/tests/utils/test_length_finish_reason.py new file mode 100644 index 00000000..69d3210c --- /dev/null +++ b/tests/utils/test_length_finish_reason.py @@ -0,0 +1,456 @@ +""" +Tests for JSON repair handling across all providers in honcho_llm_call_inner, +and Gemini thinking budget support. + +Verifies that when an LLM hits the max token limit or returns malformed JSON, +the truncated output is repaired and returned instead of crashing. +""" + +import json +from typing import Any +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from anthropic import AsyncAnthropic +from anthropic.types import TextBlock, Usage +from openai import AsyncOpenAI, LengthFinishReasonError +from openai.types.chat import ChatCompletion +from openai.types.chat.chat_completion import Choice +from openai.types.chat.chat_completion_message import ChatCompletionMessage +from openai.types.completion_usage import CompletionUsage +from pydantic import BaseModel, ValidationError + +from src.llm import CLIENTS, HonchoLLMCallResponse, honcho_llm_call_inner +from src.utils.representation import PromptRepresentation + +# --- Test models --- + + +class SimpleModel(BaseModel): + """Non-PromptRepresentation model for testing re-raise behavior.""" + + items: list[str] + + +# --- Helpers --- + +VALID_REPR_JSON = { + "explicit": [ + {"content": "hermes is 25 years old"}, + {"content": "hermes has a dog"}, + ] +} + + +def _make_truncated_completion(content: str) -> ChatCompletion: + """Build a ChatCompletion with finish_reason='length' and the given content.""" + return ChatCompletion( + id="test-truncated", + object="chat.completion", + created=1234567890, + model="test-model", + choices=[ + Choice( + index=0, + message=ChatCompletionMessage(role="assistant", content=content), + finish_reason="length", + ) + ], + usage=CompletionUsage( + prompt_tokens=1000, completion_tokens=2000, total_tokens=3000 + ), + ) + + +def _raise_length_error(content: str) -> AsyncMock: + """Return an AsyncMock that raises LengthFinishReasonError with truncated content.""" + completion = _make_truncated_completion(content) + return AsyncMock(side_effect=LengthFinishReasonError(completion=completion)) + + +def _make_anthropic_mock(text: str, stop_reason: str = "end_turn") -> AsyncMock: + """Build a mocked AsyncAnthropic client returning the given text.""" + mock_client = AsyncMock(spec=AsyncAnthropic) + mock_response = Mock() + mock_response.content = [TextBlock(text=text, type="text")] + mock_response.usage = Usage(input_tokens=100, output_tokens=50) + mock_response.stop_reason = stop_reason + mock_client.messages.create = AsyncMock(return_value=mock_response) + return mock_client + + +def _make_gemini_mock( + text: str | None = None, + parsed: Any = None, + finish_reason_name: str = "STOP", +) -> Mock: + """Build a mocked genai.Client returning the given text/parsed content.""" + mock_client = Mock() + + # Build response + mock_response = Mock() + mock_response.parsed = parsed + + # Candidates + mock_candidate = Mock() + mock_finish_reason = Mock() + mock_finish_reason.name = finish_reason_name + mock_candidate.finish_reason = mock_finish_reason + + # Content parts + if text is not None: + mock_part = Mock() + mock_part.text = text + mock_part.function_call = None + mock_content = Mock() + mock_content.parts = [mock_part] + mock_candidate.content = mock_content + else: + mock_candidate.content = None + + mock_response.candidates = [mock_candidate] + + # Usage + mock_usage = Mock() + mock_usage.prompt_token_count = 200 + mock_usage.candidates_token_count = 100 + mock_response.usage_metadata = mock_usage + + mock_client.aio.models.generate_content = AsyncMock(return_value=mock_response) + return mock_client + + +# --------------------------------------------------------------------------- +# OpenAI / Custom provider tests (LengthFinishReasonError path) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestOpenAILengthFinishReasonRepair: + """Tests that LengthFinishReasonError is caught and truncated JSON is repaired.""" + + async def test_truncated_prompt_representation_repaired_openai(self) -> None: + """Truncated but repairable PromptRepresentation JSON should be repaired (openai).""" + truncated_json = json.dumps(VALID_REPR_JSON)[:-2] + + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_client.chat.completions.parse = _raise_length_error(truncated_json) + + with patch.dict(CLIENTS, {"openai": mock_client}): + response = await honcho_llm_call_inner( + provider="openai", + model="test-model", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response, HonchoLLMCallResponse) + assert isinstance(response.content, PromptRepresentation) + assert len(response.content.explicit) >= 1 + assert response.finish_reasons == ["length"] + assert response.output_tokens == 2000 + + async def test_truncated_prompt_representation_repaired_openai_with_custom_base( + self, + ) -> None: + """Truncated but repairable PromptRepresentation JSON should be repaired.""" + truncated_json = json.dumps(VALID_REPR_JSON)[:-2] + + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_client.chat.completions.parse = _raise_length_error(truncated_json) + + with patch.dict(CLIENTS, {"openai": mock_client}): + response = await honcho_llm_call_inner( + provider="openai", + model="test-model", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response, HonchoLLMCallResponse) + assert isinstance(response.content, PromptRepresentation) + assert len(response.content.explicit) >= 1 + assert response.finish_reasons == ["length"] + + async def test_completely_broken_json_falls_back_to_empty(self) -> None: + """Completely unrepairable JSON should fall back to empty PromptRepresentation.""" + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_client.chat.completions.parse = _raise_length_error( + "this is not json at all just random text" + ) + + with patch.dict(CLIENTS, {"openai": mock_client}): + response = await honcho_llm_call_inner( + provider="openai", + model="test-model", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response.content, PromptRepresentation) + assert response.content.explicit == [] + assert response.finish_reasons == ["length"] + + async def test_empty_content_falls_back_to_empty(self) -> None: + """Empty/null content should fall back to empty PromptRepresentation.""" + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_client.chat.completions.parse = _raise_length_error("") + + with patch.dict(CLIENTS, {"openai": mock_client}): + response = await honcho_llm_call_inner( + provider="openai", + model="test-model", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response.content, PromptRepresentation) + assert response.content.explicit == [] + + async def test_non_prompt_representation_reraises_on_unfixable(self) -> None: + """Non-PromptRepresentation with unrepairable JSON should raise ValidationError.""" + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_client.chat.completions.parse = _raise_length_error("not json") + + with ( + patch.dict(CLIENTS, {"openai": mock_client}), + pytest.raises(ValidationError), + ): + await honcho_llm_call_inner( + provider="openai", + model="test-model", + prompt="Generate items", + max_tokens=2000, + response_model=SimpleModel, + json_mode=True, + ) + + async def test_token_counts_preserved(self) -> None: + """Token counts from the truncated completion should be preserved.""" + truncated_json = '{"explicit": [{"content": "fact one"}' + + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_client.chat.completions.parse = _raise_length_error(truncated_json) + + with patch.dict(CLIENTS, {"openai": mock_client}): + response = await honcho_llm_call_inner( + provider="openai", + model="test-model", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert response.input_tokens == 1000 + assert response.output_tokens == 2000 + + async def test_valid_json_with_length_finish_reason(self) -> None: + """Valid JSON despite length truncation should parse fine.""" + valid_json = json.dumps(VALID_REPR_JSON) + + mock_client = AsyncMock(spec=AsyncOpenAI) + mock_client.chat.completions.parse = _raise_length_error(valid_json) + + with patch.dict(CLIENTS, {"openai": mock_client}): + response = await honcho_llm_call_inner( + provider="openai", + model="test-model", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response.content, PromptRepresentation) + assert len(response.content.explicit) == 2 + assert response.content.explicit[0].content == "hermes is 25 years old" + + +# --------------------------------------------------------------------------- +# Anthropic provider tests (JSON parse failure -> repair path) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestAnthropicJsonRepair: + """Tests that Anthropic response_model parse failures trigger JSON repair.""" + + async def test_truncated_anthropic_response_repaired(self) -> None: + """Truncated Anthropic JSON response should be repaired.""" + # Anthropic prefills "{" so the response text starts after that + # The code prepends "{" back: json_content = "{" + text_content + truncated_text = json.dumps(VALID_REPR_JSON)[ + 1:-2 + ] # Remove leading { and trailing }] + + mock_client = _make_anthropic_mock(truncated_text, stop_reason="max_tokens") + + with patch.dict(CLIENTS, {"anthropic": mock_client}): + response = await honcho_llm_call_inner( + provider="anthropic", + model="claude-3-sonnet", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response.content, PromptRepresentation) + assert len(response.content.explicit) >= 1 + + async def test_broken_anthropic_response_falls_back_to_empty(self) -> None: + """Completely broken Anthropic JSON should fall back to empty PromptRepresentation.""" + mock_client = _make_anthropic_mock( + "random gibberish that is not json", stop_reason="max_tokens" + ) + + with patch.dict(CLIENTS, {"anthropic": mock_client}): + response = await honcho_llm_call_inner( + provider="anthropic", + model="claude-3-sonnet", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response.content, PromptRepresentation) + assert response.content.explicit == [] + + async def test_non_prompt_representation_reraises(self) -> None: + """Non-PromptRepresentation with broken JSON should raise.""" + mock_client = _make_anthropic_mock("not json", stop_reason="max_tokens") + + with ( + patch.dict(CLIENTS, {"anthropic": mock_client}), + pytest.raises(ValidationError), + ): + await honcho_llm_call_inner( + provider="anthropic", + model="claude-3-sonnet", + prompt="Generate items", + max_tokens=2000, + response_model=SimpleModel, + json_mode=True, + ) + + +# --------------------------------------------------------------------------- +# Gemini provider tests (parsed=None or type mismatch -> repair path) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestGeminiJsonRepair: + """Tests that Gemini response_model parse failures trigger JSON repair.""" + + async def test_gemini_unparsed_response_repaired(self) -> None: + """Gemini returning text but no parsed object should repair from raw text.""" + from google import genai + + valid_text = json.dumps(VALID_REPR_JSON) + mock_client = _make_gemini_mock( + text=valid_text, parsed=None, finish_reason_name="MAX_TOKENS" + ) + + with ( + patch.dict(CLIENTS, {"gemini": mock_client}), + patch.object(genai.Client, "__instancecheck__", return_value=True), + ): + # We need the match statement to hit the genai.Client case + mock_client.__class__ = genai.Client # pyright: ignore[reportAttributeAccessIssue] + response = await honcho_llm_call_inner( + provider="gemini", + model="gemini-2.5-flash", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response.content, PromptRepresentation) + assert len(response.content.explicit) == 2 + + async def test_gemini_broken_text_falls_back_to_empty(self) -> None: + """Gemini with broken text and no parsed content should fall back.""" + from google import genai + + mock_client = _make_gemini_mock( + text="broken json", parsed=None, finish_reason_name="MAX_TOKENS" + ) + mock_client.__class__ = genai.Client # pyright: ignore[reportAttributeAccessIssue] + + with patch.dict(CLIENTS, {"gemini": mock_client}): + response = await honcho_llm_call_inner( + provider="gemini", + model="gemini-2.5-flash", + prompt="Analyze messages", + max_tokens=2000, + response_model=PromptRepresentation, + json_mode=True, + ) + + assert isinstance(response.content, PromptRepresentation) + assert response.content.explicit == [] + + +# --------------------------------------------------------------------------- +# Gemini thinking budget tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +class TestGeminiThinkingBudget: + """Tests that thinking_budget_tokens is passed to Gemini via ThinkingConfig.""" + + async def test_thinking_budget_passed_to_gemini(self) -> None: + """thinking_budget_tokens should be included in Gemini config.""" + from google import genai + + mock_client = _make_gemini_mock(text="Hello", parsed=None) + mock_client.__class__ = genai.Client # pyright: ignore[reportAttributeAccessIssue] + + with patch.dict(CLIENTS, {"gemini": mock_client}): + await honcho_llm_call_inner( + provider="gemini", + model="gemini-2.5-flash", + prompt="Think about this", + max_tokens=2000, + thinking_budget_tokens=4096, + ) + + # Verify generate_content was called with thinking_config + call_args = mock_client.aio.models.generate_content.call_args + config = call_args.kwargs.get("config") or call_args[1].get("config") + assert config is not None + assert "thinking_config" in config + assert config["thinking_config"]["thinking_budget"] == 4096 + + async def test_no_thinking_config_when_budget_is_none(self) -> None: + """When thinking_budget_tokens is None, thinking_config should not be set.""" + from google import genai + + mock_client = _make_gemini_mock(text="Hello", parsed=None) + mock_client.__class__ = genai.Client # pyright: ignore[reportAttributeAccessIssue] + + with patch.dict(CLIENTS, {"gemini": mock_client}): + await honcho_llm_call_inner( + provider="gemini", + model="gemini-2.5-flash", + prompt="No thinking needed", + max_tokens=2000, + ) + + call_args = mock_client.aio.models.generate_content.call_args + config = call_args.kwargs.get("config") or call_args[1].get("config") + if config: + assert "thinking_config" not in config diff --git a/tests/utils/test_summarizer.py b/tests/utils/test_summarizer.py index 3e8f8dc9..b842ced7 100644 --- a/tests/utils/test_summarizer.py +++ b/tests/utils/test_summarizer.py @@ -10,11 +10,14 @@ from unittest.mock import AsyncMock, patch import pytest -from src.utils.clients import HonchoLLMCallResponse +from src.config import settings +from src.llm import HonchoLLMCallResponse from src.utils.summarizer import ( Summary, SummaryType, _create_summary, # pyright: ignore[reportPrivateUsage] + create_long_summary, + create_short_summary, ) # Common test arguments for _create_summary @@ -217,3 +220,61 @@ class TestCreateSummary: assert is_fallback is True assert summary["content"] == "" assert summary["token_count"] == 0 + + +@pytest.mark.asyncio +class TestSummaryCallerMigration: + async def test_create_short_summary_uses_model_config(self): + mock_response = HonchoLLMCallResponse( + content="short summary", + input_tokens=10, + output_tokens=5, + finish_reasons=["STOP"], + ) + + with patch( + "src.utils.summarizer.honcho_llm_call", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_llm_call: + await create_short_summary( + formatted_messages=_FORMATTED_MESSAGES, + input_tokens=_INPUT_TOKENS, + previous_summary=None, + ) + + await_args = mock_llm_call.await_args + if await_args is None: + raise AssertionError("Expected summary LLM call") + kwargs = await_args.kwargs + expected_config = settings.SUMMARY.MODEL_CONFIG + assert "model_config" in kwargs + assert kwargs["model_config"].model == expected_config.model + assert "llm_settings" not in kwargs + + async def test_create_long_summary_uses_model_config(self): + mock_response = HonchoLLMCallResponse( + content="long summary", + input_tokens=10, + output_tokens=5, + finish_reasons=["STOP"], + ) + + with patch( + "src.utils.summarizer.honcho_llm_call", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_llm_call: + await create_long_summary( + formatted_messages=_FORMATTED_MESSAGES, + previous_summary=None, + ) + + await_args = mock_llm_call.await_args + if await_args is None: + raise AssertionError("Expected summary LLM call") + kwargs = await_args.kwargs + expected_config = settings.SUMMARY.MODEL_CONFIG + assert "model_config" in kwargs + assert kwargs["model_config"].model == expected_config.model + assert "llm_settings" not in kwargs diff --git a/uv.lock b/uv.lock index 0d464777..6e7dffa9 100644 --- a/uv.lock +++ b/uv.lock @@ -1254,23 +1254,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/4b/45d90626aef8e65336bed690106d1382f7a43665e2249017e9527df8823b/greenlet-3.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c04c5e06ec3e022cbfe2cd4a846e1d4e50087444f875ff6d2c2ad8445495cf1a", size = 237086, upload-time = "2026-02-20T20:20:45.786Z" }, ] -[[package]] -name = "groq" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3f/12/f4099a141677fcd2ed79dcc1fcec431e60c52e0e90c9c5d935f0ffaf8c0e/groq-1.0.0.tar.gz", hash = "sha256:66cb7bb729e6eb644daac7ce8efe945e99e4eb33657f733ee6f13059ef0c25a9", size = 146068, upload-time = "2025-12-17T23:34:23.115Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/88/3175759d2ef30406ea721f4d837bfa1ba4339fde3b81ba8c5640a96ed231/groq-1.0.0-py3-none-any.whl", hash = "sha256:6e22bf92ffad988f01d2d4df7729add66b8fd5dbfb2154b5bbf3af245b72c731", size = 138292, upload-time = "2025-12-17T23:34:21.957Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -1292,7 +1275,6 @@ dependencies = [ { name = "fastapi-pagination" }, { name = "google-genai" }, { name = "greenlet" }, - { name = "groq" }, { name = "httpx" }, { name = "json-repair" }, { name = "lancedb" }, @@ -1349,7 +1331,6 @@ requires-dist = [ { name = "fastapi-pagination", specifier = ">=0.14.2" }, { name = "google-genai", specifier = ">=1.32.0" }, { name = "greenlet", specifier = ">=3.0.3" }, - { name = "groq", specifier = ">=0.31.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "json-repair", specifier = ">=0.49.0" }, { name = "lancedb", specifier = ">=0.25.3" },