diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 13bce3f884199..99da9ee2f797a 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -164,7 +164,7 @@ from agent.credential_pool import load_pool from agent.model_metadata import MINIMUM_CONTEXT_LENGTH, get_model_context_length from hermes_cli.config import get_hermes_home from hermes_constants import OPENROUTER_BASE_URL -from utils import base_url_host_matches, base_url_hostname, env_float, model_forces_max_completion_tokens, normalize_proxy_env_vars +from utils import base_url_host_matches, base_url_hostname, env_float, is_truthy_value, model_forces_max_completion_tokens, normalize_proxy_env_vars logger = logging.getLogger(__name__) @@ -948,13 +948,21 @@ _API_KEY_PROVIDER_AUX_MODELS_FALLBACK: Dict[str, str] = { # can still use this dict directly. Kept in sync with _FALLBACK above. _API_KEY_PROVIDER_AUX_MODELS: Dict[str, str] = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK -# Auxiliary tasks that prefer the provider's fast/cheap model over the user's -# main chat model when running in "auto" mode. Restricted to tasks where -# latency is user-visible and the output is short enough that a small model -# matches a frontier one. Every other task keeps "auto = my chat model". +# Auxiliary tasks that may opt into the provider's fast/cheap model instead of +# the user's main chat model. The opt-in lives in +# ``auxiliary..prefer_fast_model`` so the default ``auto = main model`` +# contract remains true on every settings surface. _FAST_MODEL_TASKS: frozenset = frozenset({"title_generation"}) +def _task_prefers_fast_model(task: Optional[str]) -> bool: + """Return whether an eligible task explicitly opts into fast-model routing.""" + if task not in _FAST_MODEL_TASKS: + return False + task_config = _get_auxiliary_task_config(task) + return is_truthy_value(task_config.get("prefer_fast_model"), default=False) + + # Vision-specific model overrides for direct providers. # When the user's main provider has a dedicated vision/multimodal model that # differs from their main chat model, map it here. The vision auto-detect @@ -5677,17 +5685,13 @@ def _resolve_auto_route( main_provider = str(runtime_provider or _read_main_provider() or "") main_model = str(runtime_model or _read_main_model() or "") - # Latency-critical tasks prefer the provider's registered fast model over - # the main chat model. Titling is the only such task: it names a visible - # sidebar row, produces ~8 tokens, and running it on a frontier reasoning - # model costs seconds per new session. Every comparable tool routes titling - # to a small tier (Claude Code → Haiku, OpenCode → small_model, Zed → - # default_fast_model, OpenClaw → utilityModel). An explicit - # auxiliary..model in config.yaml still wins — this only redirects - # the "auto" default, and only when the provider registered a cheap model. - # Every other aux task keeps the "auto means my chat model" contract - # documented above: this does NOT change compression, vision, or search. - if task in _FAST_MODEL_TASKS and main_provider and main_provider not in {"auto", ""}: + # Latency-critical tasks can explicitly prefer the provider's registered + # fast model over the main chat model. Titling is the only eligible task: + # it names a visible sidebar row, produces ~8 tokens, and running it on a + # frontier reasoning model costs seconds per new session. This remains an + # opt-in because every settings surface defines "auto" as using the main + # model; silently overriding that choice makes the selected model cosmetic. + if _task_prefers_fast_model(task) and main_provider and main_provider not in {"auto", ""}: fast_model = _get_aux_model_for_provider(main_provider, prefer_fast=True) if fast_model and fast_model != main_model: logger.debug( @@ -7284,7 +7288,11 @@ def _client_cache_key( # `auto` can now resolve through task-specific or main fallback policy, # so the task participates in the cache key. Non-auto providers keep the # old cache shape because the explicit provider/model tuple is sufficient. - task_key = (task or "") if provider == "auto" else "" + task_key = ( + (task or "", _task_prefers_fast_model(task)) + if provider == "auto" + else "" + ) pool_hint = _pool_cache_hint(provider, main_runtime=main_runtime) # The model MUST participate in the key. Two concurrent auxiliary calls to # the SAME provider/base_url/key but DIFFERENT models (e.g. a MoA reference diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index f7ee770719cf2..9941354ecbcb6 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -980,6 +980,7 @@ DEFAULT_CONFIG = { "enabled": True, "provider": "auto", "model": "", + "prefer_fast_model": False, # opt in to provider fast tier; auto otherwise uses the main model "base_url": "", "api_key": "", "timeout": 30, diff --git a/tests/agent/test_auxiliary_client.py b/tests/agent/test_auxiliary_client.py index 803b69c98c027..e49bffcc870df 100644 --- a/tests/agent/test_auxiliary_client.py +++ b/tests/agent/test_auxiliary_client.py @@ -4491,7 +4491,22 @@ class TestAutoRoutedProviderProfileHooks: class TestFastModelTier: - """The titling fast tier: rot-proof resolution, scoped to titling only.""" + """The opt-in titling fast tier: rot-proof and scoped to titling only.""" + + def test_auto_client_cache_key_tracks_fast_model_preference(self): + """Changing the routing preference must not reuse the old auto client.""" + from agent import auxiliary_client as ac + + with patch.object(ac, "_task_prefers_fast_model", return_value=False): + main_key = ac._client_cache_key( + "auto", async_mode=False, task="title_generation" + ) + with patch.object(ac, "_task_prefers_fast_model", return_value=True): + fast_key = ac._client_cache_key( + "auto", async_mode=False, task="title_generation" + ) + + assert main_key != fast_key def test_catalog_match_prefers_rolling_alias_over_pinned_id(self): """A "-latest" alias wins: it is the only id that cannot go stale.""" diff --git a/tests/agent/test_auxiliary_main_first.py b/tests/agent/test_auxiliary_main_first.py index 47032751362f9..e999364cee06b 100644 --- a/tests/agent/test_auxiliary_main_first.py +++ b/tests/agent/test_auxiliary_main_first.py @@ -23,6 +23,67 @@ from unittest.mock import MagicMock, patch class TestResolveAutoMainFirst: """_resolve_auto() must prefer main provider + main model for every user.""" + def test_title_generation_auto_honors_main_model(self): + """The default auto title route must not replace the selected main model.""" + main_model = "deepseek-v4-flash-free" + mock_client = MagicMock() + + with patch( + "agent.auxiliary_client._get_aux_model_for_provider", + return_value="gemini-3-flash", + ), patch( + "agent.auxiliary_client.resolve_provider_client", + return_value=(mock_client, main_model), + ) as mock_resolve, patch( + "agent.auxiliary_client._is_provider_unhealthy", return_value=False + ): + from agent.auxiliary_client import _resolve_auto + + client, model = _resolve_auto( + main_runtime={ + "provider": "opencode-zen", + "model": main_model, + }, + task="title_generation", + ) + + assert client is mock_client + assert model == main_model + assert mock_resolve.call_args.args[:2] == ("opencode-zen", main_model) + + def test_title_generation_can_opt_into_provider_fast_model(self): + """The latency optimization remains available as an explicit opt-in.""" + fast_model = "gemini-3-flash" + mock_client = MagicMock() + + def resolve(_provider, model, **_kwargs): + return mock_client, model + + with patch( + "agent.auxiliary_client._get_auxiliary_task_config", + return_value={"prefer_fast_model": True}, + ), patch( + "agent.auxiliary_client._get_aux_model_for_provider", + return_value=fast_model, + ), patch( + "agent.auxiliary_client.resolve_provider_client", + side_effect=resolve, + ), patch( + "agent.auxiliary_client._is_provider_unhealthy", return_value=False + ): + from agent.auxiliary_client import _resolve_auto + + client, model = _resolve_auto( + main_runtime={ + "provider": "opencode-zen", + "model": "deepseek-v4-flash-free", + }, + task="title_generation", + ) + + assert client is mock_client + assert model == fast_model + def test_moa_main_resolves_aux_to_aggregator(self, monkeypatch, tmp_path): """MoA main user → aux runs on the aggregator slot, NOT the preset name. diff --git a/tests/hermes_cli/test_aux_config.py b/tests/hermes_cli/test_aux_config.py index 72dec6469f1b1..aa67a51ea7442 100644 --- a/tests/hermes_cli/test_aux_config.py +++ b/tests/hermes_cli/test_aux_config.py @@ -39,6 +39,7 @@ def test_title_generation_present_in_default_config(): assert tg["enabled"] is True assert tg["provider"] == "auto" assert tg["model"] == "" + assert tg["prefer_fast_model"] is False assert tg["timeout"] > 0 assert tg["extra_body"] == {} diff --git a/website/docs/user-guide/configuring-models.md b/website/docs/user-guide/configuring-models.md index 13c33f0f2f308..487ae9529c167 100644 --- a/website/docs/user-guide/configuring-models.md +++ b/website/docs/user-guide/configuring-models.md @@ -67,7 +67,7 @@ Every auxiliary task defaults to `auto` — meaning Hermes tries your main model | Task | When to override | |---|---| -| **Title Gen** | Almost always. A $0.10/M flash model writes session titles as well as Opus. Default config sets this to `google/gemini-3-flash-preview` on OpenRouter. | +| **Title Gen** | When title latency or cost matters more than matching the main model. Pin a known-good flash model, or set `auxiliary.title_generation.prefer_fast_model: true` to let Hermes choose the provider's fast tier. | | **Vision** | When your main model lacks vision support. Point it at `google/gemini-2.5-flash` or `gpt-4o-mini`. | | **Compression** | When you're burning reasoning tokens on Opus/M2.7 just to summarize context. A fast chat model does the job at 1/50th the cost. | | **Approval** | For `approval_mode: smart` — a fast/cheap model (haiku, flash, gpt-5-mini) decides whether to auto-approve low-risk commands. Expensive models here are waste. |