From 314968f5fb6e27f59f7d1cf37405d504c156222c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:35:33 -0700 Subject: [PATCH] Port from PrimeIntellect-ai/prime-agent#1258: derive OpenRouter reasoning support and effort levels from catalog metadata OpenRouter's /v1/models entries advertise reasoning capability (supported_parameters + reasoning.mandatory/supported_efforts). Use that metadata as the primary gate in _supports_reasoning_extra_body instead of the hand-maintained vendor-prefix allowlist, which went stale one vendor at a time (nvidia/ missing -> #75386). Also clamp the requested effort to the nearest LOWER catalog-supported level in the OpenRouter profile so ultra/max against a high-capped route no longer 4xxes. Cache-only on the hot path: capabilities parse for free out of the existing fetch_openrouter_models() payload, a background warmer covers cold starts, and unknown models/offline catalogs fall back to the static prefix list unchanged. --- hermes_cli/models.py | 223 +++++++++++++++ .../model-providers/openrouter/__init__.py | 41 ++- run_agent.py | 25 ++ .../test_openrouter_reasoning_metadata.py | 268 ++++++++++++++++++ website/docs/user-guide/configuration.md | 12 + 5 files changed, 568 insertions(+), 1 deletion(-) create mode 100644 tests/hermes_cli/test_openrouter_reasoning_metadata.py diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 7c402a98bfe17..5d0cc6a9783ee 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -1503,6 +1503,218 @@ def _openrouter_model_supports_tools(item: Any) -> bool: return "tools" in params +def parse_openrouter_reasoning_capabilities(item: Any) -> Optional[dict[str, Any]]: + """Normalize one OpenRouter catalog entry's reasoning metadata. + + OpenRouter's ``/v1/models`` catalog advertises reasoning support two ways: + ``supported_parameters`` contains ``"reasoning"`` when the route accepts + reasoning controls at all, and a top-level ``reasoning`` object may add + detail (``mandatory``, ``supported_efforts``). Per OpenRouter semantics + the top-level object is only trusted after ``supported_parameters`` + confirms the route accepts reasoning controls; ``supported_efforts`` + omitted/None means every effort is accepted. + + Returns: + ``{"supports_reasoning": True, "supported_efforts": [...] | None, + "mandatory": bool}`` when the entry advertises reasoning controls, + ``{"supports_reasoning": False}`` when it explicitly does not + (``supported_parameters`` is a list omitting ``reasoning``), or + ``None`` when capability can't be determined from the entry + (missing/malformed ``supported_parameters``). + + Ported from PrimeIntellect-ai/prime-agent#1258 (derive reasoning levels + from provider metadata instead of hardcoded model-family lists). + """ + if not isinstance(item, dict): + return None + params = item.get("supported_parameters") + if not isinstance(params, list): + # Field absent / malformed — unknown capability (mirror the + # permissive stance of _openrouter_model_supports_tools). + return None + if "reasoning" not in params: + return {"supports_reasoning": False} + reasoning = item.get("reasoning") + mandatory = isinstance(reasoning, dict) and reasoning.get("mandatory") is True + efforts: Optional[list[str]] = None + if isinstance(reasoning, dict): + raw_efforts = reasoning.get("supported_efforts") + if isinstance(raw_efforts, list): + efforts = list(dict.fromkeys( + str(effort).strip().lower() + for effort in raw_efforts + if str(effort).strip() + )) + return { + "supports_reasoning": True, + "supported_efforts": efforts, + "mandatory": mandatory, + } + + +# model id → parsed reasoning capabilities (see +# parse_openrouter_reasoning_capabilities). Populated by one full-catalog +# fetch and kept for the process lifetime — model capabilities don't change. +_openrouter_reasoning_caps_cache: dict[str, Optional[dict[str, Any]]] | None = None +# monotonic timestamp of the last FAILED fetch; suppresses re-fetch storms +# from per-turn callers while the catalog is unreachable (60s TTL, mirrors +# the LM Studio/Ollama capability-probe caching in run_agent.py). +_openrouter_reasoning_caps_failed_at: float | None = None + + +def _fetch_openrouter_reasoning_caps(timeout: float = 6.0) -> Optional[dict[str, Optional[dict[str, Any]]]]: + """Fetch + cache per-model reasoning capabilities from the live catalog. + + Returns None (without poisoning the cache) when the catalog is + unreachable so callers can retry later and fall back in the meantime. + Failed fetches are remembered for 60 seconds so hot per-turn callers + don't pay an HTTP round-trip on every call while offline. + """ + global _openrouter_reasoning_caps_cache, _openrouter_reasoning_caps_failed_at + if _openrouter_reasoning_caps_cache is not None: + return _openrouter_reasoning_caps_cache + if ( + _openrouter_reasoning_caps_failed_at is not None + and (time.monotonic() - _openrouter_reasoning_caps_failed_at) < 60 + ): + return None + try: + req = urllib.request.Request( + "https://openrouter.ai/api/v1/models", + headers={"Accept": "application/json"}, + ) + with _urlopen_model_catalog_request(req, timeout=timeout) as resp: + payload = json.loads(resp.read().decode()) + except Exception: + _openrouter_reasoning_caps_failed_at = time.monotonic() + return None + items = payload.get("data") + if not isinstance(items, list): + _openrouter_reasoning_caps_failed_at = time.monotonic() + return None + caps_by_id: dict[str, Optional[dict[str, Any]]] = {} + for item in items: + if not isinstance(item, dict): + continue + mid = str(item.get("id") or "").strip() + if not mid: + continue + caps_by_id[mid] = parse_openrouter_reasoning_capabilities(item) + if not caps_by_id: + _openrouter_reasoning_caps_failed_at = time.monotonic() + return None + _openrouter_reasoning_caps_cache = caps_by_id + return caps_by_id + + +def openrouter_model_reasoning_capabilities( + model_id: Optional[str], + *, + timeout: float = 6.0, + allow_fetch: bool = False, +) -> Optional[dict[str, Any]]: + """Return live-catalog reasoning capabilities for an OpenRouter model. + + Tri-state contract for callers deciding whether to emit reasoning + controls: + - dict with ``supports_reasoning: True`` (+ ``supported_efforts``, + ``mandatory``) — the route advertises reasoning controls; + - dict with ``supports_reasoning: False`` — the catalog knows the model + and it does NOT accept reasoning controls (definitive negative); + - ``None`` — unknown: catalog not loaded yet, model not listed + (private/custom route), or entry malformed. Callers should fall back + to their static heuristics rather than treating this as a negative. + + By default this is a CACHE-ONLY lookup — safe on per-request hot paths + (never blocks on HTTP). The cache is populated for free whenever + ``fetch_openrouter_models()`` runs (model picker, setup), by the + non-blocking ``warm_openrouter_reasoning_caps_async()`` warmer, or by + passing ``allow_fetch=True`` from non-latency-sensitive callers. + """ + model = str(model_id or "").strip() + if not model: + return None + caps_by_id = _openrouter_reasoning_caps_cache + if caps_by_id is None and allow_fetch: + caps_by_id = _fetch_openrouter_reasoning_caps(timeout=timeout) + if caps_by_id is None: + return None + return caps_by_id.get(model) + + +_openrouter_caps_warm_started = False + + +def warm_openrouter_reasoning_caps_async() -> None: + """Warm the reasoning-capability cache in a background thread. + + Fire-and-forget: called from hot paths that found the cache cold so the + NEXT call benefits, without ever blocking a turn on HTTP. One warm + attempt per process (the fetch has its own 60s failure TTL). Skipped + under pytest — a mid-suite background fetch would make cache state, and + therefore test behavior, timing-dependent. + """ + global _openrouter_caps_warm_started + if _openrouter_caps_warm_started or _openrouter_reasoning_caps_cache is not None: + return + if os.environ.get("PYTEST_CURRENT_TEST"): + return + _openrouter_caps_warm_started = True + threading.Thread( + target=_fetch_openrouter_reasoning_caps, + name="openrouter-reasoning-caps-warm", + daemon=True, + ).start() + + +# Canonical low→high ordering used for nearest-level clamping. Superset of +# hermes_constants.VALID_REASONING_EFFORTS ("none" included so an explicit +# disable can be clamped too when a provider publishes it as a level). +_REASONING_EFFORT_ORDER = ( + "none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", +) + + +def clamp_reasoning_effort_to_supported( + effort: Optional[str], + supported_efforts: Optional[list[str]], +) -> Optional[str]: + """Clamp a requested reasoning effort to a provider's supported levels. + + Returns the requested effort unchanged when it is supported, when the + supported list is unknown (None/empty), or when the effort isn't a + recognized level (custom providers may use bespoke names — pass through + rather than guess). Otherwise returns the nearest supported level, + preferring the closest LOWER level so a clamp never silently escalates + cost (requesting ``xhigh`` against ``[low, medium, high]`` yields + ``high``; requesting ``minimal`` against ``[low, medium]`` yields + ``low`` because no lower level exists). + + Ported from PrimeIntellect-ai/prime-agent#1258's thinking-level-map + normalization. + """ + requested = str(effort or "").strip().lower() + if not requested or not supported_efforts: + return effort + supported = [ + str(level).strip().lower() + for level in supported_efforts + if str(level).strip().lower() in _REASONING_EFFORT_ORDER + ] + if not supported or requested in supported: + return effort + if requested not in _REASONING_EFFORT_ORDER: + return effort + requested_idx = _REASONING_EFFORT_ORDER.index(requested) + below = [ + level for level in supported + if _REASONING_EFFORT_ORDER.index(level) < requested_idx + ] + if below: + return max(below, key=_REASONING_EFFORT_ORDER.index) + return min(supported, key=_REASONING_EFFORT_ORDER.index) + + def fetch_openrouter_models( timeout: float = 8.0, *, @@ -1549,6 +1761,17 @@ def fetch_openrouter_models( continue live_by_id[mid] = item + # Free warm-up for the reasoning-capability cache: this is the same + # payload _fetch_openrouter_reasoning_caps would fetch, so parse it once + # here and hot-path callers (openrouter_model_reasoning_capabilities) + # never need their own HTTP round-trip. + global _openrouter_reasoning_caps_cache + if _openrouter_reasoning_caps_cache is None and live_by_id: + _openrouter_reasoning_caps_cache = { + mid: parse_openrouter_reasoning_capabilities(item) + for mid, item in live_by_id.items() + } + curated: list[tuple[str, str]] = [] silent_default = get_preferred_silent_default_model("openrouter") for preferred_id in preferred_ids: diff --git a/plugins/model-providers/openrouter/__init__.py b/plugins/model-providers/openrouter/__init__.py index fa927da412707..e2f9ef40e7c96 100644 --- a/plugins/model-providers/openrouter/__init__.py +++ b/plugins/model-providers/openrouter/__init__.py @@ -49,6 +49,43 @@ def _anthropic_reasoning_is_mandatory(model: str | None) -> bool: class OpenRouterProfile(ProviderProfile): """OpenRouter aggregator — provider preferences, reasoning config passthrough.""" + @staticmethod + def _clamp_reasoning_to_catalog(cfg: dict[str, Any], model: str | None) -> dict[str, Any]: + """Clamp ``cfg["effort"]`` to the model's catalog-advertised levels. + + OpenRouter's /v1/models entries publish ``reasoning.supported_efforts`` + per model (ported from PrimeIntellect-ai/prime-agent#1258). Sending an + unsupported effort (e.g. ``ultra`` to a route that stops at ``high``) + yields provider 4xx errors; clamp to the nearest LOWER supported level + instead. No-op when the catalog is unreachable, the model is unlisted, + or no supported_efforts list is published (None = all levels accepted). + """ + effort = cfg.get("effort") + if not effort or cfg.get("enabled") is False: + return cfg + try: + from hermes_cli.models import ( + clamp_reasoning_effort_to_supported, + openrouter_model_reasoning_capabilities, + ) + caps = openrouter_model_reasoning_capabilities(model) + if not caps or not caps.get("supports_reasoning"): + return cfg + clamped = clamp_reasoning_effort_to_supported( + effort, caps.get("supported_efforts") + ) + except Exception: + return cfg + if clamped and clamped != effort: + logger.debug( + "openrouter: clamped reasoning effort %r → %r for %s " + "(catalog supported_efforts=%s)", + effort, clamped, model, caps.get("supported_efforts"), + ) + cfg = dict(cfg) + cfg["effort"] = clamped + return cfg + def fetch_models( self, *, @@ -175,7 +212,9 @@ class OpenRouterProfile(ProviderProfile): if cfg.get("enabled", True) is not False and effort and effort != "none": top_level["verbosity"] = effort elif reasoning_config is not None: - extra_body["reasoning"] = dict(reasoning_config) + extra_body["reasoning"] = self._clamp_reasoning_to_catalog( + dict(reasoning_config), model + ) else: extra_body["reasoning"] = {"enabled": True, "effort": "medium"} diff --git a/run_agent.py b/run_agent.py index 167c02b466f18..b04002e46a7dd 100644 --- a/run_agent.py +++ b/run_agent.py @@ -7248,6 +7248,31 @@ class AIAgent: return False model = (self.model or "").lower() + # Live-catalog metadata first (ported from + # PrimeIntellect-ai/prime-agent#1258): OpenRouter's /v1/models entries + # advertise reasoning support via supported_parameters + a reasoning + # object, which covers every routed vendor without a hand-maintained + # prefix list. The static prefix allowlist below repeatedly went + # stale one vendor at a time (nvidia/ missing → #75386; same class + # as tencent/, xiaomi/ additions before it) — metadata makes new + # vendors work without a code change. One catalog fetch per process, + # cached; unknown (catalog unreachable / unlisted model) falls back + # to the static list. + try: + from hermes_cli.models import ( + openrouter_model_reasoning_capabilities, + warm_openrouter_reasoning_caps_async, + ) + caps = openrouter_model_reasoning_capabilities(self.model) + if caps is None: + # Cache cold (no picker run this process) — warm it in the + # background so subsequent turns get metadata; never block + # this turn on HTTP. + warm_openrouter_reasoning_caps_async() + except Exception: + caps = None + if caps is not None: + return bool(caps.get("supports_reasoning")) reasoning_model_prefixes = ( "deepseek/", "anthropic/", diff --git a/tests/hermes_cli/test_openrouter_reasoning_metadata.py b/tests/hermes_cli/test_openrouter_reasoning_metadata.py new file mode 100644 index 0000000000000..09517e0952fd2 --- /dev/null +++ b/tests/hermes_cli/test_openrouter_reasoning_metadata.py @@ -0,0 +1,268 @@ +"""Tests for OpenRouter reasoning-capability metadata (prime-agent#1258 port). + +Covers: + - parse_openrouter_reasoning_capabilities: catalog-entry normalization + - clamp_reasoning_effort_to_supported: nearest-lower-level clamping + - openrouter_model_reasoning_capabilities: cache + tri-state contract + - AIAgent._supports_reasoning_extra_body: metadata-first, static fallback + - OpenRouterProfile._clamp_reasoning_to_catalog: emitted effort clamping +""" + +import pytest + +from hermes_cli.models import ( + clamp_reasoning_effort_to_supported, + parse_openrouter_reasoning_capabilities, +) + + +class TestParseReasoningCapabilities: + def test_reasoning_supported_with_efforts(self): + item = { + "id": "nvidia/nemotron-3-ultra", + "supported_parameters": ["temperature", "tools", "reasoning"], + "reasoning": {"mandatory": False, "supported_efforts": ["low", "medium", "high"]}, + } + caps = parse_openrouter_reasoning_capabilities(item) + assert caps == { + "supports_reasoning": True, + "supported_efforts": ["low", "medium", "high"], + "mandatory": False, + } + + def test_reasoning_supported_all_efforts_when_field_omitted(self): + item = { + "id": "deepseek/deepseek-chat", + "supported_parameters": ["reasoning", "tools"], + "reasoning": {}, + } + caps = parse_openrouter_reasoning_capabilities(item) + assert caps["supports_reasoning"] is True + assert caps["supported_efforts"] is None # None = every effort accepted + assert caps["mandatory"] is False + + def test_reasoning_supported_without_reasoning_object(self): + # supported_parameters alone is authoritative for the on/off question. + item = {"supported_parameters": ["reasoning"]} + caps = parse_openrouter_reasoning_capabilities(item) + assert caps["supports_reasoning"] is True + assert caps["supported_efforts"] is None + + def test_mandatory_flag(self): + item = { + "supported_parameters": ["reasoning"], + "reasoning": {"mandatory": True, "supported_efforts": None}, + } + caps = parse_openrouter_reasoning_capabilities(item) + assert caps["mandatory"] is True + + def test_reasoning_object_untrusted_without_supported_parameters_entry(self): + # Top-level reasoning object present but supported_parameters omits + # "reasoning" → the route rejects reasoning controls. + item = { + "supported_parameters": ["temperature", "tools"], + "reasoning": {"supported_efforts": ["high"]}, + } + assert parse_openrouter_reasoning_capabilities(item) == { + "supports_reasoning": False + } + + def test_unknown_when_supported_parameters_missing(self): + assert parse_openrouter_reasoning_capabilities({"id": "x"}) is None + assert parse_openrouter_reasoning_capabilities({"supported_parameters": "bad"}) is None + assert parse_openrouter_reasoning_capabilities("not-a-dict") is None + + def test_effort_list_normalized_and_deduped(self): + item = { + "supported_parameters": ["reasoning"], + "reasoning": {"supported_efforts": [" High ", "high", "LOW", "", 3]}, + } + caps = parse_openrouter_reasoning_capabilities(item) + assert caps["supported_efforts"] == ["high", "low", "3"] + + +class TestClampReasoningEffort: + @pytest.mark.parametrize( + "effort,supported,expected", + [ + # Supported as-is → unchanged. + ("high", ["low", "medium", "high"], "high"), + # Unknown supported list → pass through. + ("ultra", None, "ultra"), + ("ultra", [], "ultra"), + # Clamp DOWN to nearest lower supported level. + ("ultra", ["low", "medium", "high"], "high"), + ("xhigh", ["low", "medium", "high"], "high"), + ("max", ["minimal", "low"], "low"), + # No lower level exists → nearest (lowest) supported. + ("minimal", ["low", "medium"], "low"), + ("none", ["low", "high"], "low"), + # Unrecognized custom level → pass through untouched. + ("turbo-think", ["low", "high"], "turbo-think"), + # Supported list containing only unrecognized names → pass through. + ("high", ["banana"], "high"), + # Empty/None effort → pass through. + (None, ["low"], None), + ("", ["low"], ""), + ], + ) + def test_clamping(self, effort, supported, expected): + assert clamp_reasoning_effort_to_supported(effort, supported) == expected + + def test_never_escalates(self): + # A clamp must never pick a HIGHER level when a lower one exists. + assert clamp_reasoning_effort_to_supported("medium", ["low", "xhigh"]) == "low" + + +class TestOpenRouterModelReasoningCapabilities: + def _prime_cache(self, monkeypatch, caps_by_id): + import hermes_cli.models as models_mod + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", caps_by_id) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + + def test_known_model(self, monkeypatch): + from hermes_cli.models import openrouter_model_reasoning_capabilities + self._prime_cache(monkeypatch, { + "nvidia/nemotron-3-ultra": { + "supports_reasoning": True, + "supported_efforts": ["low", "high"], + "mandatory": False, + }, + }) + caps = openrouter_model_reasoning_capabilities("nvidia/nemotron-3-ultra") + assert caps["supports_reasoning"] is True + + def test_unlisted_model_returns_none(self, monkeypatch): + from hermes_cli.models import openrouter_model_reasoning_capabilities + self._prime_cache(monkeypatch, {"a/b": {"supports_reasoning": True}}) + assert openrouter_model_reasoning_capabilities("private/custom") is None + + def test_empty_model_returns_none(self, monkeypatch): + from hermes_cli.models import openrouter_model_reasoning_capabilities + self._prime_cache(monkeypatch, {"a/b": {"supports_reasoning": True}}) + assert openrouter_model_reasoning_capabilities("") is None + assert openrouter_model_reasoning_capabilities(None) is None + + def test_catalog_unreachable_returns_none_and_rate_limits(self, monkeypatch): + import hermes_cli.models as models_mod + from hermes_cli.models import openrouter_model_reasoning_capabilities + + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", None) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + calls = {"n": 0} + + def _boom(req, *, timeout): + calls["n"] += 1 + raise OSError("offline") + + monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _boom) + assert openrouter_model_reasoning_capabilities("a/b", allow_fetch=True) is None + assert openrouter_model_reasoning_capabilities("a/b", allow_fetch=True) is None + # Second call inside the 60s failure TTL must not re-fetch. + assert calls["n"] == 1 + + def test_cache_only_by_default_never_fetches(self, monkeypatch): + import hermes_cli.models as models_mod + from hermes_cli.models import openrouter_model_reasoning_capabilities + + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", None) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + + def _boom(req, *, timeout): + raise AssertionError("hot path must not fetch") + + monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _boom) + # Default (allow_fetch=False) → cache-only, no HTTP even when cold. + assert openrouter_model_reasoning_capabilities("a/b") is None + + +class TestSupportsReasoningExtraBodyMetadataGate: + """AIAgent._supports_reasoning_extra_body: metadata-first with fallback.""" + + def _make_agent(self, model): + from run_agent import AIAgent + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model=model, + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + return agent + + def test_metadata_positive_overrides_static_list(self, monkeypatch): + import hermes_cli.models as models_mod + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", { + # nvidia/ is NOT in the static prefix allowlist (#75386) — + # metadata must make it work without a code change. + "nvidia/nemotron-3-ultra": { + "supports_reasoning": True, + "supported_efforts": ["low", "medium", "high"], + "mandatory": False, + }, + }) + agent = self._make_agent("nvidia/nemotron-3-ultra") + assert agent._supports_reasoning_extra_body() is True + + def test_metadata_negative_overrides_static_list(self, monkeypatch): + import hermes_cli.models as models_mod + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", { + # openai/ IS in the static prefix list, but the catalog says this + # route rejects reasoning controls — the definitive negative wins. + "openai/gpt-4o-mini": {"supports_reasoning": False}, + }) + agent = self._make_agent("openai/gpt-4o-mini") + assert agent._supports_reasoning_extra_body() is False + + def test_unknown_falls_back_to_static_prefixes(self, monkeypatch): + import hermes_cli.models as models_mod + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {"a/b": None}) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + # deepseek/ is in the static list; unlisted in catalog → fallback True. + agent = self._make_agent("deepseek/deepseek-chat") + assert agent._supports_reasoning_extra_body() is True + # unknown vendor absent from both → False. + agent2 = self._make_agent("someveryunknown/model-x") + assert agent2._supports_reasoning_extra_body() is False + + +class TestOpenRouterProfileClamp: + def test_clamp_applied_in_build_api_kwargs_extras(self, monkeypatch): + import hermes_cli.models as models_mod + from providers import get_provider_profile + + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", { + "qwen/qwen3-plus": { + "supports_reasoning": True, + "supported_efforts": ["low", "medium", "high"], + "mandatory": False, + }, + }) + profile = get_provider_profile("openrouter") + assert profile is not None + extra_body, _top = profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "ultra"}, + supports_reasoning=True, + model="qwen/qwen3-plus", + ) + assert extra_body["reasoning"]["effort"] == "high" + + def test_no_clamp_when_catalog_unknown(self, monkeypatch): + import hermes_cli.models as models_mod + from providers import get_provider_profile + + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {"a/b": None}) + monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None) + profile = get_provider_profile("openrouter") + assert profile is not None + extra_body, _top = profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "ultra"}, + supports_reasoning=True, + model="unlisted/model", + ) + # Unknown capability → passthrough unchanged (no silent downgrade). + assert extra_body["reasoning"]["effort"] == "ultra" diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 6a5a36dd83bed..1ea0f2ac23db4 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1535,6 +1535,18 @@ on its own adaptive default. The native Anthropic provider already controls effort directly and is unaffected. ::: +:::note OpenRouter models and supported effort levels +For other models routed through OpenRouter, Hermes reads the live model +catalog's reasoning metadata (`supported_parameters` + per-model +`reasoning.supported_efforts`) to decide whether to send reasoning controls at +all and to clamp your requested effort to the nearest level the route actually +supports (always downward — e.g. `ultra` becomes `high` on a route that stops +at `high`, never a silent escalation). New reasoning-capable vendors work +automatically without waiting for a Hermes update; when the catalog is +unreachable or a model isn't listed, Hermes falls back to its built-in +model-family list and passes your effort through unchanged. +::: + You can also change the reasoning effort at runtime with the `/reasoning` command: ```