Port from PrimeIntellect-ai/prime-agent#1258: derive OpenRouter reasoning support and effort levels from catalog metadata
OpenRouter's /v1/models entries advertise reasoning capability (supported_parameters + reasoning.mandatory/supported_efforts). Use that metadata as the primary gate in _supports_reasoning_extra_body instead of the hand-maintained vendor-prefix allowlist, which went stale one vendor at a time (nvidia/ missing -> #75386). Also clamp the requested effort to the nearest LOWER catalog-supported level in the OpenRouter profile so ultra/max against a high-capped route no longer 4xxes. Cache-only on the hot path: capabilities parse for free out of the existing fetch_openrouter_models() payload, a background warmer covers cold starts, and unknown models/offline catalogs fall back to the static prefix list unchanged.
This commit is contained in:
Teknium
parent
25363985e9
commit
314968f5fb
|
|
@ -1503,6 +1503,218 @@ def _openrouter_model_supports_tools(item: Any) -> bool:
|
|||
return "tools" in params
|
||||
|
||||
|
||||
def parse_openrouter_reasoning_capabilities(item: Any) -> Optional[dict[str, Any]]:
|
||||
"""Normalize one OpenRouter catalog entry's reasoning metadata.
|
||||
|
||||
OpenRouter's ``/v1/models`` catalog advertises reasoning support two ways:
|
||||
``supported_parameters`` contains ``"reasoning"`` when the route accepts
|
||||
reasoning controls at all, and a top-level ``reasoning`` object may add
|
||||
detail (``mandatory``, ``supported_efforts``). Per OpenRouter semantics
|
||||
the top-level object is only trusted after ``supported_parameters``
|
||||
confirms the route accepts reasoning controls; ``supported_efforts``
|
||||
omitted/None means every effort is accepted.
|
||||
|
||||
Returns:
|
||||
``{"supports_reasoning": True, "supported_efforts": [...] | None,
|
||||
"mandatory": bool}`` when the entry advertises reasoning controls,
|
||||
``{"supports_reasoning": False}`` when it explicitly does not
|
||||
(``supported_parameters`` is a list omitting ``reasoning``), or
|
||||
``None`` when capability can't be determined from the entry
|
||||
(missing/malformed ``supported_parameters``).
|
||||
|
||||
Ported from PrimeIntellect-ai/prime-agent#1258 (derive reasoning levels
|
||||
from provider metadata instead of hardcoded model-family lists).
|
||||
"""
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
params = item.get("supported_parameters")
|
||||
if not isinstance(params, list):
|
||||
# Field absent / malformed — unknown capability (mirror the
|
||||
# permissive stance of _openrouter_model_supports_tools).
|
||||
return None
|
||||
if "reasoning" not in params:
|
||||
return {"supports_reasoning": False}
|
||||
reasoning = item.get("reasoning")
|
||||
mandatory = isinstance(reasoning, dict) and reasoning.get("mandatory") is True
|
||||
efforts: Optional[list[str]] = None
|
||||
if isinstance(reasoning, dict):
|
||||
raw_efforts = reasoning.get("supported_efforts")
|
||||
if isinstance(raw_efforts, list):
|
||||
efforts = list(dict.fromkeys(
|
||||
str(effort).strip().lower()
|
||||
for effort in raw_efforts
|
||||
if str(effort).strip()
|
||||
))
|
||||
return {
|
||||
"supports_reasoning": True,
|
||||
"supported_efforts": efforts,
|
||||
"mandatory": mandatory,
|
||||
}
|
||||
|
||||
|
||||
# model id → parsed reasoning capabilities (see
|
||||
# parse_openrouter_reasoning_capabilities). Populated by one full-catalog
|
||||
# fetch and kept for the process lifetime — model capabilities don't change.
|
||||
_openrouter_reasoning_caps_cache: dict[str, Optional[dict[str, Any]]] | None = None
|
||||
# monotonic timestamp of the last FAILED fetch; suppresses re-fetch storms
|
||||
# from per-turn callers while the catalog is unreachable (60s TTL, mirrors
|
||||
# the LM Studio/Ollama capability-probe caching in run_agent.py).
|
||||
_openrouter_reasoning_caps_failed_at: float | None = None
|
||||
|
||||
|
||||
def _fetch_openrouter_reasoning_caps(timeout: float = 6.0) -> Optional[dict[str, Optional[dict[str, Any]]]]:
|
||||
"""Fetch + cache per-model reasoning capabilities from the live catalog.
|
||||
|
||||
Returns None (without poisoning the cache) when the catalog is
|
||||
unreachable so callers can retry later and fall back in the meantime.
|
||||
Failed fetches are remembered for 60 seconds so hot per-turn callers
|
||||
don't pay an HTTP round-trip on every call while offline.
|
||||
"""
|
||||
global _openrouter_reasoning_caps_cache, _openrouter_reasoning_caps_failed_at
|
||||
if _openrouter_reasoning_caps_cache is not None:
|
||||
return _openrouter_reasoning_caps_cache
|
||||
if (
|
||||
_openrouter_reasoning_caps_failed_at is not None
|
||||
and (time.monotonic() - _openrouter_reasoning_caps_failed_at) < 60
|
||||
):
|
||||
return None
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
"https://openrouter.ai/api/v1/models",
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
with _urlopen_model_catalog_request(req, timeout=timeout) as resp:
|
||||
payload = json.loads(resp.read().decode())
|
||||
except Exception:
|
||||
_openrouter_reasoning_caps_failed_at = time.monotonic()
|
||||
return None
|
||||
items = payload.get("data")
|
||||
if not isinstance(items, list):
|
||||
_openrouter_reasoning_caps_failed_at = time.monotonic()
|
||||
return None
|
||||
caps_by_id: dict[str, Optional[dict[str, Any]]] = {}
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
mid = str(item.get("id") or "").strip()
|
||||
if not mid:
|
||||
continue
|
||||
caps_by_id[mid] = parse_openrouter_reasoning_capabilities(item)
|
||||
if not caps_by_id:
|
||||
_openrouter_reasoning_caps_failed_at = time.monotonic()
|
||||
return None
|
||||
_openrouter_reasoning_caps_cache = caps_by_id
|
||||
return caps_by_id
|
||||
|
||||
|
||||
def openrouter_model_reasoning_capabilities(
|
||||
model_id: Optional[str],
|
||||
*,
|
||||
timeout: float = 6.0,
|
||||
allow_fetch: bool = False,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Return live-catalog reasoning capabilities for an OpenRouter model.
|
||||
|
||||
Tri-state contract for callers deciding whether to emit reasoning
|
||||
controls:
|
||||
- dict with ``supports_reasoning: True`` (+ ``supported_efforts``,
|
||||
``mandatory``) — the route advertises reasoning controls;
|
||||
- dict with ``supports_reasoning: False`` — the catalog knows the model
|
||||
and it does NOT accept reasoning controls (definitive negative);
|
||||
- ``None`` — unknown: catalog not loaded yet, model not listed
|
||||
(private/custom route), or entry malformed. Callers should fall back
|
||||
to their static heuristics rather than treating this as a negative.
|
||||
|
||||
By default this is a CACHE-ONLY lookup — safe on per-request hot paths
|
||||
(never blocks on HTTP). The cache is populated for free whenever
|
||||
``fetch_openrouter_models()`` runs (model picker, setup), by the
|
||||
non-blocking ``warm_openrouter_reasoning_caps_async()`` warmer, or by
|
||||
passing ``allow_fetch=True`` from non-latency-sensitive callers.
|
||||
"""
|
||||
model = str(model_id or "").strip()
|
||||
if not model:
|
||||
return None
|
||||
caps_by_id = _openrouter_reasoning_caps_cache
|
||||
if caps_by_id is None and allow_fetch:
|
||||
caps_by_id = _fetch_openrouter_reasoning_caps(timeout=timeout)
|
||||
if caps_by_id is None:
|
||||
return None
|
||||
return caps_by_id.get(model)
|
||||
|
||||
|
||||
_openrouter_caps_warm_started = False
|
||||
|
||||
|
||||
def warm_openrouter_reasoning_caps_async() -> None:
|
||||
"""Warm the reasoning-capability cache in a background thread.
|
||||
|
||||
Fire-and-forget: called from hot paths that found the cache cold so the
|
||||
NEXT call benefits, without ever blocking a turn on HTTP. One warm
|
||||
attempt per process (the fetch has its own 60s failure TTL). Skipped
|
||||
under pytest — a mid-suite background fetch would make cache state, and
|
||||
therefore test behavior, timing-dependent.
|
||||
"""
|
||||
global _openrouter_caps_warm_started
|
||||
if _openrouter_caps_warm_started or _openrouter_reasoning_caps_cache is not None:
|
||||
return
|
||||
if os.environ.get("PYTEST_CURRENT_TEST"):
|
||||
return
|
||||
_openrouter_caps_warm_started = True
|
||||
threading.Thread(
|
||||
target=_fetch_openrouter_reasoning_caps,
|
||||
name="openrouter-reasoning-caps-warm",
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
|
||||
# Canonical low→high ordering used for nearest-level clamping. Superset of
|
||||
# hermes_constants.VALID_REASONING_EFFORTS ("none" included so an explicit
|
||||
# disable can be clamped too when a provider publishes it as a level).
|
||||
_REASONING_EFFORT_ORDER = (
|
||||
"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra",
|
||||
)
|
||||
|
||||
|
||||
def clamp_reasoning_effort_to_supported(
|
||||
effort: Optional[str],
|
||||
supported_efforts: Optional[list[str]],
|
||||
) -> Optional[str]:
|
||||
"""Clamp a requested reasoning effort to a provider's supported levels.
|
||||
|
||||
Returns the requested effort unchanged when it is supported, when the
|
||||
supported list is unknown (None/empty), or when the effort isn't a
|
||||
recognized level (custom providers may use bespoke names — pass through
|
||||
rather than guess). Otherwise returns the nearest supported level,
|
||||
preferring the closest LOWER level so a clamp never silently escalates
|
||||
cost (requesting ``xhigh`` against ``[low, medium, high]`` yields
|
||||
``high``; requesting ``minimal`` against ``[low, medium]`` yields
|
||||
``low`` because no lower level exists).
|
||||
|
||||
Ported from PrimeIntellect-ai/prime-agent#1258's thinking-level-map
|
||||
normalization.
|
||||
"""
|
||||
requested = str(effort or "").strip().lower()
|
||||
if not requested or not supported_efforts:
|
||||
return effort
|
||||
supported = [
|
||||
str(level).strip().lower()
|
||||
for level in supported_efforts
|
||||
if str(level).strip().lower() in _REASONING_EFFORT_ORDER
|
||||
]
|
||||
if not supported or requested in supported:
|
||||
return effort
|
||||
if requested not in _REASONING_EFFORT_ORDER:
|
||||
return effort
|
||||
requested_idx = _REASONING_EFFORT_ORDER.index(requested)
|
||||
below = [
|
||||
level for level in supported
|
||||
if _REASONING_EFFORT_ORDER.index(level) < requested_idx
|
||||
]
|
||||
if below:
|
||||
return max(below, key=_REASONING_EFFORT_ORDER.index)
|
||||
return min(supported, key=_REASONING_EFFORT_ORDER.index)
|
||||
|
||||
|
||||
def fetch_openrouter_models(
|
||||
timeout: float = 8.0,
|
||||
*,
|
||||
|
|
@ -1549,6 +1761,17 @@ def fetch_openrouter_models(
|
|||
continue
|
||||
live_by_id[mid] = item
|
||||
|
||||
# Free warm-up for the reasoning-capability cache: this is the same
|
||||
# payload _fetch_openrouter_reasoning_caps would fetch, so parse it once
|
||||
# here and hot-path callers (openrouter_model_reasoning_capabilities)
|
||||
# never need their own HTTP round-trip.
|
||||
global _openrouter_reasoning_caps_cache
|
||||
if _openrouter_reasoning_caps_cache is None and live_by_id:
|
||||
_openrouter_reasoning_caps_cache = {
|
||||
mid: parse_openrouter_reasoning_capabilities(item)
|
||||
for mid, item in live_by_id.items()
|
||||
}
|
||||
|
||||
curated: list[tuple[str, str]] = []
|
||||
silent_default = get_preferred_silent_default_model("openrouter")
|
||||
for preferred_id in preferred_ids:
|
||||
|
|
|
|||
|
|
@ -49,6 +49,43 @@ def _anthropic_reasoning_is_mandatory(model: str | None) -> bool:
|
|||
class OpenRouterProfile(ProviderProfile):
|
||||
"""OpenRouter aggregator — provider preferences, reasoning config passthrough."""
|
||||
|
||||
@staticmethod
|
||||
def _clamp_reasoning_to_catalog(cfg: dict[str, Any], model: str | None) -> dict[str, Any]:
|
||||
"""Clamp ``cfg["effort"]`` to the model's catalog-advertised levels.
|
||||
|
||||
OpenRouter's /v1/models entries publish ``reasoning.supported_efforts``
|
||||
per model (ported from PrimeIntellect-ai/prime-agent#1258). Sending an
|
||||
unsupported effort (e.g. ``ultra`` to a route that stops at ``high``)
|
||||
yields provider 4xx errors; clamp to the nearest LOWER supported level
|
||||
instead. No-op when the catalog is unreachable, the model is unlisted,
|
||||
or no supported_efforts list is published (None = all levels accepted).
|
||||
"""
|
||||
effort = cfg.get("effort")
|
||||
if not effort or cfg.get("enabled") is False:
|
||||
return cfg
|
||||
try:
|
||||
from hermes_cli.models import (
|
||||
clamp_reasoning_effort_to_supported,
|
||||
openrouter_model_reasoning_capabilities,
|
||||
)
|
||||
caps = openrouter_model_reasoning_capabilities(model)
|
||||
if not caps or not caps.get("supports_reasoning"):
|
||||
return cfg
|
||||
clamped = clamp_reasoning_effort_to_supported(
|
||||
effort, caps.get("supported_efforts")
|
||||
)
|
||||
except Exception:
|
||||
return cfg
|
||||
if clamped and clamped != effort:
|
||||
logger.debug(
|
||||
"openrouter: clamped reasoning effort %r → %r for %s "
|
||||
"(catalog supported_efforts=%s)",
|
||||
effort, clamped, model, caps.get("supported_efforts"),
|
||||
)
|
||||
cfg = dict(cfg)
|
||||
cfg["effort"] = clamped
|
||||
return cfg
|
||||
|
||||
def fetch_models(
|
||||
self,
|
||||
*,
|
||||
|
|
@ -175,7 +212,9 @@ class OpenRouterProfile(ProviderProfile):
|
|||
if cfg.get("enabled", True) is not False and effort and effort != "none":
|
||||
top_level["verbosity"] = effort
|
||||
elif reasoning_config is not None:
|
||||
extra_body["reasoning"] = dict(reasoning_config)
|
||||
extra_body["reasoning"] = self._clamp_reasoning_to_catalog(
|
||||
dict(reasoning_config), model
|
||||
)
|
||||
else:
|
||||
extra_body["reasoning"] = {"enabled": True, "effort": "medium"}
|
||||
|
||||
|
|
|
|||
25
run_agent.py
25
run_agent.py
|
|
@ -7248,6 +7248,31 @@ class AIAgent:
|
|||
return False
|
||||
|
||||
model = (self.model or "").lower()
|
||||
# Live-catalog metadata first (ported from
|
||||
# PrimeIntellect-ai/prime-agent#1258): OpenRouter's /v1/models entries
|
||||
# advertise reasoning support via supported_parameters + a reasoning
|
||||
# object, which covers every routed vendor without a hand-maintained
|
||||
# prefix list. The static prefix allowlist below repeatedly went
|
||||
# stale one vendor at a time (nvidia/ missing → #75386; same class
|
||||
# as tencent/, xiaomi/ additions before it) — metadata makes new
|
||||
# vendors work without a code change. One catalog fetch per process,
|
||||
# cached; unknown (catalog unreachable / unlisted model) falls back
|
||||
# to the static list.
|
||||
try:
|
||||
from hermes_cli.models import (
|
||||
openrouter_model_reasoning_capabilities,
|
||||
warm_openrouter_reasoning_caps_async,
|
||||
)
|
||||
caps = openrouter_model_reasoning_capabilities(self.model)
|
||||
if caps is None:
|
||||
# Cache cold (no picker run this process) — warm it in the
|
||||
# background so subsequent turns get metadata; never block
|
||||
# this turn on HTTP.
|
||||
warm_openrouter_reasoning_caps_async()
|
||||
except Exception:
|
||||
caps = None
|
||||
if caps is not None:
|
||||
return bool(caps.get("supports_reasoning"))
|
||||
reasoning_model_prefixes = (
|
||||
"deepseek/",
|
||||
"anthropic/",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,268 @@
|
|||
"""Tests for OpenRouter reasoning-capability metadata (prime-agent#1258 port).
|
||||
|
||||
Covers:
|
||||
- parse_openrouter_reasoning_capabilities: catalog-entry normalization
|
||||
- clamp_reasoning_effort_to_supported: nearest-lower-level clamping
|
||||
- openrouter_model_reasoning_capabilities: cache + tri-state contract
|
||||
- AIAgent._supports_reasoning_extra_body: metadata-first, static fallback
|
||||
- OpenRouterProfile._clamp_reasoning_to_catalog: emitted effort clamping
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.models import (
|
||||
clamp_reasoning_effort_to_supported,
|
||||
parse_openrouter_reasoning_capabilities,
|
||||
)
|
||||
|
||||
|
||||
class TestParseReasoningCapabilities:
|
||||
def test_reasoning_supported_with_efforts(self):
|
||||
item = {
|
||||
"id": "nvidia/nemotron-3-ultra",
|
||||
"supported_parameters": ["temperature", "tools", "reasoning"],
|
||||
"reasoning": {"mandatory": False, "supported_efforts": ["low", "medium", "high"]},
|
||||
}
|
||||
caps = parse_openrouter_reasoning_capabilities(item)
|
||||
assert caps == {
|
||||
"supports_reasoning": True,
|
||||
"supported_efforts": ["low", "medium", "high"],
|
||||
"mandatory": False,
|
||||
}
|
||||
|
||||
def test_reasoning_supported_all_efforts_when_field_omitted(self):
|
||||
item = {
|
||||
"id": "deepseek/deepseek-chat",
|
||||
"supported_parameters": ["reasoning", "tools"],
|
||||
"reasoning": {},
|
||||
}
|
||||
caps = parse_openrouter_reasoning_capabilities(item)
|
||||
assert caps["supports_reasoning"] is True
|
||||
assert caps["supported_efforts"] is None # None = every effort accepted
|
||||
assert caps["mandatory"] is False
|
||||
|
||||
def test_reasoning_supported_without_reasoning_object(self):
|
||||
# supported_parameters alone is authoritative for the on/off question.
|
||||
item = {"supported_parameters": ["reasoning"]}
|
||||
caps = parse_openrouter_reasoning_capabilities(item)
|
||||
assert caps["supports_reasoning"] is True
|
||||
assert caps["supported_efforts"] is None
|
||||
|
||||
def test_mandatory_flag(self):
|
||||
item = {
|
||||
"supported_parameters": ["reasoning"],
|
||||
"reasoning": {"mandatory": True, "supported_efforts": None},
|
||||
}
|
||||
caps = parse_openrouter_reasoning_capabilities(item)
|
||||
assert caps["mandatory"] is True
|
||||
|
||||
def test_reasoning_object_untrusted_without_supported_parameters_entry(self):
|
||||
# Top-level reasoning object present but supported_parameters omits
|
||||
# "reasoning" → the route rejects reasoning controls.
|
||||
item = {
|
||||
"supported_parameters": ["temperature", "tools"],
|
||||
"reasoning": {"supported_efforts": ["high"]},
|
||||
}
|
||||
assert parse_openrouter_reasoning_capabilities(item) == {
|
||||
"supports_reasoning": False
|
||||
}
|
||||
|
||||
def test_unknown_when_supported_parameters_missing(self):
|
||||
assert parse_openrouter_reasoning_capabilities({"id": "x"}) is None
|
||||
assert parse_openrouter_reasoning_capabilities({"supported_parameters": "bad"}) is None
|
||||
assert parse_openrouter_reasoning_capabilities("not-a-dict") is None
|
||||
|
||||
def test_effort_list_normalized_and_deduped(self):
|
||||
item = {
|
||||
"supported_parameters": ["reasoning"],
|
||||
"reasoning": {"supported_efforts": [" High ", "high", "LOW", "", 3]},
|
||||
}
|
||||
caps = parse_openrouter_reasoning_capabilities(item)
|
||||
assert caps["supported_efforts"] == ["high", "low", "3"]
|
||||
|
||||
|
||||
class TestClampReasoningEffort:
|
||||
@pytest.mark.parametrize(
|
||||
"effort,supported,expected",
|
||||
[
|
||||
# Supported as-is → unchanged.
|
||||
("high", ["low", "medium", "high"], "high"),
|
||||
# Unknown supported list → pass through.
|
||||
("ultra", None, "ultra"),
|
||||
("ultra", [], "ultra"),
|
||||
# Clamp DOWN to nearest lower supported level.
|
||||
("ultra", ["low", "medium", "high"], "high"),
|
||||
("xhigh", ["low", "medium", "high"], "high"),
|
||||
("max", ["minimal", "low"], "low"),
|
||||
# No lower level exists → nearest (lowest) supported.
|
||||
("minimal", ["low", "medium"], "low"),
|
||||
("none", ["low", "high"], "low"),
|
||||
# Unrecognized custom level → pass through untouched.
|
||||
("turbo-think", ["low", "high"], "turbo-think"),
|
||||
# Supported list containing only unrecognized names → pass through.
|
||||
("high", ["banana"], "high"),
|
||||
# Empty/None effort → pass through.
|
||||
(None, ["low"], None),
|
||||
("", ["low"], ""),
|
||||
],
|
||||
)
|
||||
def test_clamping(self, effort, supported, expected):
|
||||
assert clamp_reasoning_effort_to_supported(effort, supported) == expected
|
||||
|
||||
def test_never_escalates(self):
|
||||
# A clamp must never pick a HIGHER level when a lower one exists.
|
||||
assert clamp_reasoning_effort_to_supported("medium", ["low", "xhigh"]) == "low"
|
||||
|
||||
|
||||
class TestOpenRouterModelReasoningCapabilities:
|
||||
def _prime_cache(self, monkeypatch, caps_by_id):
|
||||
import hermes_cli.models as models_mod
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", caps_by_id)
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
|
||||
def test_known_model(self, monkeypatch):
|
||||
from hermes_cli.models import openrouter_model_reasoning_capabilities
|
||||
self._prime_cache(monkeypatch, {
|
||||
"nvidia/nemotron-3-ultra": {
|
||||
"supports_reasoning": True,
|
||||
"supported_efforts": ["low", "high"],
|
||||
"mandatory": False,
|
||||
},
|
||||
})
|
||||
caps = openrouter_model_reasoning_capabilities("nvidia/nemotron-3-ultra")
|
||||
assert caps["supports_reasoning"] is True
|
||||
|
||||
def test_unlisted_model_returns_none(self, monkeypatch):
|
||||
from hermes_cli.models import openrouter_model_reasoning_capabilities
|
||||
self._prime_cache(monkeypatch, {"a/b": {"supports_reasoning": True}})
|
||||
assert openrouter_model_reasoning_capabilities("private/custom") is None
|
||||
|
||||
def test_empty_model_returns_none(self, monkeypatch):
|
||||
from hermes_cli.models import openrouter_model_reasoning_capabilities
|
||||
self._prime_cache(monkeypatch, {"a/b": {"supports_reasoning": True}})
|
||||
assert openrouter_model_reasoning_capabilities("") is None
|
||||
assert openrouter_model_reasoning_capabilities(None) is None
|
||||
|
||||
def test_catalog_unreachable_returns_none_and_rate_limits(self, monkeypatch):
|
||||
import hermes_cli.models as models_mod
|
||||
from hermes_cli.models import openrouter_model_reasoning_capabilities
|
||||
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", None)
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
calls = {"n": 0}
|
||||
|
||||
def _boom(req, *, timeout):
|
||||
calls["n"] += 1
|
||||
raise OSError("offline")
|
||||
|
||||
monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _boom)
|
||||
assert openrouter_model_reasoning_capabilities("a/b", allow_fetch=True) is None
|
||||
assert openrouter_model_reasoning_capabilities("a/b", allow_fetch=True) is None
|
||||
# Second call inside the 60s failure TTL must not re-fetch.
|
||||
assert calls["n"] == 1
|
||||
|
||||
def test_cache_only_by_default_never_fetches(self, monkeypatch):
|
||||
import hermes_cli.models as models_mod
|
||||
from hermes_cli.models import openrouter_model_reasoning_capabilities
|
||||
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", None)
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
|
||||
def _boom(req, *, timeout):
|
||||
raise AssertionError("hot path must not fetch")
|
||||
|
||||
monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _boom)
|
||||
# Default (allow_fetch=False) → cache-only, no HTTP even when cold.
|
||||
assert openrouter_model_reasoning_capabilities("a/b") is None
|
||||
|
||||
|
||||
class TestSupportsReasoningExtraBodyMetadataGate:
|
||||
"""AIAgent._supports_reasoning_extra_body: metadata-first with fallback."""
|
||||
|
||||
def _make_agent(self, model):
|
||||
from run_agent import AIAgent
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
model=model,
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
return agent
|
||||
|
||||
def test_metadata_positive_overrides_static_list(self, monkeypatch):
|
||||
import hermes_cli.models as models_mod
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {
|
||||
# nvidia/ is NOT in the static prefix allowlist (#75386) —
|
||||
# metadata must make it work without a code change.
|
||||
"nvidia/nemotron-3-ultra": {
|
||||
"supports_reasoning": True,
|
||||
"supported_efforts": ["low", "medium", "high"],
|
||||
"mandatory": False,
|
||||
},
|
||||
})
|
||||
agent = self._make_agent("nvidia/nemotron-3-ultra")
|
||||
assert agent._supports_reasoning_extra_body() is True
|
||||
|
||||
def test_metadata_negative_overrides_static_list(self, monkeypatch):
|
||||
import hermes_cli.models as models_mod
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {
|
||||
# openai/ IS in the static prefix list, but the catalog says this
|
||||
# route rejects reasoning controls — the definitive negative wins.
|
||||
"openai/gpt-4o-mini": {"supports_reasoning": False},
|
||||
})
|
||||
agent = self._make_agent("openai/gpt-4o-mini")
|
||||
assert agent._supports_reasoning_extra_body() is False
|
||||
|
||||
def test_unknown_falls_back_to_static_prefixes(self, monkeypatch):
|
||||
import hermes_cli.models as models_mod
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {"a/b": None})
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
# deepseek/ is in the static list; unlisted in catalog → fallback True.
|
||||
agent = self._make_agent("deepseek/deepseek-chat")
|
||||
assert agent._supports_reasoning_extra_body() is True
|
||||
# unknown vendor absent from both → False.
|
||||
agent2 = self._make_agent("someveryunknown/model-x")
|
||||
assert agent2._supports_reasoning_extra_body() is False
|
||||
|
||||
|
||||
class TestOpenRouterProfileClamp:
|
||||
def test_clamp_applied_in_build_api_kwargs_extras(self, monkeypatch):
|
||||
import hermes_cli.models as models_mod
|
||||
from providers import get_provider_profile
|
||||
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {
|
||||
"qwen/qwen3-plus": {
|
||||
"supports_reasoning": True,
|
||||
"supported_efforts": ["low", "medium", "high"],
|
||||
"mandatory": False,
|
||||
},
|
||||
})
|
||||
profile = get_provider_profile("openrouter")
|
||||
assert profile is not None
|
||||
extra_body, _top = profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "ultra"},
|
||||
supports_reasoning=True,
|
||||
model="qwen/qwen3-plus",
|
||||
)
|
||||
assert extra_body["reasoning"]["effort"] == "high"
|
||||
|
||||
def test_no_clamp_when_catalog_unknown(self, monkeypatch):
|
||||
import hermes_cli.models as models_mod
|
||||
from providers import get_provider_profile
|
||||
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {"a/b": None})
|
||||
monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
|
||||
profile = get_provider_profile("openrouter")
|
||||
assert profile is not None
|
||||
extra_body, _top = profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "ultra"},
|
||||
supports_reasoning=True,
|
||||
model="unlisted/model",
|
||||
)
|
||||
# Unknown capability → passthrough unchanged (no silent downgrade).
|
||||
assert extra_body["reasoning"]["effort"] == "ultra"
|
||||
|
|
@ -1535,6 +1535,18 @@ on its own adaptive default. The
|
|||
native Anthropic provider already controls effort directly and is unaffected.
|
||||
:::
|
||||
|
||||
:::note OpenRouter models and supported effort levels
|
||||
For other models routed through OpenRouter, Hermes reads the live model
|
||||
catalog's reasoning metadata (`supported_parameters` + per-model
|
||||
`reasoning.supported_efforts`) to decide whether to send reasoning controls at
|
||||
all and to clamp your requested effort to the nearest level the route actually
|
||||
supports (always downward — e.g. `ultra` becomes `high` on a route that stops
|
||||
at `high`, never a silent escalation). New reasoning-capable vendors work
|
||||
automatically without waiting for a Hermes update; when the catalog is
|
||||
unreachable or a model isn't listed, Hermes falls back to its built-in
|
||||
model-family list and passes your effort through unchanged.
|
||||
:::
|
||||
|
||||
You can also change the reasoning effort at runtime with the `/reasoning` command:
|
||||
|
||||
```
|
||||
|
|
|
|||
Loading…
Reference in New Issue