From 8c12fa7cf09c18481864c09b570a04cd80270c6e Mon Sep 17 00:00:00 2001 From: LunarNexus Date: Mon, 20 Jul 2026 14:06:21 +0000 Subject: [PATCH] fix(lmstudio): respect applied runtime context --- agent/agent_init.py | 25 ++- agent/agent_runtime_helpers.py | 73 +++++-- contributors/emails/git@lunarnexus.com | 1 + hermes_cli/models.py | 128 ++++++++--- run_agent.py | 89 +++++--- .../test_lmstudio_context_policy.py | 201 ++++++++++++++++++ tests/hermes_cli/test_urllib_security.py | 2 +- tests/run_agent/test_lmstudio_load_mode.py | 36 +++- tests/run_agent/test_switch_model_context.py | 40 ++++ website/docs/integrations/providers.md | 2 +- 10 files changed, 498 insertions(+), 99 deletions(-) create mode 100644 contributors/emails/git@lunarnexus.com create mode 100644 tests/hermes_cli/test_lmstudio_context_policy.py diff --git a/agent/agent_init.py b/agent/agent_init.py index a99a0de89916f..b14ca117f3004 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -2276,7 +2276,18 @@ def init_agent( # AFTER the custom_providers branch so per-model overrides aren't lost. agent._config_context_length = _config_context_length - agent._ensure_lmstudio_runtime_loaded(_config_context_length) + _lmstudio_runtime_context_length = agent._ensure_lmstudio_runtime_loaded( + _config_context_length + ) + if agent._lmstudio_load_was_unverified(_lmstudio_runtime_context_length): + raise RuntimeError( + "LM Studio model activation was rejected or completed without a " + "verifiable active context length; agent startup aborted" + ) + _effective_context_length = agent._effective_lmstudio_context_length( + _config_context_length, + _lmstudio_runtime_context_length, + ) @@ -2353,7 +2364,7 @@ def init_agent( agent.model, base_url=agent.base_url, api_key=getattr(agent, "api_key", ""), - config_context_length=_config_context_length, + config_context_length=_effective_context_length, provider=agent.provider, custom_providers=_custom_providers, ) @@ -2388,7 +2399,7 @@ def init_agent( quiet_mode=agent.quiet_mode, base_url=agent.base_url, api_key=getattr(agent, "api_key", ""), - config_context_length=_config_context_length, + config_context_length=_effective_context_length, provider=agent.provider, api_mode=agent.api_mode, abort_on_summary_failure=compression_abort_on_summary_failure, @@ -2417,7 +2428,13 @@ def init_agent( # Reject models whose context window is below the minimum required # for reliable tool-calling workflows (64K tokens). _ctx = getattr(agent.context_compressor, "context_length", 0) - if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH: + _allow_lmstudio_explicit_below_floor = ( + str(getattr(agent, "provider", "") or "").strip().lower() == "lmstudio" + and isinstance(agent._config_context_length, int) + and not isinstance(agent._config_context_length, bool) + and agent._config_context_length > 0 + ) + if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH and not _allow_lmstudio_explicit_below_floor: raise ValueError( f"Model {agent.model} has a context window of {_ctx:,} tokens, " f"which is below the minimum {MINIMUM_CONTEXT_LENGTH:,} required " diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 092a87da95ef9..c4619de005d61 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -2137,6 +2137,16 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo agent, "_credential_pool_entry_id", _MISSING ) + def _restore_snapshot() -> None: + for _name, _value in _snapshot.items(): + if _value is _MISSING: + # Attribute did not exist before the swap — don't fabricate it. + continue + try: + setattr(agent, _name, _value) + except Exception: # noqa: BLE001 + pass + try: # Clear the per-config context_length override so the new model's # actual context window is resolved via get_model_context_length() @@ -2305,16 +2315,42 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # caller's exception handler can surface a meaningful warning. The # exception is re-raised; cli.py / gateway/run.py / tui_gateway catch # it and print "Agent swap failed; change applied to next session". - for _name, _value in _snapshot.items(): - if _value is _MISSING: - # Attribute did not exist before the swap — don't fabricate it. - continue - try: - setattr(agent, _name, _value) - except Exception: # noqa: BLE001 - pass + _restore_snapshot() raise + # ── LM Studio: preload before probing context length ── + _sm_custom_providers = None + try: + from hermes_cli.config import ( + get_compatible_custom_providers, + get_custom_provider_context_length, + load_config, + ) + + _sm_cfg = load_config() + _sm_custom_providers = get_compatible_custom_providers(_sm_cfg) + _destination_context_intent = get_custom_provider_context_length( + model=agent.model, + base_url=agent.base_url, + custom_providers=_sm_custom_providers, + ) + except Exception: + _destination_context_intent = None + agent._config_context_length = _destination_context_intent + _runtime_context_length = agent._ensure_lmstudio_runtime_loaded( + _destination_context_intent + ) + if agent._lmstudio_load_was_unverified(_runtime_context_length): + _restore_snapshot() + raise RuntimeError( + "LM Studio model activation was rejected or completed without a " + "verifiable active context length; model switch aborted" + ) + _effective_context_length = agent._effective_lmstudio_context_length( + _destination_context_intent, + _runtime_context_length, + ) + # ── Re-evaluate prompt caching ── agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy( @@ -2325,22 +2361,15 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo ) ) - # ── LM Studio: preload before probing context length ── - agent._ensure_lmstudio_runtime_loaded() - # ── Update context compressor ── if hasattr(agent, "context_compressor") and agent.context_compressor: from agent.model_metadata import get_model_context_length - # Re-read custom_providers from live config so per-model - # context_length overrides are honored when switching to a - # custom provider mid-session (closes #15779). - _sm_custom_providers = None - try: - from hermes_cli.config import load_config, get_compatible_custom_providers - _sm_cfg = load_config() - _sm_custom_providers = get_compatible_custom_providers(_sm_cfg) - except Exception: - _sm_custom_providers = None + if _sm_custom_providers is None: + try: + from hermes_cli.config import get_compatible_custom_providers, load_config + _sm_custom_providers = get_compatible_custom_providers(load_config()) + except Exception: + _sm_custom_providers = None # ``agent.api_key`` may be a callable (Azure Foundry Entra ID # token provider). ``get_model_context_length`` expects a # string for its live-probe paths; for Foundry the context @@ -2352,7 +2381,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo base_url=agent.base_url, api_key=_ctx_api_key, provider=agent.provider, - config_context_length=getattr(agent, "_config_context_length", None), + config_context_length=_effective_context_length, custom_providers=_sm_custom_providers, ) agent.context_compressor.update_model( diff --git a/contributors/emails/git@lunarnexus.com b/contributors/emails/git@lunarnexus.com new file mode 100644 index 0000000000000..333fc43143984 --- /dev/null +++ b/contributors/emails/git@lunarnexus.com @@ -0,0 +1 @@ +LunarNexus diff --git a/hermes_cli/models.py b/hermes_cli/models.py index bc96ec8a02148..7f996de8fe7c3 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -3388,23 +3388,69 @@ def fetch_lmstudio_models( return models or [] +class LMStudioLoadResult(NamedTuple): + """Verified LM Studio runtime plus load-attempt provenance.""" + + context_length: Optional[int] + load_attempted: bool = False + rejected: bool = False + + def ensure_lmstudio_model_loaded( model: str, base_url: Optional[str], api_key: Optional[str], - target_context_length: int, + target_context_length: Optional[int], timeout: float = 120.0, -) -> Optional[int]: - """Ensure LM Studio has ``model`` loaded with at least ``target_context_length``. + *, + return_load_result: bool = False, +) -> Optional[int] | LMStudioLoadResult: + """Ensure ``model`` is loaded and return verified runtime context. - No-op when an instance is already loaded with sufficient context. Otherwise - POSTs ``/api/v1/models/load`` to (re)load with the target context, capped - at the model's ``max_context_length``. Returns the resolved loaded context - length, or ``None`` when the probe / load failed. + Existing loaded-instance context is authoritative. Cold loads omit + ``context_length`` unless the caller supplied an explicit override; the + returned context must come from LM Studio's echoed or refreshed state. """ + + def _result( + context_length: Optional[int], + *, + load_attempted: bool = False, + rejected: bool = False, + ) -> Optional[int] | LMStudioLoadResult: + value = LMStudioLoadResult(context_length, load_attempted, rejected) + return value if return_load_result else context_length + + def _positive_int(value: Any) -> Optional[int]: + if isinstance(value, int) and not isinstance(value, bool) and value > 0: + return value + return None + + def _loaded_context(entry: dict) -> Optional[int]: + instances = entry.get("loaded_instances") + if not isinstance(instances, list): + return None + for instance in instances: + config = instance.get("config") if isinstance(instance, dict) else None + context = config.get("context_length") if isinstance(config, dict) else None + parsed = _positive_int(context) + if parsed is not None: + return parsed + return None + + def _find_entry(raw_models: list[dict]) -> Optional[dict]: + for raw in raw_models: + if isinstance(raw, dict) and (raw.get("key") == model or raw.get("id") == model): + return raw + return None + server_root = _lmstudio_server_root(base_url) if not server_root: - return None + return _result(None) + + explicit_context = _positive_int(target_context_length) + if target_context_length is not None and explicit_context is None: + return _result(None) headers = _lmstudio_request_headers(api_key) @@ -3413,32 +3459,28 @@ def ensure_lmstudio_model_loaded( except Exception: raw_models = None if raw_models is None: - return None + return _result(None) - target_entry = None - for raw in raw_models: - if not isinstance(raw, dict): - continue - if raw.get("key") == model or raw.get("id") == model: - target_entry = raw - break + target_entry = _find_entry(raw_models) if target_entry is None: - return None + return _result(None) - max_ctx = target_entry.get("max_context_length") - if isinstance(max_ctx, int) and max_ctx > 0: - target_context_length = min(target_context_length, max_ctx) + max_ctx = _positive_int(target_entry.get("max_context_length")) + if explicit_context is not None and max_ctx is not None and explicit_context > max_ctx: + return _result(None, rejected=True) - for inst in target_entry.get("loaded_instances") or []: - cfg = inst.get("config") if isinstance(inst, dict) else None - loaded_ctx = cfg.get("context_length") if isinstance(cfg, dict) else None - if isinstance(loaded_ctx, int) and loaded_ctx >= target_context_length: - return loaded_ctx + current_context = _loaded_context(target_entry) + if current_context is not None: + return _result(current_context) - body = json.dumps({ - "model": model, - "context_length": target_context_length, - }).encode() + loaded_instances = target_entry.get("loaded_instances") + if not isinstance(loaded_instances, list) or loaded_instances: + return _result(None) + + load_payload: dict[str, Any] = {"model": model, "echo_load_config": True} + if explicit_context is not None: + load_payload["context_length"] = explicit_context + body = json.dumps(load_payload).encode() load_headers = dict(headers) load_headers["Content-Type"] = "application/json" try: @@ -3449,10 +3491,32 @@ def ensure_lmstudio_model_loaded( method="POST", ) with _urlopen_model_catalog_request(load_request, timeout=timeout) as resp: - resp.read() + response_body = resp.read() except Exception: - return None - return target_context_length + return _result(None, load_attempted=True) + + try: + response_payload = json.loads(response_body.decode()) + except Exception: + response_payload = None + load_config = response_payload.get("load_config") if isinstance(response_payload, dict) else None + applied_context = ( + _positive_int(load_config.get("context_length")) + if isinstance(load_config, dict) + else None + ) + if applied_context is not None: + return _result(applied_context, load_attempted=True) + + try: + refreshed_models = _lmstudio_fetch_raw_models(api_key=api_key, base_url=base_url, timeout=10) + except Exception: + refreshed_models = None + if refreshed_models is None: + return _result(None, load_attempted=True) + refreshed_entry = _find_entry(refreshed_models) + refreshed_context = _loaded_context(refreshed_entry) if refreshed_entry is not None else None + return _result(refreshed_context, load_attempted=True) def lmstudio_model_reasoning_options( diff --git a/run_agent.py b/run_agent.py index 8d003ef24ef69..62032e990c8ce 100644 --- a/run_agent.py +++ b/run_agent.py @@ -781,41 +781,66 @@ class AIAgent: except Exception as exc: logger.debug("context engine bind_session_state during reset: %s", exc) - def _ensure_lmstudio_runtime_loaded(self, config_context_length: Optional[int] = None) -> None: - """ - Preload the LM Studio model unless configured to rely on LM Studio JIT loading. - """ + @staticmethod + def _effective_lmstudio_context_length( + config_context_length: Optional[int], + runtime_context_length: Any, + ) -> Optional[int]: + """Return a safe context budget from explicit intent and verified runtime.""" + explicit = ( + config_context_length + if isinstance(config_context_length, int) + and not isinstance(config_context_length, bool) + and config_context_length > 0 + else None + ) + runtime_value = getattr(runtime_context_length, "context_length", runtime_context_length) + runtime = ( + runtime_value + if isinstance(runtime_value, int) + and not isinstance(runtime_value, bool) + and runtime_value > 0 + else None + ) + if bool(getattr(runtime_context_length, "rejected", False)) or ( + bool(getattr(runtime_context_length, "load_attempted", False)) + and runtime is None + ): + return None + if runtime is not None and explicit is not None: + return min(runtime, explicit) + return runtime if runtime is not None else explicit + + @staticmethod + def _lmstudio_load_was_unverified(load_result: Any) -> bool: + """Return true when a management load was rejected or unverifiable.""" + return bool(getattr(load_result, "rejected", False)) or ( + bool(getattr(load_result, "load_attempted", False)) + and getattr(load_result, "context_length", None) is None + ) + + def _ensure_lmstudio_runtime_loaded( + self, + config_context_length: Optional[int] = None, + ) -> Any: + """Preload LM Studio unless configured to rely on JIT loading.""" if (self.provider or "").strip().lower() != "lmstudio": - return + return None if (getattr(self, "lmstudio_load_mode", "explicit") or "explicit").strip().lower() == "jit": logger.debug("LM Studio explicit preload skipped: lmstudio_load_mode=jit") - return - try: - from agent.model_metadata import MINIMUM_CONTEXT_LENGTH - from hermes_cli.models import ensure_lmstudio_model_loaded - if config_context_length is None: - config_context_length = getattr(self, "_config_context_length", None) - target_ctx = max(config_context_length or 0, MINIMUM_CONTEXT_LENGTH) - loaded_ctx = ensure_lmstudio_model_loaded( - self.model, self.base_url, getattr(self, "api_key", ""), target_ctx, - ) - if loaded_ctx: - # Push into the live compressor so the status bar reflects the - # real loaded ctx the moment the load resolves, instead of - # holding the previous model's value (or "ctx --") through the - # next render tick. - cc = getattr(self, "context_compressor", None) - if cc is not None: - cc.update_model( - model=self.model, - context_length=loaded_ctx, - base_url=self.base_url, - api_key=getattr(self, "api_key", ""), - provider=self.provider, - api_mode=self.api_mode, - ) - except Exception as err: - logger.debug("LM Studio preload skipped: %s", err) + return None + + from hermes_cli.models import ensure_lmstudio_model_loaded + + if config_context_length is None: + config_context_length = getattr(self, "_config_context_length", None) + return ensure_lmstudio_model_loaded( + self.model, + self.base_url, + getattr(self, "api_key", ""), + config_context_length, + return_load_result=True, + ) def switch_model(self, new_model, new_provider, api_key='', base_url='', api_mode=''): """Forwarder — see ``agent.agent_runtime_helpers.switch_model``.""" diff --git a/tests/hermes_cli/test_lmstudio_context_policy.py b/tests/hermes_cli/test_lmstudio_context_policy.py new file mode 100644 index 0000000000000..fdf32a29df6f4 --- /dev/null +++ b/tests/hermes_cli/test_lmstudio_context_policy.py @@ -0,0 +1,201 @@ +import json + +import pytest + +from hermes_cli import models + + +MODEL = "publisher/model" +BASE_URL = "http://127.0.0.1:1234/v1" + + +class _JsonResponse: + def __init__(self, payload): + self._body = json.dumps(payload).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return self._body + + +def _catalog(*, loaded_context=None, maximum=262_144): + loaded_instances = [] + if loaded_context is not None: + loaded_instances.append({ + "id": f"{MODEL}:active", + "config": {"context_length": loaded_context}, + }) + return [{ + "key": MODEL, + "max_context_length": maximum, + "loaded_instances": loaded_instances, + }] + + +def _capture_load(monkeypatch, response_payload): + requests = [] + + def fake_open(request, *, timeout): + requests.append((request, timeout, json.loads(request.data.decode()))) + return _JsonResponse(response_payload) + + monkeypatch.setattr(models, "_urlopen_model_catalog_request", fake_open) + return requests + + +def test_loaded_64k_runtime_is_preserved_without_post(monkeypatch): + monkeypatch.setattr( + models, + "_lmstudio_fetch_raw_models", + lambda **_kwargs: _catalog(loaded_context=64_000), + ) + monkeypatch.setattr( + models, + "_urlopen_model_catalog_request", + lambda *_args, **_kwargs: pytest.fail("loaded model must not be reloaded"), + ) + + result = models.ensure_lmstudio_model_loaded( + MODEL, BASE_URL, api_key="", target_context_length=None + ) + + assert result == 64_000 + + +def test_unloaded_no_override_omits_context_and_requests_echo(monkeypatch): + monkeypatch.setattr( + models, "_lmstudio_fetch_raw_models", lambda **_kwargs: _catalog() + ) + requests = _capture_load(monkeypatch, { + "load_config": {"context_length": 96_000}, + }) + + result = models.ensure_lmstudio_model_loaded( + MODEL, BASE_URL, api_key="", target_context_length=None + ) + + assert result == 96_000 + assert requests[0][2] == {"model": MODEL, "echo_load_config": True} + + +@pytest.mark.parametrize("requested_context", [32_000, 100_000]) +def test_unloaded_explicit_override_sends_exact_context(monkeypatch, requested_context): + monkeypatch.setattr( + models, "_lmstudio_fetch_raw_models", lambda **_kwargs: _catalog() + ) + requests = _capture_load(monkeypatch, { + "load_config": {"context_length": requested_context}, + }) + + result = models.ensure_lmstudio_model_loaded( + MODEL, BASE_URL, api_key="", target_context_length=requested_context + ) + + assert result == requested_context + assert requests[0][2] == { + "model": MODEL, + "context_length": requested_context, + "echo_load_config": True, + } + + +def test_echoed_applied_context_wins_over_requested_context(monkeypatch): + monkeypatch.setattr( + models, "_lmstudio_fetch_raw_models", lambda **_kwargs: _catalog() + ) + _capture_load(monkeypatch, {"load_config": {"context_length": 96_000}}) + + result = models.ensure_lmstudio_model_loaded( + MODEL, BASE_URL, api_key="", target_context_length=100_000 + ) + + assert result == 96_000 + + +def test_missing_echo_refreshes_loaded_state(monkeypatch): + catalogs = iter([_catalog(), _catalog(loaded_context=88_000)]) + monkeypatch.setattr( + models, "_lmstudio_fetch_raw_models", lambda **_kwargs: next(catalogs) + ) + _capture_load(monkeypatch, {"status": "loaded"}) + + result = models.ensure_lmstudio_model_loaded( + MODEL, BASE_URL, api_key="", target_context_length=100_000 + ) + + assert result == 88_000 + + +def test_successful_load_without_verifiable_context_returns_unknown(monkeypatch): + catalogs = iter([_catalog(), _catalog()]) + monkeypatch.setattr( + models, "_lmstudio_fetch_raw_models", lambda **_kwargs: next(catalogs) + ) + _capture_load(monkeypatch, {"status": "loaded"}) + + result = models.ensure_lmstudio_model_loaded( + MODEL, + BASE_URL, + api_key="", + target_context_length=100_000, + return_load_result=True, + ) + + assert result.context_length is None + assert result.load_attempted is True + assert result.rejected is False + + +def test_explicit_override_above_known_maximum_rejects_without_post(monkeypatch): + monkeypatch.setattr( + models, + "_lmstudio_fetch_raw_models", + lambda **_kwargs: _catalog(maximum=128_000), + ) + monkeypatch.setattr( + models, + "_urlopen_model_catalog_request", + lambda *_args, **_kwargs: pytest.fail("invalid override must not be posted"), + ) + + result = models.ensure_lmstudio_model_loaded( + MODEL, + BASE_URL, + api_key="", + target_context_length=256_000, + return_load_result=True, + ) + + assert result.context_length is None + assert result.load_attempted is False + assert result.rejected is True + + +def test_explicit_override_above_known_maximum_rejects_even_when_loaded(monkeypatch): + monkeypatch.setattr( + models, + "_lmstudio_fetch_raw_models", + lambda **_kwargs: _catalog(loaded_context=64_000, maximum=128_000), + ) + monkeypatch.setattr( + models, + "_urlopen_model_catalog_request", + lambda *_args, **_kwargs: pytest.fail("invalid override must not be posted"), + ) + + result = models.ensure_lmstudio_model_loaded( + MODEL, + BASE_URL, + api_key="", + target_context_length=256_000, + return_load_result=True, + ) + + assert result.context_length is None + assert result.load_attempted is False + assert result.rejected is True diff --git a/tests/hermes_cli/test_urllib_security.py b/tests/hermes_cli/test_urllib_security.py index d21ac3bb030e9..bd8057c60dfb9 100644 --- a/tests/hermes_cli/test_urllib_security.py +++ b/tests/hermes_cli/test_urllib_security.py @@ -554,7 +554,7 @@ def test_lmstudio_load_post_drops_bearer_on_redirect(monkeypatch): source.shutdown() sink.shutdown() - assert loaded == 4096 + assert loaded is None method, headers = _RecordingHandler.requests[-1] assert method == "GET" assert "authorization" not in headers diff --git a/tests/run_agent/test_lmstudio_load_mode.py b/tests/run_agent/test_lmstudio_load_mode.py index 50244cc5ea99d..e210dc833549a 100644 --- a/tests/run_agent/test_lmstudio_load_mode.py +++ b/tests/run_agent/test_lmstudio_load_mode.py @@ -1,6 +1,7 @@ from types import SimpleNamespace from typing import Any, cast +from hermes_cli.models import LMStudioLoadResult from run_agent import AIAgent @@ -22,29 +23,32 @@ def test_lmstudio_jit_load_mode_skips_explicit_preload(monkeypatch): def fake_ensure(*args, **kwargs): calls.append((args, kwargs)) - return 64000 + return LMStudioLoadResult(64_000) monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure) - AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("jit"))) + result = AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("jit"))) + assert result is None assert calls == [] -def test_lmstudio_explicit_load_mode_preserves_preload(monkeypatch): +def test_lmstudio_explicit_load_mode_passes_no_override_as_none(monkeypatch): calls = [] def fake_ensure(*args, **kwargs): calls.append((args, kwargs)) - return 64000 + return LMStudioLoadResult(96_000, load_attempted=True) monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure) - AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("explicit"))) + result = AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("explicit"))) + assert result.context_length == 96_000 assert len(calls) == 1 assert calls[0][0][:3] == ("test/model", "http://127.0.0.1:1234/v1", "") - assert calls[0][0][3] == 64000 + assert calls[0][0][3] is None + assert calls[0][1]["return_load_result"] is True def test_missing_lmstudio_load_mode_defaults_to_explicit(monkeypatch): @@ -54,10 +58,28 @@ def test_missing_lmstudio_load_mode_defaults_to_explicit(monkeypatch): def fake_ensure(*args, **kwargs): calls.append((args, kwargs)) - return 64000 + return LMStudioLoadResult(64_000) monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure) AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, agent)) assert len(calls) == 1 + + +def test_explicit_budget_below_loaded_runtime_limits_effective_context(): + result = AIAgent._effective_lmstudio_context_length( + 80_000, + LMStudioLoadResult(120_000), + ) + + assert result == 80_000 + + +def test_attempted_unverified_load_has_no_effective_context(): + result = AIAgent._effective_lmstudio_context_length( + 100_000, + LMStudioLoadResult(None, load_attempted=True), + ) + + assert result is None diff --git a/tests/run_agent/test_switch_model_context.py b/tests/run_agent/test_switch_model_context.py index 060302c4f44e6..db63f58ab4e9d 100644 --- a/tests/run_agent/test_switch_model_context.py +++ b/tests/run_agent/test_switch_model_context.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch import pytest +from hermes_cli.models import LMStudioLoadResult from run_agent import AIAgent from agent.agent_init import _normalize_route_base_url from agent.context_compressor import ContextCompressor @@ -1020,3 +1021,42 @@ def test_direct_start_runtime_first_provider_names_require_explicit_custom_prefi base_url=base_url, ) assert custom_agent.context_compressor.config_context_length == 1_048_576 + + +def test_lmstudio_switch_uses_destination_context_and_verified_runtime(monkeypatch): + agent = _make_agent_with_compressor(config_context_length=32_768) + calls = [] + + def fake_load_config(): + return {} + + def fake_compatible(_cfg): + return [{"name": "lmstudio", "base_url": "http://127.0.0.1:1234/v1"}] + + def fake_provider_context(*, model, base_url, custom_providers): + assert model == "lmstudio/new-model" + assert base_url == "http://127.0.0.1:1234/v1" + return 120_000 + + def fake_lmstudio_load(self, config_context_length=None): + calls.append(config_context_length) + return LMStudioLoadResult(100_000) + + monkeypatch.setattr("hermes_cli.config.load_config", fake_load_config) + monkeypatch.setattr("hermes_cli.config.get_compatible_custom_providers", fake_compatible) + monkeypatch.setattr("hermes_cli.config.get_custom_provider_context_length", fake_provider_context) + monkeypatch.setattr(AIAgent, "_ensure_lmstudio_runtime_loaded", fake_lmstudio_load) + + with patch("agent.model_metadata.get_model_context_length", return_value=100_000) as mock_ctx_len: + agent.switch_model( + "lmstudio/new-model", + "lmstudio", + api_key="", + base_url="http://127.0.0.1:1234/v1", + ) + + assert calls == [120_000] + call_kwargs = mock_ctx_len.call_args.kwargs + assert call_kwargs.get("config_context_length") == 100_000 + assert agent._config_context_length == 120_000 + assert agent.context_compressor.context_length == 100_000 diff --git a/website/docs/integrations/providers.md b/website/docs/integrations/providers.md index 9ef33b90ef1b4..1b1791bf84afd 100644 --- a/website/docs/integrations/providers.md +++ b/website/docs/integrations/providers.md @@ -844,7 +844,7 @@ hermes model # If LM Studio server auth is enabled, enter LM_API_KEY when prompted ``` -By default, Hermes explicitly asks LM Studio to load the selected model with 64K context length before the first request. +Hermes preserves the context of an already-loaded LM Studio instance. For an unloaded model in the default explicit mode, Hermes omits `context_length` unless you configured one in Hermes, so LM Studio can apply its own model setting. Hermes then uses only the context length LM Studio reports after loading. To change context length in LM Studio: