fix(lmstudio): respect applied runtime context
This commit is contained in:
LunarNexus
kshitij
parent
678916b427
commit
8c12fa7cf0
|
|
@ -2276,7 +2276,18 @@ def init_agent(
|
|||
# AFTER the custom_providers branch so per-model overrides aren't lost.
|
||||
agent._config_context_length = _config_context_length
|
||||
|
||||
agent._ensure_lmstudio_runtime_loaded(_config_context_length)
|
||||
_lmstudio_runtime_context_length = agent._ensure_lmstudio_runtime_loaded(
|
||||
_config_context_length
|
||||
)
|
||||
if agent._lmstudio_load_was_unverified(_lmstudio_runtime_context_length):
|
||||
raise RuntimeError(
|
||||
"LM Studio model activation was rejected or completed without a "
|
||||
"verifiable active context length; agent startup aborted"
|
||||
)
|
||||
_effective_context_length = agent._effective_lmstudio_context_length(
|
||||
_config_context_length,
|
||||
_lmstudio_runtime_context_length,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
|
@ -2353,7 +2364,7 @@ def init_agent(
|
|||
agent.model,
|
||||
base_url=agent.base_url,
|
||||
api_key=getattr(agent, "api_key", ""),
|
||||
config_context_length=_config_context_length,
|
||||
config_context_length=_effective_context_length,
|
||||
provider=agent.provider,
|
||||
custom_providers=_custom_providers,
|
||||
)
|
||||
|
|
@ -2388,7 +2399,7 @@ def init_agent(
|
|||
quiet_mode=agent.quiet_mode,
|
||||
base_url=agent.base_url,
|
||||
api_key=getattr(agent, "api_key", ""),
|
||||
config_context_length=_config_context_length,
|
||||
config_context_length=_effective_context_length,
|
||||
provider=agent.provider,
|
||||
api_mode=agent.api_mode,
|
||||
abort_on_summary_failure=compression_abort_on_summary_failure,
|
||||
|
|
@ -2417,7 +2428,13 @@ def init_agent(
|
|||
# Reject models whose context window is below the minimum required
|
||||
# for reliable tool-calling workflows (64K tokens).
|
||||
_ctx = getattr(agent.context_compressor, "context_length", 0)
|
||||
if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH:
|
||||
_allow_lmstudio_explicit_below_floor = (
|
||||
str(getattr(agent, "provider", "") or "").strip().lower() == "lmstudio"
|
||||
and isinstance(agent._config_context_length, int)
|
||||
and not isinstance(agent._config_context_length, bool)
|
||||
and agent._config_context_length > 0
|
||||
)
|
||||
if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH and not _allow_lmstudio_explicit_below_floor:
|
||||
raise ValueError(
|
||||
f"Model {agent.model} has a context window of {_ctx:,} tokens, "
|
||||
f"which is below the minimum {MINIMUM_CONTEXT_LENGTH:,} required "
|
||||
|
|
|
|||
|
|
@ -2137,6 +2137,16 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
agent, "_credential_pool_entry_id", _MISSING
|
||||
)
|
||||
|
||||
def _restore_snapshot() -> None:
|
||||
for _name, _value in _snapshot.items():
|
||||
if _value is _MISSING:
|
||||
# Attribute did not exist before the swap — don't fabricate it.
|
||||
continue
|
||||
try:
|
||||
setattr(agent, _name, _value)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
try:
|
||||
# Clear the per-config context_length override so the new model's
|
||||
# actual context window is resolved via get_model_context_length()
|
||||
|
|
@ -2305,16 +2315,42 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
# caller's exception handler can surface a meaningful warning. The
|
||||
# exception is re-raised; cli.py / gateway/run.py / tui_gateway catch
|
||||
# it and print "Agent swap failed; change applied to next session".
|
||||
for _name, _value in _snapshot.items():
|
||||
if _value is _MISSING:
|
||||
# Attribute did not exist before the swap — don't fabricate it.
|
||||
continue
|
||||
try:
|
||||
setattr(agent, _name, _value)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
_restore_snapshot()
|
||||
raise
|
||||
|
||||
# ── LM Studio: preload before probing context length ──
|
||||
_sm_custom_providers = None
|
||||
try:
|
||||
from hermes_cli.config import (
|
||||
get_compatible_custom_providers,
|
||||
get_custom_provider_context_length,
|
||||
load_config,
|
||||
)
|
||||
|
||||
_sm_cfg = load_config()
|
||||
_sm_custom_providers = get_compatible_custom_providers(_sm_cfg)
|
||||
_destination_context_intent = get_custom_provider_context_length(
|
||||
model=agent.model,
|
||||
base_url=agent.base_url,
|
||||
custom_providers=_sm_custom_providers,
|
||||
)
|
||||
except Exception:
|
||||
_destination_context_intent = None
|
||||
agent._config_context_length = _destination_context_intent
|
||||
_runtime_context_length = agent._ensure_lmstudio_runtime_loaded(
|
||||
_destination_context_intent
|
||||
)
|
||||
if agent._lmstudio_load_was_unverified(_runtime_context_length):
|
||||
_restore_snapshot()
|
||||
raise RuntimeError(
|
||||
"LM Studio model activation was rejected or completed without a "
|
||||
"verifiable active context length; model switch aborted"
|
||||
)
|
||||
_effective_context_length = agent._effective_lmstudio_context_length(
|
||||
_destination_context_intent,
|
||||
_runtime_context_length,
|
||||
)
|
||||
|
||||
# ── Re-evaluate prompt caching ──
|
||||
agent._use_prompt_caching, agent._use_native_cache_layout = (
|
||||
agent._anthropic_prompt_cache_policy(
|
||||
|
|
@ -2325,22 +2361,15 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
)
|
||||
)
|
||||
|
||||
# ── LM Studio: preload before probing context length ──
|
||||
agent._ensure_lmstudio_runtime_loaded()
|
||||
|
||||
# ── Update context compressor ──
|
||||
if hasattr(agent, "context_compressor") and agent.context_compressor:
|
||||
from agent.model_metadata import get_model_context_length
|
||||
# Re-read custom_providers from live config so per-model
|
||||
# context_length overrides are honored when switching to a
|
||||
# custom provider mid-session (closes #15779).
|
||||
_sm_custom_providers = None
|
||||
try:
|
||||
from hermes_cli.config import load_config, get_compatible_custom_providers
|
||||
_sm_cfg = load_config()
|
||||
_sm_custom_providers = get_compatible_custom_providers(_sm_cfg)
|
||||
except Exception:
|
||||
_sm_custom_providers = None
|
||||
if _sm_custom_providers is None:
|
||||
try:
|
||||
from hermes_cli.config import get_compatible_custom_providers, load_config
|
||||
_sm_custom_providers = get_compatible_custom_providers(load_config())
|
||||
except Exception:
|
||||
_sm_custom_providers = None
|
||||
# ``agent.api_key`` may be a callable (Azure Foundry Entra ID
|
||||
# token provider). ``get_model_context_length`` expects a
|
||||
# string for its live-probe paths; for Foundry the context
|
||||
|
|
@ -2352,7 +2381,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo
|
|||
base_url=agent.base_url,
|
||||
api_key=_ctx_api_key,
|
||||
provider=agent.provider,
|
||||
config_context_length=getattr(agent, "_config_context_length", None),
|
||||
config_context_length=_effective_context_length,
|
||||
custom_providers=_sm_custom_providers,
|
||||
)
|
||||
agent.context_compressor.update_model(
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
LunarNexus
|
||||
|
|
@ -3388,23 +3388,69 @@ def fetch_lmstudio_models(
|
|||
return models or []
|
||||
|
||||
|
||||
class LMStudioLoadResult(NamedTuple):
|
||||
"""Verified LM Studio runtime plus load-attempt provenance."""
|
||||
|
||||
context_length: Optional[int]
|
||||
load_attempted: bool = False
|
||||
rejected: bool = False
|
||||
|
||||
|
||||
def ensure_lmstudio_model_loaded(
|
||||
model: str,
|
||||
base_url: Optional[str],
|
||||
api_key: Optional[str],
|
||||
target_context_length: int,
|
||||
target_context_length: Optional[int],
|
||||
timeout: float = 120.0,
|
||||
) -> Optional[int]:
|
||||
"""Ensure LM Studio has ``model`` loaded with at least ``target_context_length``.
|
||||
*,
|
||||
return_load_result: bool = False,
|
||||
) -> Optional[int] | LMStudioLoadResult:
|
||||
"""Ensure ``model`` is loaded and return verified runtime context.
|
||||
|
||||
No-op when an instance is already loaded with sufficient context. Otherwise
|
||||
POSTs ``/api/v1/models/load`` to (re)load with the target context, capped
|
||||
at the model's ``max_context_length``. Returns the resolved loaded context
|
||||
length, or ``None`` when the probe / load failed.
|
||||
Existing loaded-instance context is authoritative. Cold loads omit
|
||||
``context_length`` unless the caller supplied an explicit override; the
|
||||
returned context must come from LM Studio's echoed or refreshed state.
|
||||
"""
|
||||
|
||||
def _result(
|
||||
context_length: Optional[int],
|
||||
*,
|
||||
load_attempted: bool = False,
|
||||
rejected: bool = False,
|
||||
) -> Optional[int] | LMStudioLoadResult:
|
||||
value = LMStudioLoadResult(context_length, load_attempted, rejected)
|
||||
return value if return_load_result else context_length
|
||||
|
||||
def _positive_int(value: Any) -> Optional[int]:
|
||||
if isinstance(value, int) and not isinstance(value, bool) and value > 0:
|
||||
return value
|
||||
return None
|
||||
|
||||
def _loaded_context(entry: dict) -> Optional[int]:
|
||||
instances = entry.get("loaded_instances")
|
||||
if not isinstance(instances, list):
|
||||
return None
|
||||
for instance in instances:
|
||||
config = instance.get("config") if isinstance(instance, dict) else None
|
||||
context = config.get("context_length") if isinstance(config, dict) else None
|
||||
parsed = _positive_int(context)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
def _find_entry(raw_models: list[dict]) -> Optional[dict]:
|
||||
for raw in raw_models:
|
||||
if isinstance(raw, dict) and (raw.get("key") == model or raw.get("id") == model):
|
||||
return raw
|
||||
return None
|
||||
|
||||
server_root = _lmstudio_server_root(base_url)
|
||||
if not server_root:
|
||||
return None
|
||||
return _result(None)
|
||||
|
||||
explicit_context = _positive_int(target_context_length)
|
||||
if target_context_length is not None and explicit_context is None:
|
||||
return _result(None)
|
||||
|
||||
headers = _lmstudio_request_headers(api_key)
|
||||
|
||||
|
|
@ -3413,32 +3459,28 @@ def ensure_lmstudio_model_loaded(
|
|||
except Exception:
|
||||
raw_models = None
|
||||
if raw_models is None:
|
||||
return None
|
||||
return _result(None)
|
||||
|
||||
target_entry = None
|
||||
for raw in raw_models:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
if raw.get("key") == model or raw.get("id") == model:
|
||||
target_entry = raw
|
||||
break
|
||||
target_entry = _find_entry(raw_models)
|
||||
if target_entry is None:
|
||||
return None
|
||||
return _result(None)
|
||||
|
||||
max_ctx = target_entry.get("max_context_length")
|
||||
if isinstance(max_ctx, int) and max_ctx > 0:
|
||||
target_context_length = min(target_context_length, max_ctx)
|
||||
max_ctx = _positive_int(target_entry.get("max_context_length"))
|
||||
if explicit_context is not None and max_ctx is not None and explicit_context > max_ctx:
|
||||
return _result(None, rejected=True)
|
||||
|
||||
for inst in target_entry.get("loaded_instances") or []:
|
||||
cfg = inst.get("config") if isinstance(inst, dict) else None
|
||||
loaded_ctx = cfg.get("context_length") if isinstance(cfg, dict) else None
|
||||
if isinstance(loaded_ctx, int) and loaded_ctx >= target_context_length:
|
||||
return loaded_ctx
|
||||
current_context = _loaded_context(target_entry)
|
||||
if current_context is not None:
|
||||
return _result(current_context)
|
||||
|
||||
body = json.dumps({
|
||||
"model": model,
|
||||
"context_length": target_context_length,
|
||||
}).encode()
|
||||
loaded_instances = target_entry.get("loaded_instances")
|
||||
if not isinstance(loaded_instances, list) or loaded_instances:
|
||||
return _result(None)
|
||||
|
||||
load_payload: dict[str, Any] = {"model": model, "echo_load_config": True}
|
||||
if explicit_context is not None:
|
||||
load_payload["context_length"] = explicit_context
|
||||
body = json.dumps(load_payload).encode()
|
||||
load_headers = dict(headers)
|
||||
load_headers["Content-Type"] = "application/json"
|
||||
try:
|
||||
|
|
@ -3449,10 +3491,32 @@ def ensure_lmstudio_model_loaded(
|
|||
method="POST",
|
||||
)
|
||||
with _urlopen_model_catalog_request(load_request, timeout=timeout) as resp:
|
||||
resp.read()
|
||||
response_body = resp.read()
|
||||
except Exception:
|
||||
return None
|
||||
return target_context_length
|
||||
return _result(None, load_attempted=True)
|
||||
|
||||
try:
|
||||
response_payload = json.loads(response_body.decode())
|
||||
except Exception:
|
||||
response_payload = None
|
||||
load_config = response_payload.get("load_config") if isinstance(response_payload, dict) else None
|
||||
applied_context = (
|
||||
_positive_int(load_config.get("context_length"))
|
||||
if isinstance(load_config, dict)
|
||||
else None
|
||||
)
|
||||
if applied_context is not None:
|
||||
return _result(applied_context, load_attempted=True)
|
||||
|
||||
try:
|
||||
refreshed_models = _lmstudio_fetch_raw_models(api_key=api_key, base_url=base_url, timeout=10)
|
||||
except Exception:
|
||||
refreshed_models = None
|
||||
if refreshed_models is None:
|
||||
return _result(None, load_attempted=True)
|
||||
refreshed_entry = _find_entry(refreshed_models)
|
||||
refreshed_context = _loaded_context(refreshed_entry) if refreshed_entry is not None else None
|
||||
return _result(refreshed_context, load_attempted=True)
|
||||
|
||||
|
||||
def lmstudio_model_reasoning_options(
|
||||
|
|
|
|||
89
run_agent.py
89
run_agent.py
|
|
@ -781,41 +781,66 @@ class AIAgent:
|
|||
except Exception as exc:
|
||||
logger.debug("context engine bind_session_state during reset: %s", exc)
|
||||
|
||||
def _ensure_lmstudio_runtime_loaded(self, config_context_length: Optional[int] = None) -> None:
|
||||
"""
|
||||
Preload the LM Studio model unless configured to rely on LM Studio JIT loading.
|
||||
"""
|
||||
@staticmethod
|
||||
def _effective_lmstudio_context_length(
|
||||
config_context_length: Optional[int],
|
||||
runtime_context_length: Any,
|
||||
) -> Optional[int]:
|
||||
"""Return a safe context budget from explicit intent and verified runtime."""
|
||||
explicit = (
|
||||
config_context_length
|
||||
if isinstance(config_context_length, int)
|
||||
and not isinstance(config_context_length, bool)
|
||||
and config_context_length > 0
|
||||
else None
|
||||
)
|
||||
runtime_value = getattr(runtime_context_length, "context_length", runtime_context_length)
|
||||
runtime = (
|
||||
runtime_value
|
||||
if isinstance(runtime_value, int)
|
||||
and not isinstance(runtime_value, bool)
|
||||
and runtime_value > 0
|
||||
else None
|
||||
)
|
||||
if bool(getattr(runtime_context_length, "rejected", False)) or (
|
||||
bool(getattr(runtime_context_length, "load_attempted", False))
|
||||
and runtime is None
|
||||
):
|
||||
return None
|
||||
if runtime is not None and explicit is not None:
|
||||
return min(runtime, explicit)
|
||||
return runtime if runtime is not None else explicit
|
||||
|
||||
@staticmethod
|
||||
def _lmstudio_load_was_unverified(load_result: Any) -> bool:
|
||||
"""Return true when a management load was rejected or unverifiable."""
|
||||
return bool(getattr(load_result, "rejected", False)) or (
|
||||
bool(getattr(load_result, "load_attempted", False))
|
||||
and getattr(load_result, "context_length", None) is None
|
||||
)
|
||||
|
||||
def _ensure_lmstudio_runtime_loaded(
|
||||
self,
|
||||
config_context_length: Optional[int] = None,
|
||||
) -> Any:
|
||||
"""Preload LM Studio unless configured to rely on JIT loading."""
|
||||
if (self.provider or "").strip().lower() != "lmstudio":
|
||||
return
|
||||
return None
|
||||
if (getattr(self, "lmstudio_load_mode", "explicit") or "explicit").strip().lower() == "jit":
|
||||
logger.debug("LM Studio explicit preload skipped: lmstudio_load_mode=jit")
|
||||
return
|
||||
try:
|
||||
from agent.model_metadata import MINIMUM_CONTEXT_LENGTH
|
||||
from hermes_cli.models import ensure_lmstudio_model_loaded
|
||||
if config_context_length is None:
|
||||
config_context_length = getattr(self, "_config_context_length", None)
|
||||
target_ctx = max(config_context_length or 0, MINIMUM_CONTEXT_LENGTH)
|
||||
loaded_ctx = ensure_lmstudio_model_loaded(
|
||||
self.model, self.base_url, getattr(self, "api_key", ""), target_ctx,
|
||||
)
|
||||
if loaded_ctx:
|
||||
# Push into the live compressor so the status bar reflects the
|
||||
# real loaded ctx the moment the load resolves, instead of
|
||||
# holding the previous model's value (or "ctx --") through the
|
||||
# next render tick.
|
||||
cc = getattr(self, "context_compressor", None)
|
||||
if cc is not None:
|
||||
cc.update_model(
|
||||
model=self.model,
|
||||
context_length=loaded_ctx,
|
||||
base_url=self.base_url,
|
||||
api_key=getattr(self, "api_key", ""),
|
||||
provider=self.provider,
|
||||
api_mode=self.api_mode,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.debug("LM Studio preload skipped: %s", err)
|
||||
return None
|
||||
|
||||
from hermes_cli.models import ensure_lmstudio_model_loaded
|
||||
|
||||
if config_context_length is None:
|
||||
config_context_length = getattr(self, "_config_context_length", None)
|
||||
return ensure_lmstudio_model_loaded(
|
||||
self.model,
|
||||
self.base_url,
|
||||
getattr(self, "api_key", ""),
|
||||
config_context_length,
|
||||
return_load_result=True,
|
||||
)
|
||||
|
||||
def switch_model(self, new_model, new_provider, api_key='', base_url='', api_mode=''):
|
||||
"""Forwarder — see ``agent.agent_runtime_helpers.switch_model``."""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,201 @@
|
|||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli import models
|
||||
|
||||
|
||||
MODEL = "publisher/model"
|
||||
BASE_URL = "http://127.0.0.1:1234/v1"
|
||||
|
||||
|
||||
class _JsonResponse:
|
||||
def __init__(self, payload):
|
||||
self._body = json.dumps(payload).encode()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return self._body
|
||||
|
||||
|
||||
def _catalog(*, loaded_context=None, maximum=262_144):
|
||||
loaded_instances = []
|
||||
if loaded_context is not None:
|
||||
loaded_instances.append({
|
||||
"id": f"{MODEL}:active",
|
||||
"config": {"context_length": loaded_context},
|
||||
})
|
||||
return [{
|
||||
"key": MODEL,
|
||||
"max_context_length": maximum,
|
||||
"loaded_instances": loaded_instances,
|
||||
}]
|
||||
|
||||
|
||||
def _capture_load(monkeypatch, response_payload):
|
||||
requests = []
|
||||
|
||||
def fake_open(request, *, timeout):
|
||||
requests.append((request, timeout, json.loads(request.data.decode())))
|
||||
return _JsonResponse(response_payload)
|
||||
|
||||
monkeypatch.setattr(models, "_urlopen_model_catalog_request", fake_open)
|
||||
return requests
|
||||
|
||||
|
||||
def test_loaded_64k_runtime_is_preserved_without_post(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
models,
|
||||
"_lmstudio_fetch_raw_models",
|
||||
lambda **_kwargs: _catalog(loaded_context=64_000),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
models,
|
||||
"_urlopen_model_catalog_request",
|
||||
lambda *_args, **_kwargs: pytest.fail("loaded model must not be reloaded"),
|
||||
)
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL, BASE_URL, api_key="", target_context_length=None
|
||||
)
|
||||
|
||||
assert result == 64_000
|
||||
|
||||
|
||||
def test_unloaded_no_override_omits_context_and_requests_echo(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
models, "_lmstudio_fetch_raw_models", lambda **_kwargs: _catalog()
|
||||
)
|
||||
requests = _capture_load(monkeypatch, {
|
||||
"load_config": {"context_length": 96_000},
|
||||
})
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL, BASE_URL, api_key="", target_context_length=None
|
||||
)
|
||||
|
||||
assert result == 96_000
|
||||
assert requests[0][2] == {"model": MODEL, "echo_load_config": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested_context", [32_000, 100_000])
|
||||
def test_unloaded_explicit_override_sends_exact_context(monkeypatch, requested_context):
|
||||
monkeypatch.setattr(
|
||||
models, "_lmstudio_fetch_raw_models", lambda **_kwargs: _catalog()
|
||||
)
|
||||
requests = _capture_load(monkeypatch, {
|
||||
"load_config": {"context_length": requested_context},
|
||||
})
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL, BASE_URL, api_key="", target_context_length=requested_context
|
||||
)
|
||||
|
||||
assert result == requested_context
|
||||
assert requests[0][2] == {
|
||||
"model": MODEL,
|
||||
"context_length": requested_context,
|
||||
"echo_load_config": True,
|
||||
}
|
||||
|
||||
|
||||
def test_echoed_applied_context_wins_over_requested_context(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
models, "_lmstudio_fetch_raw_models", lambda **_kwargs: _catalog()
|
||||
)
|
||||
_capture_load(monkeypatch, {"load_config": {"context_length": 96_000}})
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL, BASE_URL, api_key="", target_context_length=100_000
|
||||
)
|
||||
|
||||
assert result == 96_000
|
||||
|
||||
|
||||
def test_missing_echo_refreshes_loaded_state(monkeypatch):
|
||||
catalogs = iter([_catalog(), _catalog(loaded_context=88_000)])
|
||||
monkeypatch.setattr(
|
||||
models, "_lmstudio_fetch_raw_models", lambda **_kwargs: next(catalogs)
|
||||
)
|
||||
_capture_load(monkeypatch, {"status": "loaded"})
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL, BASE_URL, api_key="", target_context_length=100_000
|
||||
)
|
||||
|
||||
assert result == 88_000
|
||||
|
||||
|
||||
def test_successful_load_without_verifiable_context_returns_unknown(monkeypatch):
|
||||
catalogs = iter([_catalog(), _catalog()])
|
||||
monkeypatch.setattr(
|
||||
models, "_lmstudio_fetch_raw_models", lambda **_kwargs: next(catalogs)
|
||||
)
|
||||
_capture_load(monkeypatch, {"status": "loaded"})
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL,
|
||||
BASE_URL,
|
||||
api_key="",
|
||||
target_context_length=100_000,
|
||||
return_load_result=True,
|
||||
)
|
||||
|
||||
assert result.context_length is None
|
||||
assert result.load_attempted is True
|
||||
assert result.rejected is False
|
||||
|
||||
|
||||
def test_explicit_override_above_known_maximum_rejects_without_post(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
models,
|
||||
"_lmstudio_fetch_raw_models",
|
||||
lambda **_kwargs: _catalog(maximum=128_000),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
models,
|
||||
"_urlopen_model_catalog_request",
|
||||
lambda *_args, **_kwargs: pytest.fail("invalid override must not be posted"),
|
||||
)
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL,
|
||||
BASE_URL,
|
||||
api_key="",
|
||||
target_context_length=256_000,
|
||||
return_load_result=True,
|
||||
)
|
||||
|
||||
assert result.context_length is None
|
||||
assert result.load_attempted is False
|
||||
assert result.rejected is True
|
||||
|
||||
|
||||
def test_explicit_override_above_known_maximum_rejects_even_when_loaded(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
models,
|
||||
"_lmstudio_fetch_raw_models",
|
||||
lambda **_kwargs: _catalog(loaded_context=64_000, maximum=128_000),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
models,
|
||||
"_urlopen_model_catalog_request",
|
||||
lambda *_args, **_kwargs: pytest.fail("invalid override must not be posted"),
|
||||
)
|
||||
|
||||
result = models.ensure_lmstudio_model_loaded(
|
||||
MODEL,
|
||||
BASE_URL,
|
||||
api_key="",
|
||||
target_context_length=256_000,
|
||||
return_load_result=True,
|
||||
)
|
||||
|
||||
assert result.context_length is None
|
||||
assert result.load_attempted is False
|
||||
assert result.rejected is True
|
||||
|
|
@ -554,7 +554,7 @@ def test_lmstudio_load_post_drops_bearer_on_redirect(monkeypatch):
|
|||
source.shutdown()
|
||||
sink.shutdown()
|
||||
|
||||
assert loaded == 4096
|
||||
assert loaded is None
|
||||
method, headers = _RecordingHandler.requests[-1]
|
||||
assert method == "GET"
|
||||
assert "authorization" not in headers
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
from hermes_cli.models import LMStudioLoadResult
|
||||
from run_agent import AIAgent
|
||||
|
||||
|
||||
|
|
@ -22,29 +23,32 @@ def test_lmstudio_jit_load_mode_skips_explicit_preload(monkeypatch):
|
|||
|
||||
def fake_ensure(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return 64000
|
||||
return LMStudioLoadResult(64_000)
|
||||
|
||||
monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure)
|
||||
|
||||
AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("jit")))
|
||||
result = AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("jit")))
|
||||
|
||||
assert result is None
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_lmstudio_explicit_load_mode_preserves_preload(monkeypatch):
|
||||
def test_lmstudio_explicit_load_mode_passes_no_override_as_none(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fake_ensure(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return 64000
|
||||
return LMStudioLoadResult(96_000, load_attempted=True)
|
||||
|
||||
monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure)
|
||||
|
||||
AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("explicit")))
|
||||
result = AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("explicit")))
|
||||
|
||||
assert result.context_length == 96_000
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0][:3] == ("test/model", "http://127.0.0.1:1234/v1", "")
|
||||
assert calls[0][0][3] == 64000
|
||||
assert calls[0][0][3] is None
|
||||
assert calls[0][1]["return_load_result"] is True
|
||||
|
||||
|
||||
def test_missing_lmstudio_load_mode_defaults_to_explicit(monkeypatch):
|
||||
|
|
@ -54,10 +58,28 @@ def test_missing_lmstudio_load_mode_defaults_to_explicit(monkeypatch):
|
|||
|
||||
def fake_ensure(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return 64000
|
||||
return LMStudioLoadResult(64_000)
|
||||
|
||||
monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure)
|
||||
|
||||
AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, agent))
|
||||
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_explicit_budget_below_loaded_runtime_limits_effective_context():
|
||||
result = AIAgent._effective_lmstudio_context_length(
|
||||
80_000,
|
||||
LMStudioLoadResult(120_000),
|
||||
)
|
||||
|
||||
assert result == 80_000
|
||||
|
||||
|
||||
def test_attempted_unverified_load_has_no_effective_context():
|
||||
result = AIAgent._effective_lmstudio_context_length(
|
||||
100_000,
|
||||
LMStudioLoadResult(None, load_attempted=True),
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch
|
|||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.models import LMStudioLoadResult
|
||||
from run_agent import AIAgent
|
||||
from agent.agent_init import _normalize_route_base_url
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
|
@ -1020,3 +1021,42 @@ def test_direct_start_runtime_first_provider_names_require_explicit_custom_prefi
|
|||
base_url=base_url,
|
||||
)
|
||||
assert custom_agent.context_compressor.config_context_length == 1_048_576
|
||||
|
||||
|
||||
def test_lmstudio_switch_uses_destination_context_and_verified_runtime(monkeypatch):
|
||||
agent = _make_agent_with_compressor(config_context_length=32_768)
|
||||
calls = []
|
||||
|
||||
def fake_load_config():
|
||||
return {}
|
||||
|
||||
def fake_compatible(_cfg):
|
||||
return [{"name": "lmstudio", "base_url": "http://127.0.0.1:1234/v1"}]
|
||||
|
||||
def fake_provider_context(*, model, base_url, custom_providers):
|
||||
assert model == "lmstudio/new-model"
|
||||
assert base_url == "http://127.0.0.1:1234/v1"
|
||||
return 120_000
|
||||
|
||||
def fake_lmstudio_load(self, config_context_length=None):
|
||||
calls.append(config_context_length)
|
||||
return LMStudioLoadResult(100_000)
|
||||
|
||||
monkeypatch.setattr("hermes_cli.config.load_config", fake_load_config)
|
||||
monkeypatch.setattr("hermes_cli.config.get_compatible_custom_providers", fake_compatible)
|
||||
monkeypatch.setattr("hermes_cli.config.get_custom_provider_context_length", fake_provider_context)
|
||||
monkeypatch.setattr(AIAgent, "_ensure_lmstudio_runtime_loaded", fake_lmstudio_load)
|
||||
|
||||
with patch("agent.model_metadata.get_model_context_length", return_value=100_000) as mock_ctx_len:
|
||||
agent.switch_model(
|
||||
"lmstudio/new-model",
|
||||
"lmstudio",
|
||||
api_key="",
|
||||
base_url="http://127.0.0.1:1234/v1",
|
||||
)
|
||||
|
||||
assert calls == [120_000]
|
||||
call_kwargs = mock_ctx_len.call_args.kwargs
|
||||
assert call_kwargs.get("config_context_length") == 100_000
|
||||
assert agent._config_context_length == 120_000
|
||||
assert agent.context_compressor.context_length == 100_000
|
||||
|
|
|
|||
|
|
@ -844,7 +844,7 @@ hermes model
|
|||
# If LM Studio server auth is enabled, enter LM_API_KEY when prompted
|
||||
```
|
||||
|
||||
By default, Hermes explicitly asks LM Studio to load the selected model with 64K context length before the first request.
|
||||
Hermes preserves the context of an already-loaded LM Studio instance. For an unloaded model in the default explicit mode, Hermes omits `context_length` unless you configured one in Hermes, so LM Studio can apply its own model setting. Hermes then uses only the context length LM Studio reports after loading.
|
||||
|
||||
To change context length in LM Studio:
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue