From f0794640f61be5d0b3d657750b9617cd08ff1464 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:16:52 +0530 Subject: [PATCH] feat(sessions): config-gate transcript safety limits sessions.max_resume_messages / sessions.max_export_messages (default 20000, 0 disables) replace the hardcoded hard-rejects, and the CLI 'sessions export' guard becomes per-session instead of cumulative so full-DB backups of many small sessions keep working. Error guidance now points at the config override instead of the (corruption-only) repair command. --- hermes_cli/config_defaults.py | 11 +++ hermes_cli/console_engine.py | 23 +++--- hermes_state.py | 71 ++++++++++++++++-- tests/hermes_cli/test_console_engine.py | 99 ++++++++++++++++++++++--- tests/test_hermes_state.py | 40 ++++++++++ tui_gateway/methods_session.py | 13 +++- 6 files changed, 229 insertions(+), 28 deletions(-) diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index d18b0a86eee3e..b3073d9517a2d 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2782,6 +2782,17 @@ DEFAULT_CONFIG = { # attributable per query shape. 0 logs every search. Bridged to # HERMES_SEARCH_SLOW_MS (internal carrier). "search_slow_ms": 1000, + # Transcript safety limits. A runaway session (hundreds of thousands + # of rows) can exhaust memory when its transcript is materialized in + # one shot, so interactive resume and in-memory export are guarded by + # bounded row counts. Set a limit to 0 to disable that guard. + # Max active messages (across the full compression lineage) a session + # may hold and still be resumed interactively (CLI/TUI/desktop). + "max_resume_messages": 20000, + # Max active messages a single session may hold for an in-memory + # (non-streaming) export such as `hermes sessions export`. Checked + # per session, so full-DB backups of many small sessions still work. + "max_export_messages": 20000, }, # Contextual first-touch onboarding hints (see agent/onboarding.py). diff --git a/hermes_cli/console_engine.py b/hermes_cli/console_engine.py index febfb629badd2..dd1922af0a6ed 100644 --- a/hermes_cli/console_engine.py +++ b/hermes_cli/console_engine.py @@ -1415,26 +1415,31 @@ def _sessions_export(_engine: HermesConsoleEngine, args: list[str]) -> str: def _run() -> None: from hermes_state import ( - MAX_SAFE_EXPORT_MESSAGES, SessionDB, SessionExportTooLargeError, + resolved_max_export_messages, ) db = SessionDB() try: def _guard_exports(session_ids: list[str]) -> None: - total_messages = 0 + # Per-session budget: each session is checked independently + # against the configured limit, so a full-DB backup of many + # small sessions never trips the guard — only an individual + # runaway transcript does. 0 disables the guard. + limit = resolved_max_export_messages() + if limit <= 0: + return try: for session_id in session_ids: - total_messages += db.assert_export_safe( - session_id, - max_messages=MAX_SAFE_EXPORT_MESSAGES - total_messages, - ) + db.assert_export_safe(session_id, max_messages=limit) except SessionExportTooLargeError as exc: raise ConsoleCommandError( - f"Export includes more than {MAX_SAFE_EXPORT_MESSAGES:,} active " - f"messages (limit reached at session '{exc.session_id}'). " - "Use the Sessions page's streaming Export action instead." + f"Session '{exc.session_id}' has more than {limit:,} active " + "messages; in-memory export is capped per session. " + "Use the Sessions page's streaming Export action, or set " + "sessions.max_export_messages: 0 in config.yaml to disable " + "the guard." ) from exc if ns.session_id: diff --git a/hermes_state.py b/hermes_state.py index 8271f3ad9b17a..3bb21e0ec54ae 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -88,14 +88,51 @@ MAX_SAFE_RESUME_MESSAGES = 20_000 MAX_SAFE_EXPORT_MESSAGES = 20_000 +def _configured_transcript_limit(key: str, fallback: int) -> int: + """Resolve a transcript safety limit from config at call time. + + Reads ``sessions.`` from config.yaml lazily (avoiding a circular + import at module load) and falls back to the module constant when the + config subsystem is unavailable (scaffold installs, stripped test + environments). A value of 0 disables the guard entirely. No caching: + ``load_config_readonly`` is already mtime-cached, and resolving fresh + keeps tests that monkeypatch config or the module constants working. + """ + try: + from hermes_cli.config import load_config_readonly + + sessions_cfg = load_config_readonly().get("sessions") or {} + value = sessions_cfg.get(key) + if value is None: + return fallback + limit = int(value) + return limit if limit >= 0 else fallback + except Exception: + return fallback + + +def resolved_max_resume_messages() -> int: + """Config-resolved resume guard limit (0 disables the guard).""" + return _configured_transcript_limit( + "max_resume_messages", MAX_SAFE_RESUME_MESSAGES + ) + + +def resolved_max_export_messages() -> int: + """Config-resolved in-memory export guard limit (0 disables the guard).""" + return _configured_transcript_limit( + "max_export_messages", MAX_SAFE_EXPORT_MESSAGES + ) + + class SessionResumeTooLargeError(ValueError): def __init__(self, message_count: int, limit: int = MAX_SAFE_RESUME_MESSAGES): self.message_count = message_count self.limit = limit super().__init__( f"session has at least {message_count} active messages across its lineage; " - f"safe resume limit is {limit}. Export or repair the session before " - "resuming it." + f"safe resume limit is {limit}. Export the session instead, or set " + "sessions.max_resume_messages: 0 in config.yaml to disable the guard." ) @@ -7869,11 +7906,22 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) def assert_resume_safe( self, session_id: str, - max_messages: int = MAX_SAFE_RESUME_MESSAGES, + max_messages: Optional[int] = None, ) -> int: - """Return resume row count or reject a transcript too large to load.""" + """Return resume row count or reject a transcript too large to load. + + ``max_messages=None`` resolves the limit from config + (``sessions.max_resume_messages``); 0 disables the guard and returns + the (bounded) count without raising. + """ + if max_messages is None: + max_messages = resolved_max_resume_messages() if max_messages < 0: raise ValueError("max_messages must be non-negative") + if max_messages == 0: + # Guard disabled by config — never materialize an unbounded + # COUNT here; callers only need "safe", not an exact figure. + return self.get_resume_message_count(session_id) session_ids = self._session_lineage_root_to_tip(session_id) placeholders = ",".join("?" for _ in session_ids) with self._read_ctx() as conn: @@ -7892,17 +7940,30 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) def assert_export_safe( self, session_id: str, - max_messages: int = MAX_SAFE_EXPORT_MESSAGES, + max_messages: Optional[int] = None, ) -> int: """Return active row count or reject an unsafe in-memory export. Exporting one session does not include compression ancestors, so this guard deliberately counts only the requested segment. The limited subquery stops as soon as it proves the transcript exceeds the bound. + + ``max_messages=None`` resolves the limit from config + (``sessions.max_export_messages``); 0 disables the guard and returns + the active row count without raising. """ + if max_messages is None: + max_messages = resolved_max_export_messages() if max_messages < 0: raise ValueError("max_messages must be non-negative") with self._read_ctx() as conn: + if max_messages == 0: + row = conn.execute( + "SELECT COUNT(*) FROM messages " + "WHERE session_id = ? AND active = 1", + (session_id,), + ).fetchone() + return int(row[0] if row else 0) row = conn.execute( "SELECT COUNT(*) FROM (" "SELECT 1 FROM messages WHERE session_id = ? AND active = 1 LIMIT ?" diff --git a/tests/hermes_cli/test_console_engine.py b/tests/hermes_cli/test_console_engine.py index fbd406e12af10..b99f7108db40a 100644 --- a/tests/hermes_cli/test_console_engine.py +++ b/tests/hermes_cli/test_console_engine.py @@ -280,7 +280,7 @@ def test_sessions_export_rejects_oversized_single_before_touching_output( finally: db.close() - monkeypatch.setattr(hermes_state, "MAX_SAFE_EXPORT_MESSAGES", 2) + monkeypatch.setattr(hermes_state, "resolved_max_export_messages", lambda: 2) materialized = [] original_export_session = SessionDB.export_session @@ -305,7 +305,55 @@ def test_sessions_export_rejects_oversized_single_before_touching_output( assert output.read_text(encoding="utf-8") == "keep me\n" -def test_sessions_export_all_preflights_before_export_all( +def test_sessions_export_all_uses_per_session_budget( + _isolate_hermes_home, + monkeypatch, + tmp_path, +): + """N small sessions export fine; ONE oversized session still rejects. + + The budget is per session, not cumulative across the export set — + a cumulative budget broke full-DB backups of many small sessions. + """ + import json + + import hermes_state + from hermes_state import SessionDB + + db = SessionDB() + try: + for name in ("first-safe", "second-safe", "third-safe"): + db.create_session(name, source="cli") + db.append_messages_batch( + name, + [{"role": "user", "content": f"{name}-{i}"} for i in range(2)], + ) + finally: + db.close() + + monkeypatch.setattr(hermes_state, "resolved_max_export_messages", lambda: 3) + output = tmp_path / "all-sessions.jsonl" + + # 3 sessions x 2 messages = 6 total > 3, but each session is under the + # per-session limit, so the full-DB export succeeds. + result = HermesConsoleEngine().execute( + f"sessions export {output}", + confirmed=True, + ) + assert result.status == "ok" + exported = [ + json.loads(line) + for line in output.read_text(encoding="utf-8").splitlines() + if line + ] + assert {row["id"] for row in exported} == { + "first-safe", + "second-safe", + "third-safe", + } + + +def test_sessions_export_all_rejects_single_oversized_session( _isolate_hermes_home, monkeypatch, tmp_path, @@ -315,20 +363,20 @@ def test_sessions_export_all_preflights_before_export_all( db = SessionDB() try: - db.create_session("first-safe", source="cli") + db.create_session("small", source="cli") db.append_messages_batch( - "first-safe", - [{"role": "user", "content": f"first-{i}"} for i in range(2)], + "small", + [{"role": "user", "content": f"small-{i}"} for i in range(2)], ) - db.create_session("second-safe", source="cli") + db.create_session("runaway", source="cli") db.append_messages_batch( - "second-safe", - [{"role": "user", "content": f"second-{i}"} for i in range(2)], + "runaway", + [{"role": "user", "content": f"runaway-{i}"} for i in range(4)], ) finally: db.close() - monkeypatch.setattr(hermes_state, "MAX_SAFE_EXPORT_MESSAGES", 3) + monkeypatch.setattr(hermes_state, "resolved_max_export_messages", lambda: 3) export_all_calls = [] def tracked_export_all(self, source=None): @@ -344,12 +392,43 @@ def test_sessions_export_all_preflights_before_export_all( ) assert result.status == "error" - assert "more than 3 active messages" in result.output + assert "runaway" in result.output + assert "more than 3 active" in result.output assert "streaming Export" in result.output + assert "max_export_messages" in result.output assert export_all_calls == [] assert not output.exists() +def test_sessions_export_zero_limit_disables_guard( + _isolate_hermes_home, + monkeypatch, + tmp_path, +): + import hermes_state + from hermes_state import SessionDB + + db = SessionDB() + try: + db.create_session("huge", source="cli") + db.append_messages_batch( + "huge", + [{"role": "user", "content": f"huge-{i}"} for i in range(5)], + ) + finally: + db.close() + + monkeypatch.setattr(hermes_state, "resolved_max_export_messages", lambda: 0) + output = tmp_path / "huge.jsonl" + + result = HermesConsoleEngine().execute( + f"sessions export {output} --session-id huge", + confirmed=True, + ) + assert result.status == "ok" + assert output.exists() + + def test_cron_pause_resume_and_run_require_confirmation(_isolate_hermes_home): from cron.jobs import create_job, get_job diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index debfa60b93017..3c08c8744a842 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -3820,6 +3820,46 @@ class TestGetMessagesPagination: assert exc_info.value.message_count == 3 assert exc_info.value.limit == 2 + def test_zero_limit_disables_resume_and_export_guards(self, db, monkeypatch): + """sessions.max_*_messages: 0 disables the guard entirely.""" + db.create_session(session_id="big", source="cli") + db.append_messages_batch( + "big", + [{"role": "user", "content": f"msg-{i}"} for i in range(5)], + ) + + # A small explicit limit rejects... + with pytest.raises(hermes_state.SessionResumeTooLargeError): + db.assert_resume_safe("big", max_messages=2) + with pytest.raises(hermes_state.SessionExportTooLargeError): + db.assert_export_safe("big", max_messages=2) + + # ...but a config-resolved limit of 0 disables both guards and + # returns the true count without raising. + monkeypatch.setattr(hermes_state, "resolved_max_resume_messages", lambda: 0) + monkeypatch.setattr(hermes_state, "resolved_max_export_messages", lambda: 0) + assert db.assert_resume_safe("big") == 5 + assert db.assert_export_safe("big") == 5 + # An explicit 0 disables too, independent of config. + assert db.assert_resume_safe("big", max_messages=0) == 5 + assert db.assert_export_safe("big", max_messages=0) == 5 + + def test_guard_limits_resolve_from_config_at_call_time(self, db, monkeypatch): + db.create_session(session_id="cfg", source="cli") + db.append_messages_batch( + "cfg", + [{"role": "user", "content": f"msg-{i}"} for i in range(4)], + ) + + monkeypatch.setattr(hermes_state, "resolved_max_resume_messages", lambda: 3) + monkeypatch.setattr(hermes_state, "resolved_max_export_messages", lambda: 3) + with pytest.raises(hermes_state.SessionResumeTooLargeError) as resume_exc: + db.assert_resume_safe("cfg") + assert resume_exc.value.limit == 3 + with pytest.raises(hermes_state.SessionExportTooLargeError) as export_exc: + db.assert_export_safe("cfg") + assert export_exc.value.limit == 3 + diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 736910b212794..fe92b116d71f3 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -382,17 +382,22 @@ def _(rid, params: dict) -> dict: # omit_messages suppresses the response copy. Count the complete lineage # before any reopen/history read so a runaway transcript cannot exhaust # the dashboard. The metadata fallback keeps lightweight test/adaptor DBs - # that predate the shared SessionDB guard compatible. - from hermes_state import MAX_SAFE_RESUME_MESSAGES, SessionResumeTooLargeError + # that predate the shared SessionDB guard compatible. The limit resolves + # from config (sessions.max_resume_messages, 0 disables). + from hermes_state import ( + SessionResumeTooLargeError, + resolved_max_resume_messages, + ) safety_check = getattr(db, "assert_resume_safe", None) try: if callable(safety_check): safety_check(target) else: + resume_limit = resolved_max_resume_messages() stored_message_count = int(found.get("message_count") or 0) - if stored_message_count > MAX_SAFE_RESUME_MESSAGES: - raise SessionResumeTooLargeError(stored_message_count) + if resume_limit and stored_message_count > resume_limit: + raise SessionResumeTooLargeError(stored_message_count, resume_limit) except SessionResumeTooLargeError as exc: return _err(rid, 4130, str(exc)) except Exception as exc: