diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 458db27140c91..7155809b0ce98 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -216,6 +216,111 @@ def check_info(text: str): print(f" {color('→', Colors.CYAN)} {text}") +# ── state.db health/stats thresholds (advisory only — module constants, +# deliberately NOT config: doctor warnings are guidance, not policy) ── +STATE_DB_SIZE_WARN_BYTES = 1 * 1024 * 1024 * 1024 # 1 GiB logical size +STATE_DB_WAL_WARN_BYTES = 256 * 1024 * 1024 # 256 MiB WAL + + +def _human_bytes(n) -> str: + """1234567 → '1.2 MB' (GB/MB/KB/B).""" + try: + n = int(n) + except (TypeError, ValueError): + return "?" + if n >= 1024 ** 3: + return f"{n / (1024 ** 3):.1f} GB" + if n >= 1024 ** 2: + return f"{n / (1024 ** 2):.1f} MB" + if n >= 1024: + return f"{n / 1024:.1f} KB" + return f"{n} B" + + +def _render_state_db_stats(stats: dict, holders=None) -> list: + """Turn a collect_state_db_stats() dict into doctor output lines. + + Returns a list of ``(kind, text, detail)`` tuples where kind is one of + 'info' / 'warn'. Pure formatting — no I/O — so it is unit-testable + without spawning the doctor CLI. Tolerates None in every field. + """ + lines: list = [] + stats = stats or {} + + logical = stats.get("logical_size_bytes") + wal = stats.get("wal_size_bytes") + freelist = stats.get("freelist_count") + + size_bits = [] + if logical is not None: + size_bits.append(f"logical size {_human_bytes(logical)}") + if stats.get("page_count") is not None: + size_bits.append(f"{stats['page_count']:,} pages") + if freelist is not None: + size_bits.append(f"{freelist:,} free") + if wal is not None: + size_bits.append(f"WAL {_human_bytes(wal)}") + if size_bits: + lines.append(("info", "state.db " + ", ".join(size_bits), "")) + + row_bits = [] + if stats.get("messages") is not None: + row_bits.append(f"{stats['messages']:,} messages") + if stats.get("sessions") is not None: + row_bits.append(f"{stats['sessions']:,} sessions") + if stats.get("journal_mode"): + row_bits.append(f"journal_mode={stats['journal_mode']}") + if holders is not None: + row_bits.append(f"{holders} process(es) holding the DB open") + if row_bits: + lines.append(("info", ", ".join(row_bits), "")) + + fts = stats.get("fts_tables") + if fts: + present = [t for t, ok in fts.items() if ok] + lines.append(( + "info", + "FTS tables: " + (", ".join(present) if present else "none"), + "", + )) + + # Advisory: oversized database. Suggest auto_prune, and — when the v23 + # FTS rebuild is pending OR the DB still carries the legacy inline + # trigram layout (fts_storage_version marker absent) — the offline + # optimize-storage pass that migrates/compacts the FTS indexes. + if logical is not None and logical > STATE_DB_SIZE_WARN_BYTES: + detail = ( + "consider enabling sessions.auto_prune in config.yaml " + "to bound growth" + ) + legacy_trigram = ( + fts is not None + and fts.get("messages_fts_trigram") + and stats.get("fts_storage_version") is None + ) + if stats.get("fts_rebuild_pending") or legacy_trigram: + detail += ( + "; run 'hermes sessions optimize-storage' offline " + "(with the gateway stopped) to compact FTS storage" + ) + lines.append(( + "warn", + f"state.db is large ({_human_bytes(logical)})", + f"({detail})", + )) + + # Advisory: WAL runaway — checkpoints are not landing. + if wal is not None and wal > STATE_DB_WAL_WARN_BYTES: + lines.append(( + "warn", + f"state.db WAL is very large ({_human_bytes(wal)})", + "(checkpoints may not be completing — a long-lived reader or " + "stuck process can block them; see 'hermes doctor --fix')", + )) + + return lines + + def _section(title: str) -> None: """Print a doctor section banner: blank line + bold cyan ◆ title.""" print() @@ -1594,6 +1699,36 @@ def run_doctor(args): ) else: check_warn(f"{_DHH}/state.db exists but has issues: {e}") + + # Health/stats snapshot (#statedb-visibility): a multi-GB state.db + # with a runaway WAL was previously invisible to every Hermes + # surface. Strictly read-only (mode=ro) so it is safe against a + # live DB held by the gateway; any failure degrades to one info + # line rather than failing doctor. + try: + from hermes_state import collect_state_db_stats, count_db_holders + + _db_stats = collect_state_db_stats(state_db_path) + _db_holders = count_db_holders(state_db_path) + for _kind, _text, _detail in _render_state_db_stats( + _db_stats, holders=_db_holders + ): + if _kind == "warn": + check_warn(_text, _detail) + if "auto_prune" in _detail: + issues.append( + "state.db is large — enable sessions.auto_prune " + "in config.yaml" + + ( + " and run 'hermes sessions optimize-storage' " + "offline (gateway stopped)" + if "optimize-storage" in _detail else "" + ) + ) + else: + check_info(_text + (f" {_detail}" if _detail else "")) + except Exception as _stats_exc: + check_info(f"state.db stats unavailable ({_stats_exc})") else: check_info(f"{_DHH}/state.db not created yet (will be created on first session)") diff --git a/hermes_state.py b/hermes_state.py index 917270840359e..3d98d4241cc88 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -2162,6 +2162,178 @@ def quarantine_zeroed_state_db(path: Path) -> Optional[Path]: handle.close() +# ── Read-only health/stats probes (hermes doctor, dashboards) ────────── + + +def collect_state_db_stats(db_path: Path) -> Dict[str, Any]: + """Best-effort, strictly read-only stats snapshot of a state.db file. + + Opens the database with ``mode=ro`` (URI) and a short timeout so it can + run against a *live* database held by a gateway without ever taking a + write lock or mutating the file. Every field is collected independently: + a failed pragma/SELECT yields ``None`` for that field, and the helper + itself never raises. + + Deliberately does NOT instantiate :class:`SessionDB` — its constructor + runs schema DDL (migrations, FTS table creation), which is exactly the + kind of write a diagnostics probe must never perform. + + Returned keys (all present, any may be None on failure): + + - ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values + - ``logical_size_bytes`` — page_count * page_size (post-checkpoint size) + - ``wal_size_bytes`` — stat() of ``-wal`` (0 when absent) + - ``journal_mode`` — PRAGMA journal_mode string + - ``messages`` / ``sessions`` — row counts + - ``fts_tables`` — dict of {table_name: bool} presence for + messages_fts / messages_fts_trigram / messages_fts_cjk + - ``fts_storage_version`` — int from state_meta, None when the marker is + absent (legacy pre-v23 inline layout) + - ``fts_rebuild_pending`` — True when the deferred v23 backfill has not + finished (high_water present and progress < high_water) + - ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints + """ + stats: Dict[str, Any] = { + "page_count": None, + "page_size": None, + "freelist_count": None, + "logical_size_bytes": None, + "wal_size_bytes": None, + "journal_mode": None, + "messages": None, + "sessions": None, + "fts_tables": None, + "fts_storage_version": None, + "fts_rebuild_pending": None, + "fts_rebuild_high_water": None, + "fts_rebuild_progress": None, + } + + # WAL sidecar size needs no connection at all. + try: + wal_path = Path(str(db_path) + "-wal") + stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0 + except OSError: + pass + + conn = None + try: + # mode=ro refuses to create the file and refuses every write; a + # short timeout keeps doctor snappy when a writer holds the lock. + conn = sqlite3.connect( + f"file:{Path(db_path)}?mode=ro", uri=True, timeout=2.0 + ) + except Exception as exc: + logger.debug("collect_state_db_stats: cannot open %s read-only: %s", + db_path, exc) + return stats + + def _scalar(sql: str) -> Any: + try: + row = conn.execute(sql).fetchone() + return row[0] if row else None + except Exception: + return None + + try: + pc = _scalar("PRAGMA page_count") + ps = _scalar("PRAGMA page_size") + stats["page_count"] = int(pc) if pc is not None else None + stats["page_size"] = int(ps) if ps is not None else None + if stats["page_count"] is not None and stats["page_size"] is not None: + stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"] + + fl = _scalar("PRAGMA freelist_count") + stats["freelist_count"] = int(fl) if fl is not None else None + + jm = _scalar("PRAGMA journal_mode") + stats["journal_mode"] = str(jm) if jm is not None else None + + msgs = _scalar("SELECT COUNT(*) FROM messages") + stats["messages"] = int(msgs) if msgs is not None else None + sess = _scalar("SELECT COUNT(*) FROM sessions") + stats["sessions"] = int(sess) if sess is not None else None + + # FTS table presence via sqlite_master (never SELECTs from the + # virtual tables themselves — a corrupt index must not fail stats). + try: + names = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name IN (?, ?, ?)", + ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"), + ).fetchall() + } + stats["fts_tables"] = { + t: (t in names) + for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") + } + except Exception: + pass + + # Raw state_meta reads — cheap, and independent of SessionDB. + def _meta_int(key: str) -> Optional[int]: + try: + row = conn.execute( + "SELECT value FROM state_meta WHERE key = ?", (key,) + ).fetchone() + return int(row[0]) if row and row[0] is not None else None + except Exception: + return None + + stats["fts_storage_version"] = _meta_int("fts_storage_version") + high_water = _meta_int("fts_rebuild_high_water") + progress = _meta_int("fts_rebuild_progress") + stats["fts_rebuild_high_water"] = high_water + stats["fts_rebuild_progress"] = progress + if high_water is None: + stats["fts_rebuild_pending"] = False + else: + stats["fts_rebuild_pending"] = (progress or 0) < high_water + finally: + try: + conn.close() + except Exception: + pass + + return stats + + +def count_db_holders(db_path: Path) -> Optional[int]: + """Best-effort count of processes holding ``db_path`` open (Linux only). + + Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns + the number of distinct PIDs with the file open, or ``None`` on any + error or on non-Linux platforms. Never raises; no lsof dependency. + Unreadable per-process fd dirs (other users' processes without root) + are silently skipped, so the count is a lower bound. + """ + try: + if not sys.platform.startswith("linux"): + return None + target = os.path.realpath(str(db_path)) + holders = 0 + for pid in os.listdir("/proc"): + if not pid.isdigit(): + continue + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + continue # process gone or not ours + for fd in fds: + try: + if os.readlink(f"{fd_dir}/{fd}") == target: + holders += 1 + break # one hit per PID + except OSError: + continue + return holders + except Exception: + return None + + class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin): """ SQLite-backed session storage with FTS5 search. diff --git a/tests/test_state_db_stats.py b/tests/test_state_db_stats.py new file mode 100644 index 0000000000000..fff1d2920da26 --- /dev/null +++ b/tests/test_state_db_stats.py @@ -0,0 +1,227 @@ +"""Tests for state.db health/stats collection (hermes doctor section). + +Covers: +- ``hermes_state.collect_state_db_stats``: read-only, best-effort stats + (page_count, freelist, WAL size, journal mode, row counts, FTS presence, + pending v23 FTS-rebuild bookkeeping). +- ``hermes_state.count_db_holders``: /proc-based best-effort probe for how + many processes hold the DB file open (Linux only; None elsewhere/on error). +- ``hermes_cli.doctor._render_state_db_stats``: formatting/threshold helper + the doctor state.db section prints from. +""" + +import os +import sqlite3 +import sys +from pathlib import Path + +import pytest + +from hermes_state import SessionDB, collect_state_db_stats, count_db_holders + + +@pytest.fixture() +def populated_db(tmp_path): + """A real state.db built through SessionDB's public API, then closed.""" + db_path = tmp_path / "state.db" + db = SessionDB(db_path=db_path) + db.create_session("sess-alpha", "cli") + db.create_session("sess-beta", "cli") + db.append_message("sess-alpha", "user", "hello world one") + db.append_message("sess-alpha", "assistant", "hi back") + db.append_message("sess-beta", "user", "second session message") + db.close() + return db_path + + +# ── collect_state_db_stats ────────────────────────────────────────────── + + +def test_collect_stats_sane_values(populated_db): + stats = collect_state_db_stats(populated_db) + + assert isinstance(stats, dict) + assert stats["page_count"] and stats["page_count"] > 0 + assert stats["page_size"] and stats["page_size"] > 0 + assert stats["logical_size_bytes"] == stats["page_count"] * stats["page_size"] + assert isinstance(stats["freelist_count"], int) and stats["freelist_count"] >= 0 + assert isinstance(stats["wal_size_bytes"], int) and stats["wal_size_bytes"] >= 0 + assert isinstance(stats["journal_mode"], str) and stats["journal_mode"] + assert stats["messages"] >= 3 + assert stats["sessions"] >= 2 + + fts = stats["fts_tables"] + assert set(fts) == {"messages_fts", "messages_fts_trigram", "messages_fts_cjk"} + for name, present in fts.items(): + assert isinstance(present, bool), name + # Layout marker is absent (None) on legacy DBs, an int once stamped. + assert stats["fts_storage_version"] is None or isinstance( + stats["fts_storage_version"], int + ) + + # No rebuild pending on a fresh DB. + assert stats["fts_rebuild_pending"] in (False, None) + + +def test_collect_stats_is_read_only(populated_db): + before_bytes = populated_db.read_bytes() + before_mtime = populated_db.stat().st_mtime_ns + + collect_state_db_stats(populated_db) + + assert populated_db.read_bytes() == before_bytes + assert populated_db.stat().st_mtime_ns == before_mtime + # No stray -wal/-shm growth from the probe either: a subsequent normal + # open must still work. + db = SessionDB(db_path=populated_db) + assert db.get_session("sess-alpha") is not None + db.close() + + +def test_collect_stats_missing_file_never_raises(tmp_path): + stats = collect_state_db_stats(tmp_path / "nope" / "state.db") + assert isinstance(stats, dict) + assert stats["page_count"] is None + assert stats["messages"] is None + assert stats["logical_size_bytes"] is None + + +def test_collect_stats_rebuild_pending_flag(populated_db): + # Simulate the v23 deferred-rebuild bookkeeping directly in state_meta. + conn = sqlite3.connect(str(populated_db)) + conn.execute( + "INSERT OR REPLACE INTO state_meta(key, value) VALUES ('fts_rebuild_high_water', '100')" + ) + conn.execute( + "INSERT OR REPLACE INTO state_meta(key, value) VALUES ('fts_rebuild_progress', '40')" + ) + conn.commit() + conn.close() + + stats = collect_state_db_stats(populated_db) + assert stats["fts_rebuild_pending"] is True + assert stats["fts_rebuild_high_water"] == 100 + assert stats["fts_rebuild_progress"] == 40 + + +# ── count_db_holders ──────────────────────────────────────────────────── + + +def test_count_db_holders_sees_open_connection(populated_db): + conn = sqlite3.connect(str(populated_db)) + try: + holders = count_db_holders(populated_db) + if sys.platform.startswith("linux"): + assert isinstance(holders, int) + assert holders >= 1 + else: + assert holders is None + finally: + conn.close() + + +def test_count_db_holders_missing_path_no_raise(tmp_path): + holders = count_db_holders(tmp_path / "absent.db") + assert holders is None or isinstance(holders, int) + + +# ── doctor rendering helper ───────────────────────────────────────────── + + +def _base_stats(**overrides): + stats = { + "page_count": 100, + "page_size": 4096, + "freelist_count": 2, + "logical_size_bytes": 100 * 4096, + "wal_size_bytes": 1024, + "journal_mode": "wal", + "messages": 42, + "sessions": 7, + "fts_tables": { + "messages_fts": True, + "messages_fts_trigram": True, + "messages_fts_cjk": False, + }, + "fts_storage_version": 1, + "fts_rebuild_pending": False, + "fts_rebuild_high_water": None, + "fts_rebuild_progress": None, + } + stats.update(overrides) + return stats + + +def test_render_healthy_stats_no_warnings(): + from hermes_cli.doctor import _render_state_db_stats + + lines = _render_state_db_stats(_base_stats(), holders=2) + kinds = [k for k, *_ in lines] + assert "warn" not in kinds + joined = " | ".join(text for _, text, *_ in lines) + assert "42" in joined # messages + assert "7" in joined # sessions + assert "wal" in joined.lower() + assert "2" in joined # holders + + +def test_render_warns_on_large_db(): + from hermes_cli.doctor import ( + STATE_DB_SIZE_WARN_BYTES, + _render_state_db_stats, + ) + + big = STATE_DB_SIZE_WARN_BYTES + 1 + lines = _render_state_db_stats( + _base_stats(logical_size_bytes=big, page_count=big // 4096, page_size=4096), + holders=None, + ) + warns = [t for k, t, *rest in lines if k == "warn"] + [ + " ".join(rest) for k, t, *rest in lines if k == "warn" + ] + blob = " ".join(str(x) for x in warns) + assert "auto_prune" in blob + assert "config.yaml" in blob + + +def test_render_large_db_with_pending_rebuild_suggests_optimize(): + from hermes_cli.doctor import STATE_DB_SIZE_WARN_BYTES, _render_state_db_stats + + big = STATE_DB_SIZE_WARN_BYTES + 1 + lines = _render_state_db_stats( + _base_stats(logical_size_bytes=big, fts_rebuild_pending=True), + holders=None, + ) + blob = " ".join(" ".join(str(p) for p in line) for line in lines) + assert "optimize-storage" in blob + + +def test_render_large_db_legacy_trigram_suggests_optimize(): + from hermes_cli.doctor import STATE_DB_SIZE_WARN_BYTES, _render_state_db_stats + + big = STATE_DB_SIZE_WARN_BYTES + 1 + lines = _render_state_db_stats( + _base_stats(logical_size_bytes=big, fts_storage_version=None), + holders=None, + ) + blob = " ".join(" ".join(str(p) for p in line) for line in lines) + assert "optimize-storage" in blob + + +def test_render_warns_on_large_wal(): + from hermes_cli.doctor import STATE_DB_WAL_WARN_BYTES, _render_state_db_stats + + lines = _render_state_db_stats( + _base_stats(wal_size_bytes=STATE_DB_WAL_WARN_BYTES + 1), holders=None + ) + blob = " ".join(" ".join(str(p) for p in line) for line in lines).lower() + assert "checkpoint" in blob + + +def test_render_handles_all_none_stats(): + from hermes_cli.doctor import _render_state_db_stats + + empty = {k: None for k in _base_stats()} + empty["fts_tables"] = None + lines = _render_state_db_stats(empty, holders=None) + assert isinstance(lines, list) # must not raise