feat(doctor): state.db health stats — size, WAL, FTS shape, holders, growth warnings

Operators had no Hermes surface showing state.db size, WAL health, index
family shape, or how many processes hold the database — all of which were
needed to diagnose a lock-contention incident on a 4.5 GB multi-writer
install.

Adds collect_state_db_stats() (strictly read-only URI connection, no
SessionDB instantiation, per-field best-effort) and a /proc-based
count_db_holders() to hermes_state, and wires a stats block into hermes
doctor's state.db section: logical size, pages/freelist, WAL size,
message/session counts, journal mode, holder count, FTS table presence
and deferred-rebuild status. Advisory warnings at >1 GiB (suggest
sessions.auto_prune and, when the v23 rebuild is pending or the legacy
trigram shape is detected, an offline 'hermes sessions optimize-storage')
and >256 MiB WAL (checkpoint health). Any stats failure degrades to a
single info line.
This commit is contained in:
Victor Kyriazakos 2026-08-07 00:52:33 +00:00 committed by kshitij
parent 01bc8a8752
commit 64c342c1c9
3 changed files with 534 additions and 0 deletions

View File

@ -216,6 +216,111 @@ def check_info(text: str):
print(f" {color('', Colors.CYAN)} {text}")
# ── state.db health/stats thresholds (advisory only — module constants,
# deliberately NOT config: doctor warnings are guidance, not policy) ──
STATE_DB_SIZE_WARN_BYTES = 1 * 1024 * 1024 * 1024 # 1 GiB logical size
STATE_DB_WAL_WARN_BYTES = 256 * 1024 * 1024 # 256 MiB WAL
def _human_bytes(n) -> str:
"""1234567 → '1.2 MB' (GB/MB/KB/B)."""
try:
n = int(n)
except (TypeError, ValueError):
return "?"
if n >= 1024 ** 3:
return f"{n / (1024 ** 3):.1f} GB"
if n >= 1024 ** 2:
return f"{n / (1024 ** 2):.1f} MB"
if n >= 1024:
return f"{n / 1024:.1f} KB"
return f"{n} B"
def _render_state_db_stats(stats: dict, holders=None) -> list:
"""Turn a collect_state_db_stats() dict into doctor output lines.
Returns a list of ``(kind, text, detail)`` tuples where kind is one of
'info' / 'warn'. Pure formatting no I/O so it is unit-testable
without spawning the doctor CLI. Tolerates None in every field.
"""
lines: list = []
stats = stats or {}
logical = stats.get("logical_size_bytes")
wal = stats.get("wal_size_bytes")
freelist = stats.get("freelist_count")
size_bits = []
if logical is not None:
size_bits.append(f"logical size {_human_bytes(logical)}")
if stats.get("page_count") is not None:
size_bits.append(f"{stats['page_count']:,} pages")
if freelist is not None:
size_bits.append(f"{freelist:,} free")
if wal is not None:
size_bits.append(f"WAL {_human_bytes(wal)}")
if size_bits:
lines.append(("info", "state.db " + ", ".join(size_bits), ""))
row_bits = []
if stats.get("messages") is not None:
row_bits.append(f"{stats['messages']:,} messages")
if stats.get("sessions") is not None:
row_bits.append(f"{stats['sessions']:,} sessions")
if stats.get("journal_mode"):
row_bits.append(f"journal_mode={stats['journal_mode']}")
if holders is not None:
row_bits.append(f"{holders} process(es) holding the DB open")
if row_bits:
lines.append(("info", ", ".join(row_bits), ""))
fts = stats.get("fts_tables")
if fts:
present = [t for t, ok in fts.items() if ok]
lines.append((
"info",
"FTS tables: " + (", ".join(present) if present else "none"),
"",
))
# Advisory: oversized database. Suggest auto_prune, and — when the v23
# FTS rebuild is pending OR the DB still carries the legacy inline
# trigram layout (fts_storage_version marker absent) — the offline
# optimize-storage pass that migrates/compacts the FTS indexes.
if logical is not None and logical > STATE_DB_SIZE_WARN_BYTES:
detail = (
"consider enabling sessions.auto_prune in config.yaml "
"to bound growth"
)
legacy_trigram = (
fts is not None
and fts.get("messages_fts_trigram")
and stats.get("fts_storage_version") is None
)
if stats.get("fts_rebuild_pending") or legacy_trigram:
detail += (
"; run 'hermes sessions optimize-storage' offline "
"(with the gateway stopped) to compact FTS storage"
)
lines.append((
"warn",
f"state.db is large ({_human_bytes(logical)})",
f"({detail})",
))
# Advisory: WAL runaway — checkpoints are not landing.
if wal is not None and wal > STATE_DB_WAL_WARN_BYTES:
lines.append((
"warn",
f"state.db WAL is very large ({_human_bytes(wal)})",
"(checkpoints may not be completing — a long-lived reader or "
"stuck process can block them; see 'hermes doctor --fix')",
))
return lines
def _section(title: str) -> None:
"""Print a doctor section banner: blank line + bold cyan ◆ title."""
print()
@ -1594,6 +1699,36 @@ def run_doctor(args):
)
else:
check_warn(f"{_DHH}/state.db exists but has issues: {e}")
# Health/stats snapshot (#statedb-visibility): a multi-GB state.db
# with a runaway WAL was previously invisible to every Hermes
# surface. Strictly read-only (mode=ro) so it is safe against a
# live DB held by the gateway; any failure degrades to one info
# line rather than failing doctor.
try:
from hermes_state import collect_state_db_stats, count_db_holders
_db_stats = collect_state_db_stats(state_db_path)
_db_holders = count_db_holders(state_db_path)
for _kind, _text, _detail in _render_state_db_stats(
_db_stats, holders=_db_holders
):
if _kind == "warn":
check_warn(_text, _detail)
if "auto_prune" in _detail:
issues.append(
"state.db is large — enable sessions.auto_prune "
"in config.yaml"
+ (
" and run 'hermes sessions optimize-storage' "
"offline (gateway stopped)"
if "optimize-storage" in _detail else ""
)
)
else:
check_info(_text + (f" {_detail}" if _detail else ""))
except Exception as _stats_exc:
check_info(f"state.db stats unavailable ({_stats_exc})")
else:
check_info(f"{_DHH}/state.db not created yet (will be created on first session)")

View File

@ -2162,6 +2162,178 @@ def quarantine_zeroed_state_db(path: Path) -> Optional[Path]:
handle.close()
# ── Read-only health/stats probes (hermes doctor, dashboards) ──────────
def collect_state_db_stats(db_path: Path) -> Dict[str, Any]:
"""Best-effort, strictly read-only stats snapshot of a state.db file.
Opens the database with ``mode=ro`` (URI) and a short timeout so it can
run against a *live* database held by a gateway without ever taking a
write lock or mutating the file. Every field is collected independently:
a failed pragma/SELECT yields ``None`` for that field, and the helper
itself never raises.
Deliberately does NOT instantiate :class:`SessionDB` its constructor
runs schema DDL (migrations, FTS table creation), which is exactly the
kind of write a diagnostics probe must never perform.
Returned keys (all present, any may be None on failure):
- ``page_count``, ``page_size``, ``freelist_count`` PRAGMA values
- ``logical_size_bytes`` page_count * page_size (post-checkpoint size)
- ``wal_size_bytes`` stat() of ``<db>-wal`` (0 when absent)
- ``journal_mode`` PRAGMA journal_mode string
- ``messages`` / ``sessions`` row counts
- ``fts_tables`` dict of {table_name: bool} presence for
messages_fts / messages_fts_trigram / messages_fts_cjk
- ``fts_storage_version`` int from state_meta, None when the marker is
absent (legacy pre-v23 inline layout)
- ``fts_rebuild_pending`` True when the deferred v23 backfill has not
finished (high_water present and progress < high_water)
- ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` raw ints
"""
stats: Dict[str, Any] = {
"page_count": None,
"page_size": None,
"freelist_count": None,
"logical_size_bytes": None,
"wal_size_bytes": None,
"journal_mode": None,
"messages": None,
"sessions": None,
"fts_tables": None,
"fts_storage_version": None,
"fts_rebuild_pending": None,
"fts_rebuild_high_water": None,
"fts_rebuild_progress": None,
}
# WAL sidecar size needs no connection at all.
try:
wal_path = Path(str(db_path) + "-wal")
stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0
except OSError:
pass
conn = None
try:
# mode=ro refuses to create the file and refuses every write; a
# short timeout keeps doctor snappy when a writer holds the lock.
conn = sqlite3.connect(
f"file:{Path(db_path)}?mode=ro", uri=True, timeout=2.0
)
except Exception as exc:
logger.debug("collect_state_db_stats: cannot open %s read-only: %s",
db_path, exc)
return stats
def _scalar(sql: str) -> Any:
try:
row = conn.execute(sql).fetchone()
return row[0] if row else None
except Exception:
return None
try:
pc = _scalar("PRAGMA page_count")
ps = _scalar("PRAGMA page_size")
stats["page_count"] = int(pc) if pc is not None else None
stats["page_size"] = int(ps) if ps is not None else None
if stats["page_count"] is not None and stats["page_size"] is not None:
stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"]
fl = _scalar("PRAGMA freelist_count")
stats["freelist_count"] = int(fl) if fl is not None else None
jm = _scalar("PRAGMA journal_mode")
stats["journal_mode"] = str(jm) if jm is not None else None
msgs = _scalar("SELECT COUNT(*) FROM messages")
stats["messages"] = int(msgs) if msgs is not None else None
sess = _scalar("SELECT COUNT(*) FROM sessions")
stats["sessions"] = int(sess) if sess is not None else None
# FTS table presence via sqlite_master (never SELECTs from the
# virtual tables themselves — a corrupt index must not fail stats).
try:
names = {
row[0]
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' "
"AND name IN (?, ?, ?)",
("messages_fts", "messages_fts_trigram", "messages_fts_cjk"),
).fetchall()
}
stats["fts_tables"] = {
t: (t in names)
for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
}
except Exception:
pass
# Raw state_meta reads — cheap, and independent of SessionDB.
def _meta_int(key: str) -> Optional[int]:
try:
row = conn.execute(
"SELECT value FROM state_meta WHERE key = ?", (key,)
).fetchone()
return int(row[0]) if row and row[0] is not None else None
except Exception:
return None
stats["fts_storage_version"] = _meta_int("fts_storage_version")
high_water = _meta_int("fts_rebuild_high_water")
progress = _meta_int("fts_rebuild_progress")
stats["fts_rebuild_high_water"] = high_water
stats["fts_rebuild_progress"] = progress
if high_water is None:
stats["fts_rebuild_pending"] = False
else:
stats["fts_rebuild_pending"] = (progress or 0) < high_water
finally:
try:
conn.close()
except Exception:
pass
return stats
def count_db_holders(db_path: Path) -> Optional[int]:
"""Best-effort count of processes holding ``db_path`` open (Linux only).
Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns
the number of distinct PIDs with the file open, or ``None`` on any
error or on non-Linux platforms. Never raises; no lsof dependency.
Unreadable per-process fd dirs (other users' processes without root)
are silently skipped, so the count is a lower bound.
"""
try:
if not sys.platform.startswith("linux"):
return None
target = os.path.realpath(str(db_path))
holders = 0
for pid in os.listdir("/proc"):
if not pid.isdigit():
continue
fd_dir = f"/proc/{pid}/fd"
try:
fds = os.listdir(fd_dir)
except OSError:
continue # process gone or not ours
for fd in fds:
try:
if os.readlink(f"{fd_dir}/{fd}") == target:
holders += 1
break # one hit per PID
except OSError:
continue
return holders
except Exception:
return None
class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin):
"""
SQLite-backed session storage with FTS5 search.

View File

@ -0,0 +1,227 @@
"""Tests for state.db health/stats collection (hermes doctor section).
Covers:
- ``hermes_state.collect_state_db_stats``: read-only, best-effort stats
(page_count, freelist, WAL size, journal mode, row counts, FTS presence,
pending v23 FTS-rebuild bookkeeping).
- ``hermes_state.count_db_holders``: /proc-based best-effort probe for how
many processes hold the DB file open (Linux only; None elsewhere/on error).
- ``hermes_cli.doctor._render_state_db_stats``: formatting/threshold helper
the doctor state.db section prints from.
"""
import os
import sqlite3
import sys
from pathlib import Path
import pytest
from hermes_state import SessionDB, collect_state_db_stats, count_db_holders
@pytest.fixture()
def populated_db(tmp_path):
"""A real state.db built through SessionDB's public API, then closed."""
db_path = tmp_path / "state.db"
db = SessionDB(db_path=db_path)
db.create_session("sess-alpha", "cli")
db.create_session("sess-beta", "cli")
db.append_message("sess-alpha", "user", "hello world one")
db.append_message("sess-alpha", "assistant", "hi back")
db.append_message("sess-beta", "user", "second session message")
db.close()
return db_path
# ── collect_state_db_stats ──────────────────────────────────────────────
def test_collect_stats_sane_values(populated_db):
stats = collect_state_db_stats(populated_db)
assert isinstance(stats, dict)
assert stats["page_count"] and stats["page_count"] > 0
assert stats["page_size"] and stats["page_size"] > 0
assert stats["logical_size_bytes"] == stats["page_count"] * stats["page_size"]
assert isinstance(stats["freelist_count"], int) and stats["freelist_count"] >= 0
assert isinstance(stats["wal_size_bytes"], int) and stats["wal_size_bytes"] >= 0
assert isinstance(stats["journal_mode"], str) and stats["journal_mode"]
assert stats["messages"] >= 3
assert stats["sessions"] >= 2
fts = stats["fts_tables"]
assert set(fts) == {"messages_fts", "messages_fts_trigram", "messages_fts_cjk"}
for name, present in fts.items():
assert isinstance(present, bool), name
# Layout marker is absent (None) on legacy DBs, an int once stamped.
assert stats["fts_storage_version"] is None or isinstance(
stats["fts_storage_version"], int
)
# No rebuild pending on a fresh DB.
assert stats["fts_rebuild_pending"] in (False, None)
def test_collect_stats_is_read_only(populated_db):
before_bytes = populated_db.read_bytes()
before_mtime = populated_db.stat().st_mtime_ns
collect_state_db_stats(populated_db)
assert populated_db.read_bytes() == before_bytes
assert populated_db.stat().st_mtime_ns == before_mtime
# No stray -wal/-shm growth from the probe either: a subsequent normal
# open must still work.
db = SessionDB(db_path=populated_db)
assert db.get_session("sess-alpha") is not None
db.close()
def test_collect_stats_missing_file_never_raises(tmp_path):
stats = collect_state_db_stats(tmp_path / "nope" / "state.db")
assert isinstance(stats, dict)
assert stats["page_count"] is None
assert stats["messages"] is None
assert stats["logical_size_bytes"] is None
def test_collect_stats_rebuild_pending_flag(populated_db):
# Simulate the v23 deferred-rebuild bookkeeping directly in state_meta.
conn = sqlite3.connect(str(populated_db))
conn.execute(
"INSERT OR REPLACE INTO state_meta(key, value) VALUES ('fts_rebuild_high_water', '100')"
)
conn.execute(
"INSERT OR REPLACE INTO state_meta(key, value) VALUES ('fts_rebuild_progress', '40')"
)
conn.commit()
conn.close()
stats = collect_state_db_stats(populated_db)
assert stats["fts_rebuild_pending"] is True
assert stats["fts_rebuild_high_water"] == 100
assert stats["fts_rebuild_progress"] == 40
# ── count_db_holders ────────────────────────────────────────────────────
def test_count_db_holders_sees_open_connection(populated_db):
conn = sqlite3.connect(str(populated_db))
try:
holders = count_db_holders(populated_db)
if sys.platform.startswith("linux"):
assert isinstance(holders, int)
assert holders >= 1
else:
assert holders is None
finally:
conn.close()
def test_count_db_holders_missing_path_no_raise(tmp_path):
holders = count_db_holders(tmp_path / "absent.db")
assert holders is None or isinstance(holders, int)
# ── doctor rendering helper ─────────────────────────────────────────────
def _base_stats(**overrides):
stats = {
"page_count": 100,
"page_size": 4096,
"freelist_count": 2,
"logical_size_bytes": 100 * 4096,
"wal_size_bytes": 1024,
"journal_mode": "wal",
"messages": 42,
"sessions": 7,
"fts_tables": {
"messages_fts": True,
"messages_fts_trigram": True,
"messages_fts_cjk": False,
},
"fts_storage_version": 1,
"fts_rebuild_pending": False,
"fts_rebuild_high_water": None,
"fts_rebuild_progress": None,
}
stats.update(overrides)
return stats
def test_render_healthy_stats_no_warnings():
from hermes_cli.doctor import _render_state_db_stats
lines = _render_state_db_stats(_base_stats(), holders=2)
kinds = [k for k, *_ in lines]
assert "warn" not in kinds
joined = " | ".join(text for _, text, *_ in lines)
assert "42" in joined # messages
assert "7" in joined # sessions
assert "wal" in joined.lower()
assert "2" in joined # holders
def test_render_warns_on_large_db():
from hermes_cli.doctor import (
STATE_DB_SIZE_WARN_BYTES,
_render_state_db_stats,
)
big = STATE_DB_SIZE_WARN_BYTES + 1
lines = _render_state_db_stats(
_base_stats(logical_size_bytes=big, page_count=big // 4096, page_size=4096),
holders=None,
)
warns = [t for k, t, *rest in lines if k == "warn"] + [
" ".join(rest) for k, t, *rest in lines if k == "warn"
]
blob = " ".join(str(x) for x in warns)
assert "auto_prune" in blob
assert "config.yaml" in blob
def test_render_large_db_with_pending_rebuild_suggests_optimize():
from hermes_cli.doctor import STATE_DB_SIZE_WARN_BYTES, _render_state_db_stats
big = STATE_DB_SIZE_WARN_BYTES + 1
lines = _render_state_db_stats(
_base_stats(logical_size_bytes=big, fts_rebuild_pending=True),
holders=None,
)
blob = " ".join(" ".join(str(p) for p in line) for line in lines)
assert "optimize-storage" in blob
def test_render_large_db_legacy_trigram_suggests_optimize():
from hermes_cli.doctor import STATE_DB_SIZE_WARN_BYTES, _render_state_db_stats
big = STATE_DB_SIZE_WARN_BYTES + 1
lines = _render_state_db_stats(
_base_stats(logical_size_bytes=big, fts_storage_version=None),
holders=None,
)
blob = " ".join(" ".join(str(p) for p in line) for line in lines)
assert "optimize-storage" in blob
def test_render_warns_on_large_wal():
from hermes_cli.doctor import STATE_DB_WAL_WARN_BYTES, _render_state_db_stats
lines = _render_state_db_stats(
_base_stats(wal_size_bytes=STATE_DB_WAL_WARN_BYTES + 1), holders=None
)
blob = " ".join(" ".join(str(p) for p in line) for line in lines).lower()
assert "checkpoint" in blob
def test_render_handles_all_none_stats():
from hermes_cli.doctor import _render_state_db_stats
empty = {k: None for k in _base_stats()}
empty["fts_tables"] = None
lines = _render_state_db_stats(empty, holders=None)
assert isinstance(lines, list) # must not raise