fix(state): retry transient 'no more rows available' across all sqlite3.Error classes

Under dual gateway/agent WAL contention (FTS5 trigram sync holding the
write lock on large appends) the SQLite engine can raise a transient
'no more rows available' error. The exception CLASS varies with the
build — some surface it as InterfaceError, a SIBLING of DatabaseError —
so it escaped both existing retry branches in _execute_write on attempt
0 and killed the turn as session_persistence_failed even though the
identical write succeeds standalone.

Port of #74934 onto the deadline-patience rewrite (8da8a7887d): the
PR's attempt-counted constants (60 retries / 300ms jitter / 2.0s engine
timeout) predate that rewrite and are superseded by the patience
budget, so they are intentionally NOT carried over. Instead the check
is message-scoped and rides the existing deadline/patience loop:

- extract the jittered-sleep-within-deadline logic into a shared
  _sleep_before_retry helper (behavior-preserving for locked/busy)
- retry 'no more rows available' from OperationalError, DatabaseError
  (checked BEFORE the FTS-corruption rebuild path so it is not
  misrouted), and a message-scoped sqlite3.Error catch-all
- any other error in any class propagates untouched on attempt 0

Tests: transient InterfaceError retried to success; unrelated
InterfaceError propagates immediately; DatabaseError variant retried;
exhausted patience surfaces the original error.
This commit is contained in:
Dannoob 2026-07-31 21:56:30 -07:00 committed by Teknium
parent d5463e5f6d
commit 14eca89779
2 changed files with 117 additions and 1 deletions

View File

@ -2405,6 +2405,17 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
# Set on the first compression-busy collision so the short wait is
# measured from then, not from the start of the write.
compression_deadline: Optional[float] = None
# Transient engine-level error observed on contended WAL appends
# (dual gateway/agent writers; FTS5 trigram sync holds the write
# lock). The identical write succeeds standalone, so it is
# retryable like locked/busy. The exception CLASS varies with the
# SQLite build — some surface it as InterfaceError, which lives
# OUTSIDE DatabaseError and escaped the retry net entirely on
# attempt 0 — so the check is message-scoped, not class-scoped.
def _is_no_more_rows(exc: sqlite3.Error) -> bool:
return "no more rows available" in str(exc).lower()
while True:
try:
with self._lock:
@ -2460,9 +2471,13 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
"a large WAL checkpoint, or an older pre-update "
"process; the database itself is healthy)"
) from exc
# Non-lock error — propagate.
if _is_no_more_rows(exc) and _sleep_before_retry():
continue
# Non-lock error or patience exhausted — propagate.
raise
except sqlite3.DatabaseError as exc:
if _is_no_more_rows(exc) and _sleep_before_retry():
continue
# Corrupt FTS shadow tables make every write raise the
# malformed/corrupt error class through the FTS sync triggers
# while the canonical messages table is intact. The gateway
@ -2475,6 +2490,15 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
if not self._try_runtime_fts_rebuild(exc):
raise
continue
except sqlite3.Error as exc:
# Catch-all for builds that surface 'no more rows available'
# as InterfaceError (a sibling of DatabaseError, not a
# subclass) or another sqlite3.Error class outside the two
# handlers above. Message-scoped: anything else propagates
# untouched.
if _is_no_more_rows(exc) and _sleep_before_retry():
continue
raise
def _sleep_before_write_retry(
self, deadline: float, patience_s: float

View File

@ -0,0 +1,92 @@
"""Retry of transient 'no more rows available' engine errors (#74934 port).
Under dual gateway/agent WAL contention (FTS5 trigram sync holding the
write lock on large appends), the SQLite engine can raise a transient
'no more rows available' error. The exception CLASS varies with the
SQLite build some surface it as ``sqlite3.InterfaceError``, which is a
sibling of ``DatabaseError`` (not a subclass) and therefore escaped both
existing retry branches in ``_execute_write`` on attempt 0, killing the
turn as ``session_persistence_failed`` while the identical write would
have succeeded milliseconds later.
The fix is message-scoped, not class-scoped: any ``sqlite3.Error`` whose
text contains 'no more rows available' retries within the existing
deadline/patience loop; every other error propagates untouched.
"""
import sqlite3
import pytest
from hermes_state import SessionDB
@pytest.fixture
def db(tmp_path, monkeypatch):
# Keep retries fast: tiny jitter, short-but-sufficient patience.
monkeypatch.setattr(SessionDB, "_WRITE_PATIENCE_S", 2.0)
monkeypatch.setattr(SessionDB, "_WRITE_RETRY_MIN_S", 0.001)
monkeypatch.setattr(SessionDB, "_WRITE_RETRY_MAX_S", 0.005)
d = SessionDB(db_path=tmp_path / "state.db")
yield d
d.close()
class TestNoMoreRowsRetry:
def test_transient_interface_error_is_retried_to_success(self, db):
"""InterfaceError('no more rows available') must be retried inside
the deadline/patience loop and succeed once the contention clears."""
calls = {"n": 0}
def flaky(conn):
calls["n"] += 1
if calls["n"] <= 3:
raise sqlite3.InterfaceError("no more rows available")
conn.execute(
"INSERT INTO state_meta (key, value) VALUES ('nmr', 'ok') "
"ON CONFLICT(key) DO UPDATE SET value=excluded.value"
)
return "done"
assert db._execute_write(flaky) == "done"
assert calls["n"] == 4
assert db.get_meta("nmr") == "ok"
def test_unrelated_interface_error_propagates_immediately(self, db):
"""The catch-all is message-scoped: an InterfaceError with any other
text must escape on the first attempt, not be swallowed/retried."""
calls = {"n": 0}
def broken(conn):
calls["n"] += 1
raise sqlite3.InterfaceError("bad parameter or other API misuse")
with pytest.raises(sqlite3.InterfaceError, match="bad parameter"):
db._execute_write(broken)
assert calls["n"] == 1
def test_no_more_rows_via_database_error_is_retried(self, db):
"""Some builds raise the same transient message through the generic
DatabaseError class it must ride the same retry loop instead of
being misrouted into the FTS-corruption rebuild path."""
calls = {"n": 0}
def flaky(conn):
calls["n"] += 1
if calls["n"] <= 2:
raise sqlite3.DatabaseError("no more rows available")
return "ok"
assert db._execute_write(flaky) == "ok"
assert calls["n"] == 3
def test_exhausted_patience_propagates_the_transient_error(self, db, monkeypatch):
"""If contention never clears within the patience budget, the
original error must surface rather than looping forever."""
monkeypatch.setattr(SessionDB, "_WRITE_PATIENCE_S", 0.05)
def always(conn):
raise sqlite3.InterfaceError("no more rows available")
with pytest.raises(sqlite3.InterfaceError, match="no more rows"):
db._execute_write(always)