From 373ec23e37600330bd4c3e03b7ba9a17ec995c30 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 21 Jul 2026 05:40:25 -0700 Subject: [PATCH] fix(state): extend search-path FTS self-heal to the CJK/trigram branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trigram MATCH branch in search_messages() had the same OperationalError-only catch that #66420 fixed on the main FTS5 branch: a corrupt messages_fts_trigram shadow table raises the malformed / 'fts5: corrupt structure record' class (sqlite3.DatabaseError, parent of OperationalError), which propagated straight out of search_messages and crashed CJK session/history search for read-only sessions. Route that class through the shared one-shot _try_runtime_fts_rebuild() and retry the trigram query (catch moved outside self._lock so rebuild_fts() can re-acquire it, mirroring the main branch). If the rebuild is refused (guard consumed / FTS disabled / different error) or the retry fails, fall through to the existing LIKE substring fallback — which reads only the canonical messages table — instead of raising, so CJK search degrades gracefully rather than crashing. Adds two regression tests: trigram search self-heals in place after shadow-table corruption (answers from the rebuilt trigram index, not the LIKE fallback), and degrades to LIKE without raising when the one-shot rebuild was already consumed. Follow-up to #66420; refs #66296 #66724 --- hermes_state.py | 44 +++++++++++++++--- tests/state/test_fts_runtime_rebuild.py | 61 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/hermes_state.py b/hermes_state.py index 07fbcbcd5d28d..97fbb404390f6 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -5701,15 +5701,47 @@ class SessionDB: LIMIT ? OFFSET ? """ tri_params.extend([limit, offset]) - with self._lock: - try: + try: + with self._lock: tri_cursor = self._conn.execute(tri_sql, tri_params) - except sqlite3.OperationalError: - # Trigram query failed at runtime — fall through to LIKE. - pass - else: matches = [dict(row) for row in tri_cursor.fetchall()] _trigram_succeeded = True + except sqlite3.OperationalError: + # Trigram query failed at runtime — fall through to LIKE. + pass + except sqlite3.DatabaseError as exc: + # Same corruption class the main FTS5 MATCH branch + # self-heals above: a corrupt trigram shadow table raises + # malformed / "fts5: corrupt structure record", which is a + # DatabaseError (parent of the OperationalError syntax arm + # caught first). Rebuild once outside the lock — the lock + # is released here so rebuild_fts() can re-acquire it — + # and retry the trigram query. If the rebuild is refused + # (already attempted / FTS disabled / different error + # class) or the retry fails again, fall through to the + # LIKE substring path, which reads only the canonical + # messages table, so CJK search stays available. + if self._try_runtime_fts_rebuild(exc): + try: + with self._lock: + tri_cursor = self._conn.execute( + tri_sql, tri_params + ) + matches = [ + dict(row) for row in tri_cursor.fetchall() + ] + _trigram_succeeded = True + except sqlite3.DatabaseError: + logger.warning( + "Trigram FTS search still failing after " + "in-place rebuild; falling back to LIKE." + ) + else: + logger.warning( + "Trigram FTS search hit a corruption error (%s) " + "and no in-place rebuild was possible; falling " + "back to LIKE.", exc, + ) if not _trigram_succeeded: # Short / mixed CJK query, trigram unavailable, or trigram # <3 CJK chars. Fall back to LIKE substring search. diff --git a/tests/state/test_fts_runtime_rebuild.py b/tests/state/test_fts_runtime_rebuild.py index cf91b474e12a2..3808ca6bb0fb1 100644 --- a/tests/state/test_fts_runtime_rebuild.py +++ b/tests/state/test_fts_runtime_rebuild.py @@ -38,6 +38,16 @@ def _corrupt_fts(db_path): raw.close() +def _corrupt_trigram_fts(db_path): + raw = sqlite3.connect(str(db_path)) + raw.execute( + "UPDATE messages_fts_trigram_data " + "SET block = X'DEADBEEFDEADBEEFDEADBEEFDEADBEEF'" + ) + raw.commit() + raw.close() + + def _message_contents(db_path): raw = sqlite3.connect(str(db_path)) rows = raw.execute("SELECT content FROM messages ORDER BY id").fetchall() @@ -116,6 +126,57 @@ class TestRuntimeFtsRebuild: assert results # non-empty: the rebuilt index matched the query assert any("needle" in (r.get("snippet") or "") for r in results) + def test_trigram_search_self_heals_after_fts_corruption(self, db, tmp_path): + """The CJK/trigram MATCH branch has the same read-corruption exposure + as the main FTS5 branch: it caught only OperationalError (query + syntax), so a corrupt trigram shadow table raised DatabaseError + straight out of search_messages. It must self-heal via the shared + one-shot rebuild and answer from the rebuilt trigram index. + """ + if not db._fts_enabled: + pytest.skip("FTS5 unavailable in this build") + if not db._trigram_available: + pytest.skip("trigram tokenizer unavailable in this build") + db.create_session("s1", source="test") + db.append_message("s1", "user", "关于大别山项目的进展报告") + + _corrupt_trigram_fts(tmp_path / "state.db") + assert db._fts_runtime_rebuild_attempted is False + + # >=3 CJK chars per token → routed to the trigram branch. + results = db.search_messages("大别山项目") + + assert db._fts_runtime_rebuild_attempted is True # search rebuilt it + assert results + # The rebuilt trigram index answered (trigram snippets use >>> <<<), + # i.e. we did not silently degrade to the LIKE fallback. + assert any(">>>" in (r.get("snippet") or "") for r in results) + + def test_trigram_search_falls_back_to_like_when_rebuild_consumed( + self, db, tmp_path + ): + """When the one-shot rebuild was already consumed, a corrupt trigram + index must NOT crash search_messages — it degrades to the LIKE + substring fallback, which reads only the canonical messages table. + """ + if not db._fts_enabled: + pytest.skip("FTS5 unavailable in this build") + if not db._trigram_available: + pytest.skip("trigram tokenizer unavailable in this build") + db.create_session("s1", source="test") + db.append_message("s1", "user", "关于大别山项目的进展报告") + + # Consume the one-shot guard, then corrupt again. + _corrupt_trigram_fts(tmp_path / "state.db") + db.append_message("s1", "user", "seed to trigger write-path heal") + assert db._fts_runtime_rebuild_attempted is True + _corrupt_trigram_fts(tmp_path / "state.db") + + # Before the fix this raised sqlite3.DatabaseError. + results = db.search_messages("大别山项目") + assert results # LIKE fallback found the canonical row + assert any("大别山项目" in (r.get("snippet") or "") for r in results) + def test_rebuild_is_one_shot_per_instance(self, db, tmp_path): if not db._fts_enabled: pytest.skip("FTS5 unavailable in this build")