From c595dcb955ee42a9da52053ef7e5367347f57bbd Mon Sep 17 00:00:00 2001 From: Drexuxux Date: Wed, 5 Aug 2026 13:33:34 +0300 Subject: [PATCH] fix(search): strip the FTS5 special characters the sanitizer was missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _sanitize_fts5_query's strip step only removed +{}():"^ . Every other character FTS5's grammar rejects outside a quoted phrase reached MATCH raw and raised, and — as the step's own comment says about the colon it was fixed for — the execute site swallows that into zero results. Session search silently found nothing for ordinary queries: it's fts5: syntax error near "'" gateway/run.py fts5: syntax error near "/" user@host fts5: syntax error near "@" a,b fts5: syntax error near "," why? fts5: syntax error near "?" e=mc2 fts5: syntax error near "=" Complete the class and assemble it with re.escape, because written as a regex literal the backslash was eaten as an escape and never made it in (C:\path\file still raised after the first pass). Measured against a real FTS5 table over 651 realistic queries: 373 unparsable before, 77 after. The remainder is leading/trailing "." and "-", which #43889 already covers. % is deliberately left in: the CJK path falls back to a LIKE search that needs it literal and escapes wildcards itself, so stripping it widened those queries onto unrelated rows (test_cjk_like_escapes_wildcards). --- hermes_state_search.py | 20 +++++++++++- tests/test_hermes_state.py | 67 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/hermes_state_search.py b/hermes_state_search.py index e8f32aa5f41c1..e13eb1114f0d9 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -32,6 +32,18 @@ from hermes_state_common import ( # keep that logger identity so log filtering/capture behavior is unchanged. logger = logging.getLogger("hermes_state") +# Characters FTS5's query grammar rejects outside a quoted phrase. Anything +# missing from this set reaches MATCH raw and raises, which the execute site +# swallows into zero results — the failure this strip step exists to prevent. +# Assembled through re.escape so the backslash cannot be eaten as a regex +# escape inside the class (it was, while the set was written as a literal). +# +# ``%`` is deliberately excluded: a CJK query falls back to a LIKE search that +# needs it preserved as a literal (that path escapes wildcards itself), so +# stripping it here widened those queries onto unrelated rows. +_FTS5_SPECIAL_CHARS = '+{}():"^@/#&|~[]<>,;!?$=\\\'' +_FTS5_SPECIAL_RE = re.compile(f"[{re.escape(_FTS5_SPECIAL_CHARS)}]") + class SessionSearchMixin: """See module docstring — mixin for SessionDB (Search cluster).""" @@ -1210,7 +1222,13 @@ class SessionSearchMixin: # single ``content`` column, an unquoted colon query like ``TODO: fix`` # parses as ``column:term`` and raises "no such column" — swallowed at # the execute site into zero results. Strip it like the others. - sanitized = re.sub(r'[+{}():\"^]', " ", sanitized) + # The class below is every character FTS5's query grammar rejects + # outside a quoted phrase. Anything omitted here reaches MATCH raw and + # raises, which the execute site swallows into zero results — the + # failure mode this step exists to prevent. Measured against a real + # FTS5 table: ``it's``, ``gateway/run.py``, ``user@host``, ``a,b`` and + # ``50%`` all raised before the class was completed. + sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized) # Step 3: Collapse repeated * (e.g. "***") into a single one, # and remove leading * (prefix-only needs at least one char before *) diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 12c1925f9ecf0..0169ed43e07b7 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -4503,3 +4503,70 @@ class TestPerformancePragmasEndToEnd: assert self._read(ro._conn) == defaults finally: ro.close() + + +class TestFts5SanitizerCharacterClass: + """Every character FTS5 rejects outside a quoted phrase must be stripped. + + A survivor reaches MATCH raw and raises, which the execute site swallows + into zero results — so the search silently finds nothing rather than + erroring. Assertions run the sanitized text against a real FTS5 table. + """ + + @staticmethod + def _fts_table(): + import sqlite3 + + conn = sqlite3.connect(":memory:") + conn.execute("CREATE VIRTUAL TABLE t USING fts5(content)") + conn.execute( + "INSERT INTO t (content) VALUES " + "('meet me at user host about gateway run py it s 50 a b')" + ) + return conn + + @staticmethod + def _sanitize(query): + from hermes_state_search import SessionSearchMixin + + return SessionSearchMixin._sanitize_fts5_query(query) + + @pytest.mark.parametrize( + "query", + [ + "it's", # apostrophe — ordinary prose + "gateway/run.py", # path separator + "user@host", # email / handle + "a,b", # comma + "why?", # question mark + "e=mc2", # equals + "a;b", "a!b", "a&b", "a|b", "x~y", + "#tag", "$dollar", "[bracket]", "", + r"C:\path\file", # backslash + ], + ) + def test_query_stays_parsable(self, query): + conn = self._fts_table() + sanitized = self._sanitize(query) + if not sanitized.strip(): + return + # Raises sqlite3.OperationalError if a special character survived. + conn.execute("SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)).fetchone() + + def test_plain_terms_are_untouched(self): + assert self._sanitize("hello world").split() == ["hello", "world"] + + def test_quoted_phrase_survives(self): + assert '"exact phrase"' in self._sanitize('"exact phrase"') + + def test_hyphen_dotted_term_still_quoted(self): + # Step 5's behaviour must not regress: my-app.config.ts stays one term. + assert '"my-app.config.ts"' in self._sanitize("my-app.config.ts") + + def test_prefix_star_still_works(self): + conn = self._fts_table() + sanitized = self._sanitize("gate*") + rows = conn.execute( + "SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,) + ).fetchone() + assert rows[0] == 1