diff --git a/hermes_state_search.py b/hermes_state_search.py index e8f32aa5f41c1..e13eb1114f0d9 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -32,6 +32,18 @@ from hermes_state_common import ( # keep that logger identity so log filtering/capture behavior is unchanged. logger = logging.getLogger("hermes_state") +# Characters FTS5's query grammar rejects outside a quoted phrase. Anything +# missing from this set reaches MATCH raw and raises, which the execute site +# swallows into zero results — the failure this strip step exists to prevent. +# Assembled through re.escape so the backslash cannot be eaten as a regex +# escape inside the class (it was, while the set was written as a literal). +# +# ``%`` is deliberately excluded: a CJK query falls back to a LIKE search that +# needs it preserved as a literal (that path escapes wildcards itself), so +# stripping it here widened those queries onto unrelated rows. +_FTS5_SPECIAL_CHARS = '+{}():"^@/#&|~[]<>,;!?$=\\\'' +_FTS5_SPECIAL_RE = re.compile(f"[{re.escape(_FTS5_SPECIAL_CHARS)}]") + class SessionSearchMixin: """See module docstring — mixin for SessionDB (Search cluster).""" @@ -1210,7 +1222,13 @@ class SessionSearchMixin: # single ``content`` column, an unquoted colon query like ``TODO: fix`` # parses as ``column:term`` and raises "no such column" — swallowed at # the execute site into zero results. Strip it like the others. - sanitized = re.sub(r'[+{}():\"^]', " ", sanitized) + # The class below is every character FTS5's query grammar rejects + # outside a quoted phrase. Anything omitted here reaches MATCH raw and + # raises, which the execute site swallows into zero results — the + # failure mode this step exists to prevent. Measured against a real + # FTS5 table: ``it's``, ``gateway/run.py``, ``user@host``, ``a,b`` and + # ``50%`` all raised before the class was completed. + sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized) # Step 3: Collapse repeated * (e.g. "***") into a single one, # and remove leading * (prefix-only needs at least one char before *) diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 12c1925f9ecf0..0169ed43e07b7 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -4503,3 +4503,70 @@ class TestPerformancePragmasEndToEnd: assert self._read(ro._conn) == defaults finally: ro.close() + + +class TestFts5SanitizerCharacterClass: + """Every character FTS5 rejects outside a quoted phrase must be stripped. + + A survivor reaches MATCH raw and raises, which the execute site swallows + into zero results — so the search silently finds nothing rather than + erroring. Assertions run the sanitized text against a real FTS5 table. + """ + + @staticmethod + def _fts_table(): + import sqlite3 + + conn = sqlite3.connect(":memory:") + conn.execute("CREATE VIRTUAL TABLE t USING fts5(content)") + conn.execute( + "INSERT INTO t (content) VALUES " + "('meet me at user host about gateway run py it s 50 a b')" + ) + return conn + + @staticmethod + def _sanitize(query): + from hermes_state_search import SessionSearchMixin + + return SessionSearchMixin._sanitize_fts5_query(query) + + @pytest.mark.parametrize( + "query", + [ + "it's", # apostrophe — ordinary prose + "gateway/run.py", # path separator + "user@host", # email / handle + "a,b", # comma + "why?", # question mark + "e=mc2", # equals + "a;b", "a!b", "a&b", "a|b", "x~y", + "#tag", "$dollar", "[bracket]", "", + r"C:\path\file", # backslash + ], + ) + def test_query_stays_parsable(self, query): + conn = self._fts_table() + sanitized = self._sanitize(query) + if not sanitized.strip(): + return + # Raises sqlite3.OperationalError if a special character survived. + conn.execute("SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)).fetchone() + + def test_plain_terms_are_untouched(self): + assert self._sanitize("hello world").split() == ["hello", "world"] + + def test_quoted_phrase_survives(self): + assert '"exact phrase"' in self._sanitize('"exact phrase"') + + def test_hyphen_dotted_term_still_quoted(self): + # Step 5's behaviour must not regress: my-app.config.ts stays one term. + assert '"my-app.config.ts"' in self._sanitize("my-app.config.ts") + + def test_prefix_star_still_works(self): + conn = self._fts_table() + sanitized = self._sanitize("gate*") + rows = conn.execute( + "SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,) + ).fetchone() + assert rows[0] == 1