fix(search): strip the FTS5 special characters the sanitizer was missing

_sanitize_fts5_query's strip step only removed +{}():"^ . Every other
character FTS5's grammar rejects outside a quoted phrase reached MATCH
raw and raised, and — as the step's own comment says about the colon it
was fixed for — the execute site swallows that into zero results. Session
search silently found nothing for ordinary queries:

  it's            fts5: syntax error near "'"
  gateway/run.py  fts5: syntax error near "/"
  user@host       fts5: syntax error near "@"
  a,b             fts5: syntax error near ","
  why?            fts5: syntax error near "?"
  e=mc2           fts5: syntax error near "="

Complete the class and assemble it with re.escape, because written as a
regex literal the backslash was eaten as an escape and never made it in
(C:\path\file still raised after the first pass).

Measured against a real FTS5 table over 651 realistic queries:
373 unparsable before, 77 after. The remainder is leading/trailing "." and
"-", which #43889 already covers.

% is deliberately left in: the CJK path falls back to a LIKE search that
needs it literal and escapes wildcards itself, so stripping it widened
those queries onto unrelated rows (test_cjk_like_escapes_wildcards).
This commit is contained in:
Drexuxux 2026-08-05 13:33:34 +03:00 committed by Teknium
parent 0569c001d0
commit c595dcb955
2 changed files with 86 additions and 1 deletions

View File

@ -32,6 +32,18 @@ from hermes_state_common import (
# keep that logger identity so log filtering/capture behavior is unchanged.
logger = logging.getLogger("hermes_state")
# Characters FTS5's query grammar rejects outside a quoted phrase. Anything
# missing from this set reaches MATCH raw and raises, which the execute site
# swallows into zero results — the failure this strip step exists to prevent.
# Assembled through re.escape so the backslash cannot be eaten as a regex
# escape inside the class (it was, while the set was written as a literal).
#
# ``%`` is deliberately excluded: a CJK query falls back to a LIKE search that
# needs it preserved as a literal (that path escapes wildcards itself), so
# stripping it here widened those queries onto unrelated rows.
_FTS5_SPECIAL_CHARS = '+{}():"^@/#&|~[]<>,;!?$=\\\''
_FTS5_SPECIAL_RE = re.compile(f"[{re.escape(_FTS5_SPECIAL_CHARS)}]")
class SessionSearchMixin:
"""See module docstring — mixin for SessionDB (Search cluster)."""
@ -1210,7 +1222,13 @@ class SessionSearchMixin:
# single ``content`` column, an unquoted colon query like ``TODO: fix``
# parses as ``column:term`` and raises "no such column" — swallowed at
# the execute site into zero results. Strip it like the others.
sanitized = re.sub(r'[+{}():\"^]', " ", sanitized)
# The class below is every character FTS5's query grammar rejects
# outside a quoted phrase. Anything omitted here reaches MATCH raw and
# raises, which the execute site swallows into zero results — the
# failure mode this step exists to prevent. Measured against a real
# FTS5 table: ``it's``, ``gateway/run.py``, ``user@host``, ``a,b`` and
# ``50%`` all raised before the class was completed.
sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized)
# Step 3: Collapse repeated * (e.g. "***") into a single one,
# and remove leading * (prefix-only needs at least one char before *)

View File

@ -4503,3 +4503,70 @@ class TestPerformancePragmasEndToEnd:
assert self._read(ro._conn) == defaults
finally:
ro.close()
class TestFts5SanitizerCharacterClass:
"""Every character FTS5 rejects outside a quoted phrase must be stripped.
A survivor reaches MATCH raw and raises, which the execute site swallows
into zero results so the search silently finds nothing rather than
erroring. Assertions run the sanitized text against a real FTS5 table.
"""
@staticmethod
def _fts_table():
import sqlite3
conn = sqlite3.connect(":memory:")
conn.execute("CREATE VIRTUAL TABLE t USING fts5(content)")
conn.execute(
"INSERT INTO t (content) VALUES "
"('meet me at user host about gateway run py it s 50 a b')"
)
return conn
@staticmethod
def _sanitize(query):
from hermes_state_search import SessionSearchMixin
return SessionSearchMixin._sanitize_fts5_query(query)
@pytest.mark.parametrize(
"query",
[
"it's", # apostrophe — ordinary prose
"gateway/run.py", # path separator
"user@host", # email / handle
"a,b", # comma
"why?", # question mark
"e=mc2", # equals
"a;b", "a!b", "a&b", "a|b", "x~y",
"#tag", "$dollar", "[bracket]", "<tag>",
r"C:\path\file", # backslash
],
)
def test_query_stays_parsable(self, query):
conn = self._fts_table()
sanitized = self._sanitize(query)
if not sanitized.strip():
return
# Raises sqlite3.OperationalError if a special character survived.
conn.execute("SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)).fetchone()
def test_plain_terms_are_untouched(self):
assert self._sanitize("hello world").split() == ["hello", "world"]
def test_quoted_phrase_survives(self):
assert '"exact phrase"' in self._sanitize('"exact phrase"')
def test_hyphen_dotted_term_still_quoted(self):
# Step 5's behaviour must not regress: my-app.config.ts stays one term.
assert '"my-app.config.ts"' in self._sanitize("my-app.config.ts")
def test_prefix_star_still_works(self):
conn = self._fts_table()
sanitized = self._sanitize("gate*")
rows = conn.execute(
"SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)
).fetchone()
assert rows[0] == 1