fix(search): strip the FTS5 special characters the sanitizer was missing
_sanitize_fts5_query's strip step only removed +{}():"^ . Every other
character FTS5's grammar rejects outside a quoted phrase reached MATCH
raw and raised, and — as the step's own comment says about the colon it
was fixed for — the execute site swallows that into zero results. Session
search silently found nothing for ordinary queries:
it's fts5: syntax error near "'"
gateway/run.py fts5: syntax error near "/"
user@host fts5: syntax error near "@"
a,b fts5: syntax error near ","
why? fts5: syntax error near "?"
e=mc2 fts5: syntax error near "="
Complete the class and assemble it with re.escape, because written as a
regex literal the backslash was eaten as an escape and never made it in
(C:\path\file still raised after the first pass).
Measured against a real FTS5 table over 651 realistic queries:
373 unparsable before, 77 after. The remainder is leading/trailing "." and
"-", which #43889 already covers.
% is deliberately left in: the CJK path falls back to a LIKE search that
needs it literal and escapes wildcards itself, so stripping it widened
those queries onto unrelated rows (test_cjk_like_escapes_wildcards).
This commit is contained in:
parent
0569c001d0
commit
c595dcb955
|
|
@ -32,6 +32,18 @@ from hermes_state_common import (
|
|||
# keep that logger identity so log filtering/capture behavior is unchanged.
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
# Characters FTS5's query grammar rejects outside a quoted phrase. Anything
|
||||
# missing from this set reaches MATCH raw and raises, which the execute site
|
||||
# swallows into zero results — the failure this strip step exists to prevent.
|
||||
# Assembled through re.escape so the backslash cannot be eaten as a regex
|
||||
# escape inside the class (it was, while the set was written as a literal).
|
||||
#
|
||||
# ``%`` is deliberately excluded: a CJK query falls back to a LIKE search that
|
||||
# needs it preserved as a literal (that path escapes wildcards itself), so
|
||||
# stripping it here widened those queries onto unrelated rows.
|
||||
_FTS5_SPECIAL_CHARS = '+{}():"^@/#&|~[]<>,;!?$=\\\''
|
||||
_FTS5_SPECIAL_RE = re.compile(f"[{re.escape(_FTS5_SPECIAL_CHARS)}]")
|
||||
|
||||
|
||||
class SessionSearchMixin:
|
||||
"""See module docstring — mixin for SessionDB (Search cluster)."""
|
||||
|
|
@ -1210,7 +1222,13 @@ class SessionSearchMixin:
|
|||
# single ``content`` column, an unquoted colon query like ``TODO: fix``
|
||||
# parses as ``column:term`` and raises "no such column" — swallowed at
|
||||
# the execute site into zero results. Strip it like the others.
|
||||
sanitized = re.sub(r'[+{}():\"^]', " ", sanitized)
|
||||
# The class below is every character FTS5's query grammar rejects
|
||||
# outside a quoted phrase. Anything omitted here reaches MATCH raw and
|
||||
# raises, which the execute site swallows into zero results — the
|
||||
# failure mode this step exists to prevent. Measured against a real
|
||||
# FTS5 table: ``it's``, ``gateway/run.py``, ``user@host``, ``a,b`` and
|
||||
# ``50%`` all raised before the class was completed.
|
||||
sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized)
|
||||
|
||||
# Step 3: Collapse repeated * (e.g. "***") into a single one,
|
||||
# and remove leading * (prefix-only needs at least one char before *)
|
||||
|
|
|
|||
|
|
@ -4503,3 +4503,70 @@ class TestPerformancePragmasEndToEnd:
|
|||
assert self._read(ro._conn) == defaults
|
||||
finally:
|
||||
ro.close()
|
||||
|
||||
|
||||
class TestFts5SanitizerCharacterClass:
|
||||
"""Every character FTS5 rejects outside a quoted phrase must be stripped.
|
||||
|
||||
A survivor reaches MATCH raw and raises, which the execute site swallows
|
||||
into zero results — so the search silently finds nothing rather than
|
||||
erroring. Assertions run the sanitized text against a real FTS5 table.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fts_table():
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.execute("CREATE VIRTUAL TABLE t USING fts5(content)")
|
||||
conn.execute(
|
||||
"INSERT INTO t (content) VALUES "
|
||||
"('meet me at user host about gateway run py it s 50 a b')"
|
||||
)
|
||||
return conn
|
||||
|
||||
@staticmethod
|
||||
def _sanitize(query):
|
||||
from hermes_state_search import SessionSearchMixin
|
||||
|
||||
return SessionSearchMixin._sanitize_fts5_query(query)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
"it's", # apostrophe — ordinary prose
|
||||
"gateway/run.py", # path separator
|
||||
"user@host", # email / handle
|
||||
"a,b", # comma
|
||||
"why?", # question mark
|
||||
"e=mc2", # equals
|
||||
"a;b", "a!b", "a&b", "a|b", "x~y",
|
||||
"#tag", "$dollar", "[bracket]", "<tag>",
|
||||
r"C:\path\file", # backslash
|
||||
],
|
||||
)
|
||||
def test_query_stays_parsable(self, query):
|
||||
conn = self._fts_table()
|
||||
sanitized = self._sanitize(query)
|
||||
if not sanitized.strip():
|
||||
return
|
||||
# Raises sqlite3.OperationalError if a special character survived.
|
||||
conn.execute("SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)).fetchone()
|
||||
|
||||
def test_plain_terms_are_untouched(self):
|
||||
assert self._sanitize("hello world").split() == ["hello", "world"]
|
||||
|
||||
def test_quoted_phrase_survives(self):
|
||||
assert '"exact phrase"' in self._sanitize('"exact phrase"')
|
||||
|
||||
def test_hyphen_dotted_term_still_quoted(self):
|
||||
# Step 5's behaviour must not regress: my-app.config.ts stays one term.
|
||||
assert '"my-app.config.ts"' in self._sanitize("my-app.config.ts")
|
||||
|
||||
def test_prefix_star_still_works(self):
|
||||
conn = self._fts_table()
|
||||
sanitized = self._sanitize("gate*")
|
||||
rows = conn.execute(
|
||||
"SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)
|
||||
).fetchone()
|
||||
assert rows[0] == 1
|
||||
|
|
|
|||
Loading…
Reference in New Issue