feat(search): auto-enable multiline mode for newline patterns

A regex \n (or a raw newline) in a search_files content pattern cannot
match in rg's default line-oriented mode. It previously either
hard-errored ('the literal "\n" is not allowed in a regex' — 17
occurrences in the production window) or, after the newline-warning
patch, returned 0 matches with an explanation — either way the model's
cross-line search intent required a manual workaround.

The rg engine now detects the pattern shape (_pattern_has_regex_newline,
already used by the warning path: odd-backslash \n escape or raw
newline; escaped \\n literals excluded) and enables -U/--multiline up
front, noting the mode switch in the result warning. Plain patterns are
untouched; the old line-oriented explanation is retained for the grep
fallback engine, which has no multiline mode.
This commit is contained in:
Teknium 2026-08-02 14:34:02 -07:00
parent 2a3a7e6f53
commit 1cefabc8af
2 changed files with 70 additions and 1 deletions

View File

@ -0,0 +1,51 @@
"""Tests for search_files auto-multiline routing on \\n patterns."""
import json
import pytest
from tools.file_tools import search_tool
@pytest.fixture
def proj(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
d = tmp_path / "proj"
d.mkdir()
(d / "mod.py").write_text(
"def setup():\n init_db()\n return True\n\n"
"def teardown():\n close_db()\n"
)
return d
class TestAutoMultiline:
def test_newline_regex_matches_across_lines(self, proj):
r = json.loads(search_tool(r"def setup\(\):\n init_db\(\)", path=str(proj), task_id="t-ml"))
assert "error" not in r
assert r["total_count"] >= 1
assert "multiline" in r.get("warning", "")
def test_literal_newline_in_pattern_matches(self, proj):
# A raw newline in the pattern (not the \n escape) also routes to
# multiline mode. Keep the pattern free of regex metachars.
r = json.loads(search_tool("return True\n\ndef teardown", path=str(proj), task_id="t-ml"))
assert "error" not in r
assert r["total_count"] >= 1
def test_plain_pattern_unaffected(self, proj):
r = json.loads(search_tool("init_db", path=str(proj), task_id="t-ml"))
assert r["total_count"] == 1
assert "multiline" not in r.get("warning", "")
def test_escaped_backslash_n_stays_literal(self, proj):
# \\n = literal backslash+n search, not a newline: no multiline mode.
(proj / "strings.py").write_text('SEP = "a\\\\nb"\n')
r = json.loads(search_tool(r"a\\nb", path=str(proj), task_id="t-ml"))
assert "error" not in r
assert "multiline" not in (r.get("warning") or "")
def test_multiline_zero_match_is_clean(self, proj):
r = json.loads(search_tool(r"def missing\(\):\n nope\(\)", path=str(proj), task_id="t-ml"))
assert "error" not in r
assert r["total_count"] == 0

View File

@ -2478,6 +2478,9 @@ class ShellFileOperations(FileOperations):
if self._has_command('rg'):
result = self._search_with_rg(pattern, path, file_glob, limit, offset,
output_mode, context)
# rg auto-enables --multiline for \n patterns, so the line-
# oriented warning below no longer applies to this engine.
return result
elif self._has_command('grep'):
result = self._search_with_grep(pattern, path, file_glob, limit, offset,
output_mode, context)
@ -2506,7 +2509,16 @@ class ShellFileOperations(FileOperations):
limit: int, offset: int, output_mode: str, context: int) -> SearchResult:
"""Search using ripgrep."""
cmd_parts = ["rg", "--line-number", "--no-heading", "--with-filename"]
# Auto-multiline: a regex `\n` (or a literal newline in the pattern)
# cannot match in rg's default line-oriented mode — it used to hard
# error ("the literal \"\\n\" is not allowed") and burn a turn. When
# the pattern clearly wants to cross lines, enable -U/--multiline
# up front and note it in the result.
multiline = _pattern_has_regex_newline(pattern)
if multiline:
cmd_parts.append("--multiline")
# Add context if requested
if context > 0:
cmd_parts.extend(["-C", str(context)])
@ -2556,6 +2568,10 @@ class ShellFileOperations(FileOperations):
# Parse the diagnostic-free payload so error text never becomes a match.
stdout = payload
_ml_note = (
"Pattern contains \\n — multiline mode (-U) was enabled automatically "
"so the regex can match across line boundaries."
) if multiline else None
# Parse results based on output mode
if output_mode == "files_only":
all_files = [f for f in stdout.strip().split('\n') if f]
@ -2566,6 +2582,7 @@ class ShellFileOperations(FileOperations):
total_count=total,
truncated=bool(limit_reason),
limit_reason=limit_reason,
warning=_ml_note,
)
elif output_mode == "count":
@ -2626,6 +2643,7 @@ class ShellFileOperations(FileOperations):
total_count=total,
truncated=total > offset + limit or bool(limit_reason),
limit_reason=limit_reason,
warning=_ml_note,
)
def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str],