fix(redact): don't join across controls when a fragment already matches

CI slice 1/12 caught a regression in _mask_control_split_tokens: a
COMPLETE prefix token at end-of-line followed by ordinary text (browser
accessibility annotations: 'ghp_<tok>\nbutton [ref=e3]: Copy') was
joined across the newline into one stripped-copy match, and the mask
swallowed the adjacent line ('button' disappeared).

Join only when no fragment inside the span matches _PREFIX_RE on its
own — a self-matching fragment is already handled by the ordinary
prefix pass, so joining can only cause damage. All smuggling shapes
(ESC/ZWSP/newline splits with under-length fragments) still mask;
regression test added and mutation-checked (fails without the guard).
This commit is contained in:
kshitij 2026-08-07 16:00:46 +05:30
parent 8969ebac1c
commit aecb9ca894
2 changed files with 22 additions and 0 deletions

View File

@ -507,6 +507,15 @@ def _mask_control_split_tokens(text: str, mask_fn) -> str:
body = m.group(1)
start_orig = orig_idx[m.start(1)]
end_orig = orig_idx[m.end(1) - 1] + 1
# If any fragment inside the original span already matches _PREFIX_RE
# on its own, the ordinary prefix pass will mask it — do NOT join.
# Joining here would swallow adjacent legitimate text: a complete
# token at end-of-line followed by a word line (``ghp_<token>\n
# button [ref=e3]``) joins into one stripped-copy match and the
# mask eats ``button``. Join only when fragments alone are too
# short/broken to match (the actual smuggling shape).
if _PREFIX_RE.search(text[start_orig:end_orig]):
continue
# Reject matches whose original span crosses a non-token char
# (e.g. ``sk_abc…\nTAVILY_API_KEY=…`` — the ``=`` is not part of a
# token body, so the regex matched across unrelated lines). Also

View File

@ -170,6 +170,19 @@ class TestControlCharSplitTokens:
tok = "ghp_abcdef1234567890ABCDEF1234567890abcdef"
self._assert_split_masked(f"{tok[:10]}\u200b{tok[10:]}", tok)
def test_complete_token_does_not_swallow_next_line(self):
# A COMPLETE token at end-of-line followed by ordinary text must not
# be joined across the newline — the ordinary prefix pass masks the
# token; joining would swallow the adjacent line (browser
# accessibility annotations regressed this way: "button [ref=e3]"
# disappeared into the mask).
tok = "ghp_" + "F" * 29
text = f"text: Token: {tok}\nbutton [ref=e3]: Copy\n"
result = redact_sensitive_text(text, force=True)
assert "F" * 20 not in result
assert "button" in result
assert "ref=e3" in result
def test_env_dump_lines_not_joined(self):
# Control-stripping must not join unrelated env lines into one match
env_dump = (