fix(redact): don't join across controls when a fragment already matches
CI slice 1/12 caught a regression in _mask_control_split_tokens: a
COMPLETE prefix token at end-of-line followed by ordinary text (browser
accessibility annotations: 'ghp_<tok>\nbutton [ref=e3]: Copy') was
joined across the newline into one stripped-copy match, and the mask
swallowed the adjacent line ('button' disappeared).
Join only when no fragment inside the span matches _PREFIX_RE on its
own — a self-matching fragment is already handled by the ordinary
prefix pass, so joining can only cause damage. All smuggling shapes
(ESC/ZWSP/newline splits with under-length fragments) still mask;
regression test added and mutation-checked (fails without the guard).
This commit is contained in:
parent
8969ebac1c
commit
aecb9ca894
|
|
@ -507,6 +507,15 @@ def _mask_control_split_tokens(text: str, mask_fn) -> str:
|
|||
body = m.group(1)
|
||||
start_orig = orig_idx[m.start(1)]
|
||||
end_orig = orig_idx[m.end(1) - 1] + 1
|
||||
# If any fragment inside the original span already matches _PREFIX_RE
|
||||
# on its own, the ordinary prefix pass will mask it — do NOT join.
|
||||
# Joining here would swallow adjacent legitimate text: a complete
|
||||
# token at end-of-line followed by a word line (``ghp_<token>\n
|
||||
# button [ref=e3]``) joins into one stripped-copy match and the
|
||||
# mask eats ``button``. Join only when fragments alone are too
|
||||
# short/broken to match (the actual smuggling shape).
|
||||
if _PREFIX_RE.search(text[start_orig:end_orig]):
|
||||
continue
|
||||
# Reject matches whose original span crosses a non-token char
|
||||
# (e.g. ``sk_abc…\nTAVILY_API_KEY=…`` — the ``=`` is not part of a
|
||||
# token body, so the regex matched across unrelated lines). Also
|
||||
|
|
|
|||
|
|
@ -170,6 +170,19 @@ class TestControlCharSplitTokens:
|
|||
tok = "ghp_abcdef1234567890ABCDEF1234567890abcdef"
|
||||
self._assert_split_masked(f"{tok[:10]}\u200b{tok[10:]}", tok)
|
||||
|
||||
def test_complete_token_does_not_swallow_next_line(self):
|
||||
# A COMPLETE token at end-of-line followed by ordinary text must not
|
||||
# be joined across the newline — the ordinary prefix pass masks the
|
||||
# token; joining would swallow the adjacent line (browser
|
||||
# accessibility annotations regressed this way: "button [ref=e3]"
|
||||
# disappeared into the mask).
|
||||
tok = "ghp_" + "F" * 29
|
||||
text = f"text: Token: {tok}\nbutton [ref=e3]: Copy\n"
|
||||
result = redact_sensitive_text(text, force=True)
|
||||
assert "F" * 20 not in result
|
||||
assert "button" in result
|
||||
assert "ref=e3" in result
|
||||
|
||||
def test_env_dump_lines_not_joined(self):
|
||||
# Control-stripping must not join unrelated env lines into one match
|
||||
env_dump = (
|
||||
|
|
|
|||
Loading…
Reference in New Issue