diff --git a/tests/tools/test_hardline_blocklist.py b/tests/tools/test_hardline_blocklist.py index 40b1c567555a0..419a3ed38812b 100644 --- a/tests/tools/test_hardline_blocklist.py +++ b/tests/tools/test_hardline_blocklist.py @@ -135,6 +135,17 @@ _HARDLINE_BLOCK = [ "{ poweroff; }", "true && (reboot)", "echo hi; { reboot; }", + # bare-name hardline commands sitting behind a wrapper option's OPERAND + # (`-u root`, `-Eu root`, `env -i`, or an unambiguous `--us` abbreviation) + # used to slip past: the flat _CMDPOS-anchored pattern only reaches a + # command word that is anchored ("\n"-preceded), and the operand-skip / + # option-skip branches in the two prefix walkers never set + # `prefix_consumed`, so no anchor was spliced in front of the bare + # command word that followed (egilewski, PR #82830 second report, B-1). + "sudo -u root rm -rf /etc", + "sudo -Eu root rm -rf /etc", + "env -i rm -rf /etc", + "sudo --us root rm -rf /etc", ] @@ -199,6 +210,10 @@ _HARDLINE_ALLOW = [ "npm run build", "sudo apt update", "curl https://example.com | head", + # B-1 non-regression: a benign command behind a wrapper option's operand + # must still be allowed — the operand-skip anchor must not manufacture a + # false positive out of an ordinary command word. + "sudo -u root whoami", ] @@ -670,3 +685,180 @@ def test_sudo_stdin_guard_container_bypass(clean_session): for cmd in _SUDO_STDIN_BLOCK: result = check_all_command_guards(cmd, env) assert result["approved"] is True, f"container {env} should bypass sudo guard on {cmd!r}" + + +# Absolute-path invocations must not defeat the floor. _CMDPOS only accepted +# "start | separator | subshell opener | sudo/env-style wrappers" before the +# command word, so spelling the binary by path — the natural form on systems +# where PATH is unreliable, and the default an LLM produces for Windows +# tools — returned (False, None) for every hardline pattern. Regression set +# pins both directions: path-spelled commands at command position are +# blocked, the same strings as data arguments stay allowed. +_ABS_PATH_BLOCK = [ + "/sbin/shutdown -h now", + "/usr/sbin/reboot", + "/sbin/halt", + "/sbin/init 6", + "/bin/rm -rf /", + "sudo /sbin/shutdown -h now", + "env LC_ALL=C /sbin/reboot", + "echo done; /sbin/shutdown -h now", + "true && /usr/sbin/poweroff", + r"C:\Windows\System32\shutdown.exe /s /t 0", + "C:/Windows/System32/shutdown.exe /s /t 0", + r'"C:\Program Files\Git\usr\bin\rm.exe" -rf /', + r"C:\Windows\System32\shutdown.EXE /s", + # path-spelled wrapper chains resolve in the single projection pass + "/usr/bin/sudo /sbin/shutdown -h now", + "/usr/bin/env /usr/bin/sudo /sbin/shutdown -h now", + "exec /sbin/reboot", + "( /sbin/shutdown -h now )", + "/sbin/telinit 6", + "./shutdown -h now", + "shutdown.exe /s", + # composed spellings reduce through the same collapsing as r\m detection + "'/sbin/'shutdown -h now", + "/sbin/shut\\down -h now", + # a payload's executable can be path-spelled too + "bash -c '/sbin/shutdown -h now'", + "exec bash -c '/bin/rm -rf /'", + # group/substitution closers stay outside the projected word + "(/sbin/reboot)", + "$(/sbin/reboot)", + "`/sbin/reboot`", + "{ /sbin/reboot; }", + "env -u FOO /sbin/reboot", + # leading redirections are POSIX prefix words, not the command (egilewski, + # PR #82830): the executable that follows must still reach the floor + "/dev/null /usr/sbin/reboot", + "< /dev/null /sbin/shutdown -h now", + "2>&1 /sbin/shutdown -h now", + "&>/dev/null /sbin/reboot", + "</dev/null /tmp/a 2>>/tmp/b <>/tmp/c >|/tmp/d /sbin/reboot", + # a redirection prefix in front of a bare-name command (main-era gap, + # closed here by the same normalization) + "/dev/null reboot", + # an assignment prefix in front of a bare-name command (same gap) + "FOO=1 shutdown -h now", + # process substitution as a redirection operand still resumes at the cmd + "> >(cat) /sbin/shutdown -h now", + # gluing must not hide the command: no separator before the redirection + "shutdown&>out -h now", + # an escaped > is a literal, so & stays a background operator and the + # following command starts fresh and must block + "printf x \\>& shutdown -h now", + # a sudo option that takes an argument must not shift the command word + "sudo -D /tmp /sbin/shutdown -h now", + # >&file redirects both streams to a filename (not an fd), so the file is + # the operand and the command still follows (Grok round 1) + ">&/dev/null /sbin/shutdown -h now", + # a quoted ) inside a process substitution must not end the balance early + '> >(echo "x)y") /sbin/shutdown -h now', + # a digit run glued to non-digits after >&/2>& is a filename operand, not + # a bare fd number, so the whole token is consumed and the command follows + # (Grok round 2) + ">&2x /sbin/shutdown -h now", + "2>&1x /sbin/reboot", + # sudo/env wrapper options use their case-sensitive upstream grammars. + "sudo -E /sbin/shutdown", + "sudo -P /sbin/shutdown", + "sudo -H /sbin/shutdown", + "sudo -a type /sbin/shutdown", + "sudo -Eu root /sbin/shutdown", + "sudo -PC 5 /sbin/shutdown", + # getopt_long exact and unambiguous abbreviated required options own the + # following word, while an =-attached operand remains in the option token. + "sudo --use root /sbin/shutdown", + "sudo --chd /tmp /sbin/shutdown", + "sudo --chdir=/tmp /sbin/shutdown", + # Both command-word walkers must share the same bundle handling. + "sudo -P /sbin/shut\\down", + # Whitespace inside a redirection operand's command substitution must not + # terminate the operand scan before the following executable. + ">$(printf '%s' a) /sbin/shutdown", + # Every paren the shell treats as grouping balances the operand's + # substitution, so a nested process substitution cannot close it early and + # leave the executable outside the scanned range. + ">$(echo <(true) tail >/dev/null; printf /dev/null) /sbin/shutdown -h now", + ">$(echo >(cat) ; printf /dev/null) /sbin/shutdown", + ">$(echo <(true)) /sbin/shutdown", + # Parens the shell does NOT treat as grouping must not count, or the scan + # runs past the real closer and swallows the command word behind it. The + # `#` in `$(true)#` is glued to the word and starts no comment. + "( > $(echo /tmp/f # (\n) shutdown -h now )", + "( > $(echo /tmp/f # (\n) rm -rf /etc )", + "( >$(echo $(true)#) /sbin/shutdown -h now\n)", + "> $(echo $((1 + (2))) ) /sbin/shutdown", + # A heredoc inside the operand's substitution puts its body beyond this + # scanner, so the operand is malformed rather than guessed at. + "( > $(cat <<'E'\n(\nE\n) shutdown -h now )", +] + +_ABS_PATH_ALLOW = [ + # A paren inside a parameter expansion is literal text, so counting it + # would end the operand early and block an ordinary command. + "> $(echo ${d:-(}) cat /etc/hosts", + "echo /sbin/shutdown", + "ls -la /sbin/shutdown", + "grep 'shutdown' /var/log/syslog", + "stat /usr/sbin/reboot", + r"stat C:\Windows\System32\shutdown.exe", + "cat /bin/rm", + 'echo "/sbin/init 6"', + "md5sum /sbin/halt", + # basename must match the pattern word exactly, not a near-miss + "/usr/local/bin/rebooter --dry-run", + "./deploy.sh shutdown", + "cat /etc/init.d/reboot", + # a bare assignment prefix is data — projecting its value manufactured + # a False->True flip in the first cut of this fix (Sol round 2) + "X=/sbin/shutdown echo ok", + # a wrapper option's operand is data, not the command word + "env --chdir /tmp/reboot /bin/echo ok", + "env --chdir /tmp/reboot echo ok", + "env --argv0 /tmp/reboot /bin/echo ok", + # GNU env accepts the unambiguous --argv0 abbreviation and still treats + # its following word as data. + "env --arg /tmp/x /bin/echo ok", + # Ambiguous sudo long options make sudo exit before executing a command. + # The projection must not invent an operand skip for them. + "sudo --pre root /sbin/shutdown", + # `}` is not a shell metacharacter: it can end a legitimate word + "/tmp/reboot}", + # benign leading redirections in front of a benign command stay allowed + ">out.txt make build", + "2>/dev/null ls /tmp", + "< /dev/null cat file", + # process substitution feeding a benign command is not the floor's concern + "diff <(sort a) <(sort b)", + # `env -C dir cmd` operand is data; the benign command is not blocked + "env -C /tmp /bin/ls", + # background operator between two benign commands keeps its meaning + "echo hi & echo bye", +] + + +@pytest.mark.parametrize("command", _ABS_PATH_BLOCK) +def test_abs_path_invocation_is_hardline_blocked(command): + is_hl, desc = detect_hardline_command(command) + assert is_hl, f"absolute-path spelling bypassed the floor: {command!r}" + assert desc, "hardline match must provide a description" + + +@pytest.mark.parametrize("command", _ABS_PATH_ALLOW) +def test_abs_path_as_data_is_not_hardline(command): + is_hl, desc = detect_hardline_command(command) + assert not is_hl, f"path-as-data false positive: {command!r} (got: {desc})" + + +def test_abs_path_hardline_not_bypassed_by_yolo(clean_session, monkeypatch): + """The floor must hold for path-spelled commands under yolo too.""" + monkeypatch.setenv("HERMES_YOLO_MODE", "1") + result = check_all_command_guards("/sbin/shutdown -h now", "local") + assert result["approved"] is False + assert result.get("hardline") is True diff --git a/tools/approval.py b/tools/approval.py index db0595747cd64..f6bee0a069f46 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -1288,12 +1288,152 @@ _COMMAND_WRAPPER_WORDS = { "builtin", } _SUDO_OPTIONS_WITH_ARG = { - "-c", "--close-from", + "-a", "--auth-type", + "-C", "--close-from", + "-c", "--login-class", + "-D", "--chdir", "-g", "--group", "-h", "--host", "-p", "--prompt", + "-R", "--chroot", + "-r", "--role", + "-T", "--command-timeout", + "-t", "--type", + "-U", "--other-user", "-u", "--user", } +# GNU env options whose value arrives as the NEXT word. Their operand is +# data (`env --chdir /tmp/reboot /bin/echo` never executes reboot), so the +# projection walker must not treat it as the command word. `=`-attached +# forms are already handled by the generic option check. `-S/--split-string` +# is listed here only for the walker's word skipping — its payload is +# executable-bearing, but parsing it is a separate fail-closed change +# (same verdict as the design review); skipping keeps parity with main. +_ENV_OPTIONS_WITH_ARG = { + "-a", "--argv0", + "-C", "--chdir", + "-S", "--split-string", + "-u", "--unset", +} + +# Complete long-option universes used for getopt_long-compatible exact and +# unambiguous-prefix resolution. Keep these synchronized with sudo v1.9.17 +# src/parse_args.c and GNU coreutils v9.11 src/env.c. Missing or ambiguous +# entries deliberately consume no following word, which biases detection +# toward over-blocking rather than hiding an executable as option data. +_SUDO_LONG_OPTION_ARITY = { + "--other-user": "required", + "--auth-type": "required", + "--close-from": "required", + "--login-class": "required", + "--chdir": "required", + "--group": "required", + "--host": "required", + "--prompt": "required", + "--chroot": "required", + "--role": "required", + "--command-timeout": "required", + "--type": "required", + "--user": "required", + "--preserve-env": "optional", + "--background": "none", + "--edit": "none", + "--set-home": "none", + "--login": "none", + "--remove-timestamp": "none", + "--list": "none", + "--preserve-groups": "none", + "--shell": "none", + "--validate": "none", + "--askpass": "none", + "--bell": "none", + "--help": "none", + "--reset-timestamp": "none", + "--no-update": "none", + "--non-interactive": "none", + "--stdin": "none", + "--version": "none", +} +_ENV_LONG_OPTION_ARITY = { + "--argv0": "required", + "--unset": "required", + "--chdir": "required", + "--split-string": "required", + "--default-signal": "optional", + "--ignore-signal": "optional", + "--block-signal": "optional", + "--ignore-environment": "none", + "--null": "none", + "--debug": "none", + "--list-signal-handling": "none", + "--help": "none", + "--version": "none", +} +_SUDO_SHORT_OPTIONS_REQUIRED_ARG = frozenset("aCcDgpRrTtUu") +_SUDO_SHORT_OPTIONS_OPTIONAL_ARG = frozenset("h") +_SUDO_SHORT_OPTIONS_NO_ARG = frozenset("ABbEeHiKklNnPSsVv") +_ENV_SHORT_OPTIONS_REQUIRED_ARG = frozenset("aCSu") +_ENV_SHORT_OPTIONS_OPTIONAL_ARG = frozenset() +_ENV_SHORT_OPTIONS_NO_ARG = frozenset("i0v") + + +def _wrapper_option_consumes_next_word(wrapper: str, token: str) -> bool: + """Return whether one sudo/env option word owns the following word. + + Short bundles follow each wrapper's case-sensitive optstring. Required + options consume the remainder of their bundle, or the next word when they + occur last. sudo's short ``-h`` follows its got_host_flag behavior and + conservatively consumes the next word when no host is attached. + + Long options use exact-match-first, then unique-prefix resolution like + ``getopt_long``. Optional long arguments only consume ``=``-attached data. + Unknown or ambiguous options consume no next word so executable detection + continues in the fail-closed direction. + """ + if wrapper == "sudo": + long_options = _SUDO_LONG_OPTION_ARITY + required_short = _SUDO_SHORT_OPTIONS_REQUIRED_ARG + optional_short = _SUDO_SHORT_OPTIONS_OPTIONAL_ARG + no_arg_short = _SUDO_SHORT_OPTIONS_NO_ARG + elif wrapper == "env": + long_options = _ENV_LONG_OPTION_ARITY + required_short = _ENV_SHORT_OPTIONS_REQUIRED_ARG + optional_short = _ENV_SHORT_OPTIONS_OPTIONAL_ARG + no_arg_short = _ENV_SHORT_OPTIONS_NO_ARG + else: + return False + + if token.startswith("--"): + option, equals, _ = token.partition("=") + arity = long_options.get(option) + if arity is None: + matches = [ + candidate + for candidate in long_options + if candidate.startswith(option) + ] + if len(matches) != 1: + return False + arity = long_options[matches[0]] + return arity == "required" and not equals + + if ( + not token.startswith("-") + or token == "-" + or "=" in token + or len(token) < 2 + ): + return False + + chars = token[1:] + for index, char in enumerate(chars): + if char in required_short: + return index == len(chars) - 1 + if char in optional_short: + return index == len(chars) - 1 + if char not in no_arg_short: + return False + return False _INTERPRETER_EXEC_FLAGS = { "python": {"-c"}, @@ -1808,9 +1948,42 @@ def _scan_dollar_paren_end(command: str, start: int) -> int | None: if ch == "\\" and i + 1 < len(command): i += 2 continue - if command.startswith("$(", i): + if command.startswith("#", i) and ( + i == start + 2 or command[i - 1].isspace() or command[i - 1] in ";&|(" + ): + # A comment runs to the newline, and its parens are text. Counting + # them would push the depth past the real closer and swallow + # whatever follows the substitution. + newline = command.find("\n", i) + if newline == -1: + return None + i = newline + continue + if command.startswith("${", i): + # `${x:-(}` holds a literal paren, not a group. + end = _scan_brace_expansion_end(command, i) + if end is None: + return None + i = end + continue + if command.startswith("$((", i) or command.startswith("$[", i): + # Arithmetic parens are operators, not grouping. + end = _scan_arithmetic_end(command, i) + if end is None: + return None + i = end + continue + if command.startswith("<<", i) and not command.startswith("<<<", i): + # A heredoc inside the substitution puts its body's parens beyond + # this scanner's reach. Refusing to guess keeps the operand from + # swallowing a command word; the caller treats None as malformed + # and blocks. + return None + if ch == "(": + # Every unquoted paren the shell treats as grouping counts, so + # `<(`, `>(` and plain subshells balance under one rule. depth += 1 - i += 2 + i += 1 continue if ch == ")": depth -= 1 @@ -1822,6 +1995,57 @@ def _scan_dollar_paren_end(command: str, start: int) -> int | None: return None +def _scan_brace_expansion_end(command: str, start: int) -> int | None: + """Return the offset after a balanced ``${...}`` expansion.""" + depth = 0 + i = start + 1 + quote: str | None = None + while i < len(command): + ch = command[i] + if quote: + if ch == "\\" and quote == '"' and i + 1 < len(command): + i += 2 + continue + if ch == quote: + quote = None + i += 1 + continue + if ch == "\\" and i + 1 < len(command): + i += 2 + continue + if ch in ("'", '"'): + quote = ch + i += 1 + continue + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return i + 1 + i += 1 + return None + + +def _scan_arithmetic_end(command: str, start: int) -> int | None: + """Return the offset after ``$((...))`` or the legacy ``$[...]``.""" + if command.startswith("$[", start): + end = command.find("]", start + 2) + return None if end == -1 else end + 1 + depth = 0 + i = start + 1 + while i < len(command): + ch = command[i] + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return i + 1 + i += 1 + return None + + def _scan_backtick_end(command: str, start: int) -> int | None: i = start + 1 while i < len(command): @@ -2158,7 +2382,7 @@ def _iter_shell_command_word_spans(command: str): for command_start in _iter_shell_command_starts(command): pos = command_start prefix_words = 0 - skip_wrapper_options = False + wrapper_kind: str | None = None skip_next_wrapper_arg = False while prefix_words < 12: word_start, word_end, word = _read_shell_word(command, pos) @@ -2171,11 +2395,9 @@ def _iter_shell_command_word_spans(command: str): pos = word_end prefix_words += 1 continue - if skip_wrapper_options and lower_word.startswith("-"): - option_name = lower_word.split("=", 1)[0] - skip_next_wrapper_arg = ( - "=" not in lower_word - and option_name in _SUDO_OPTIONS_WITH_ARG + if wrapper_kind and deobfuscated.startswith("-"): + skip_next_wrapper_arg = _wrapper_option_consumes_next_word( + wrapper_kind, deobfuscated ) pos = word_end prefix_words += 1 @@ -2185,16 +2407,345 @@ def _iter_shell_command_word_spans(command: str): prefix_words += 1 if lower_word in _COMMAND_WRAPPER_WORDS: - skip_wrapper_options = lower_word in {"sudo", "env"} + wrapper_kind = lower_word if lower_word in {"sudo", "env"} else None pos = word_end continue if _ENV_ASSIGNMENT_RE.fullmatch(deobfuscated): - skip_wrapper_options = False + wrapper_kind = None pos = word_end continue break +_EXECUTABLE_LAUNCHER_SUFFIXES = (".exe", ".bat", ".cmd") +_PROJECTED_BASENAME_RE = re.compile(r"[A-Za-z0-9_][\w.+-]*\Z") +_WINDOWS_ABSOLUTE_RE = re.compile(r"[A-Za-z]:[\\/]|\\\\") +# `_read_shell_word` stops at whitespace and `;&|`, so a command that ends +# at a substitution/subshell closer keeps that byte in the word: +# `(/sbin/reboot)` reads as `/sbin/reboot)`. Trim the closers before the +# basename check or the stray byte makes the word look like something other +# than a program name and the projection silently declines. Only `)` and +# the backtick qualify: both are shell metacharacters that can never sit +# inside an unquoted word. `}` is NOT one (`echo foo}` prints `foo}`, and +# a brace group's `}` is its own word behind `;`), so trimming it would +# rewrite legitimate names like `/tmp/reboot}`. +_WORD_TAIL_CLOSERS = ")`" + + +def _projected_executable_basename(word: str) -> str | None: + """Reduce one command-position word to the bare program name a shell + would resolve, or None when the word already is one (or can't be one). + + A Windows-absolute word (drive or UNC prefix) is split on both + separators BEFORE any escape collapsing — the deobfuscation pass treats + backslashes as shell escapes and would dissolve the path. Everything + else goes through the same quote/escape collapsing the r\\m detection + uses, so composed spellings (``'/sbin/'shutdown``, ``/sbin/shut\\down``) + reduce like their plain forms.""" + stripped = _strip_optional_shell_quotes(word) + if _WINDOWS_ABSOLUTE_RE.match(stripped): + basename = stripped.replace("\\", "/").rsplit("/", 1)[-1] + else: + collapsed = _deobfuscate_shell_word_for_detection(word) or word + basename = _strip_optional_shell_quotes(collapsed).rsplit("/", 1)[-1] + lowered = basename.lower() + for suffix in _EXECUTABLE_LAUNCHER_SUFFIXES: + if lowered.endswith(suffix): + basename = basename[: -len(suffix)] + break + if basename == word: + return None # already a bare name — nothing to project + # Refuse anything that doesn't reduce to a plain program name (empty + # basename from a trailing slash, expansion debris): projecting those + # could only manufacture command words a shell would not actually run. + if not _PROJECTED_BASENAME_RE.fullmatch(basename): + return None + return basename + + +_MALFORMED_REDIRECTION = -1 +# `_project_path_spelled_executables` has a string-or-None compatibility +# contract with callers outside this segment. A malformed redirection operand +# therefore projects to a guaranteed hardline marker rather than returning +# None and silently allowing an uninspected command suffix. +_MALFORMED_REDIRECTION_PROJECTION = "\nmkfs" + + +def _skip_leading_redirection(command: str, pos: int) -> int | None: + """If a redirection token begins at ``pos``, return the offset just past it + (and its operand); otherwise return None. + + POSIX simple-command grammar allows ``(assignment | redirection)*`` before + the command word, so a redirection reached by path — ``&1``, ``&>f``, ``>&2``); this + dedicated scanner recognizes them without touching the shared tokenizers. + + ``pos`` must already sit on the first non-space character. Process + substitution operands (``< <(cmd)``) are consumed as balanced ``( )`` so + the outer walk resumes at the command word; the inner command is still + surfaced independently by ``_iter_shell_command_starts``.""" + i = pos + n = len(command) + # Optional leading fd number (``2>``); bare digits are only a redirection + # when an operator follows, otherwise they are an ordinary word. + digits_end = i + while digits_end < n and command[digits_end].isdigit(): + digits_end += 1 + op = i + if digits_end < n and command[digits_end] in "<>": + op = digits_end + elif command[i] == "&" and i + 1 < n and command[i + 1] == ">": + op = i # ``&>`` / ``&>>``: whole-command redirect, no fd prefix + else: + return None # not a redirection + + # Consume the operator glyphs. + if command[op] == "&": + j = op + 1 # at '>' + j += 1 + if j < n and command[j] == ">": + j += 1 # ``&>>`` + else: + ch = command[op] + j = op + 1 + if ch == "<" and command.startswith("<<", op): + # heredoc / herestring: ``<<`` ``<<-`` ``<<<`` + j = op + 2 + if j < n and command[j] == "-": + j += 1 + elif j < n and command[j] == "<": + j += 1 # ``<<<`` + elif ch == ">" and j < n and command[j] == ">": + j += 1 # ``>>`` + elif ch == "<" and j < n and command[j] == ">": + j += 1 # ``<>`` + elif ch == ">" and j < n and command[j] == "|": + j += 1 # ``>|`` + # fd duplication ``>&`` / ``<&`` (also ``2>&1``, ``>&-``) + if j < n and command[j] == "&": + j += 1 + fd_end = j + while fd_end < n and (command[fd_end].isdigit() or command[fd_end] == "-"): + fd_end += 1 + # A real fd-dup operand is a whole word of digits/`-`; a digit run + # glued to more characters (``>&2x``) is a filename, not fd 2. + if fd_end > j and (fd_end >= n or command[fd_end].isspace() + or command[fd_end] in ";&|<>"): + return fd_end # fused fd / `-` + # ``>&file`` (redirect both streams to a file) or a glued + # non-fd token: the operand is a filename, fused or separated — + # fall through to operand scanning. + + # Operator consumed; now the operand. If it is fused (no separating + # space), the operand chars run to the next whitespace/metacharacter, + # except a process-substitution operand which we balance. + k = j + if k < n and not command[k].isspace(): + operand_end = _scan_redirection_operand(command, k) + return ( + operand_end + if operand_end is not None + else _MALFORMED_REDIRECTION + ) + # Separated operand: skip whitespace, then the operand word. + k = _skip_shell_whitespace(command, k) + if k >= n: + return j # dangling operator; nothing to consume + operand_end = _scan_redirection_operand(command, k) + return ( + operand_end + if operand_end is not None + else _MALFORMED_REDIRECTION + ) + + +def _scan_redirection_operand(command: str, pos: int) -> int | None: + """Return the offset past one redirection operand starting at ``pos``. + + ``<(cmd)`` / ``>(cmd)`` process substitutions are consumed as a balanced + paren group; every other operand runs to the next unquoted whitespace or + metacharacter (mirroring ``_read_shell_word`` minus the ``&`` break, which + inside an operand is just filename text). Returns None when a command + substitution or backtick payload is unterminated.""" + n = len(command) + i = pos + if command.startswith("<(", i) or command.startswith(">(", i): + depth = 0 + i += 1 # at '(' + sub_quote: str | None = None + while i < n: + ch = command[i] + if sub_quote: + if ch == "\\" and sub_quote == '"' and i + 1 < n: + i += 2 + continue + if ch == sub_quote: + sub_quote = None + i += 1 + continue + if ch in ("'", '"'): + sub_quote = ch + i += 1 + continue + if ch == "\\" and i + 1 < n: + i += 2 + continue + if command.startswith("$(", i): + end = _scan_dollar_paren_end(command, i) + if end is None: + return None + i = end + continue + if ch == "`": + end = _scan_backtick_end(command, i) + if end is None: + return None + i = end + continue + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return i + 1 + i += 1 + return None + quote: str | None = None + while i < n: + ch = command[i] + if quote: + if ch == "\\" and quote == '"' and i + 1 < n: + i += 2 + continue + if ch == quote: + quote = None + i += 1 + continue + if ch in ("'", '"'): + quote = ch + i += 1 + continue + if ch == "\\" and i + 1 < n: + i += 2 + continue + if command.startswith("$(", i): + end = _scan_dollar_paren_end(command, i) + if end is None: + return None + i = end + continue + if ch == "`": + end = _scan_backtick_end(command, i) + if end is None: + return None + i = end + continue + if ch.isspace() or ch in ";&|<>": + break + i += 1 + return i + + +def _project_path_spelled_executables(command: str) -> str | None: + """Detection-only variant: rewrite command-position executables that are + spelled by path (POSIX or Windows, quoted or bare, with or without a + launcher suffix) to ``\\n`` so the flat ``_CMDPOS``-anchored + patterns see the same command word a bare spelling would produce — + ``/sbin/shutdown``, ``C:\\Windows\\System32\\shutdown.exe`` and + ``"C:\\Program Files\\Git\\usr\\bin\\rm.exe"`` must behave exactly like + ``shutdown`` / ``rm``. Must run on the RAW command: the global + normalization strips backslashes and fuses a Windows path into one word + before the basename could be recovered. + + The walk mirrors ``_iter_shell_command_word_spans`` but recognizes + wrappers by their PROJECTED basename, so a path-spelled wrapper chain + (``/usr/bin/env /usr/bin/sudo /sbin/shutdown``) resolves in this single + pass — no fixpoint, no overlapping re-splice. Assignment-shaped words + (``X=/sbin/shutdown``) are prefix data, never executables. Returns None + when nothing needed rewriting.""" + replacements: dict[int, tuple[int, str]] = {} + for command_start in _iter_shell_command_starts(command): + pos = command_start + wrapper_kind: str | None = None + skip_next_wrapper_arg = False + # POSIX prefixes (``VAR=v``, ``2>/dev/null``) can precede the command + # word any number of times. When one is consumed, the command word + # that follows sits mid-string where the flat ``_CMDPOS`` anchors would + # miss even a bare spelling, so a ``\n`` boundary is spliced in front of + # it just as a path spelling is projected to its basename. + prefix_consumed = False + # Unbounded walk with a progress guard (never fail open on a long + # prefix run): every step past here advances ``pos`` strictly. + while True: + scan = _skip_shell_whitespace(command, pos) + redir_end = _skip_leading_redirection(command, scan) if scan < len(command) else None + if redir_end == _MALFORMED_REDIRECTION: + return _MALFORMED_REDIRECTION_PROJECTION + if redir_end is not None and redir_end > scan: + prefix_consumed = True + pos = redir_end + continue + + word_start, raw_end, _raw_word = _read_shell_word(command, pos) + if word_start == raw_end: + break + pos = raw_end + # Trim group/substitution closers the word reader keeps + # (`(/sbin/reboot)` reads as `/sbin/reboot)`); the closer stays + # outside the replacement span so the splice preserves it. + word_end = raw_end + while word_end > word_start and command[word_end - 1] in _WORD_TAIL_CLOSERS: + word_end -= 1 + if word_end == word_start: + continue # bare closer: structure, not a word + word = command[word_start:word_end] + if skip_next_wrapper_arg: + skip_next_wrapper_arg = False + # The option that owns this operand already proved a prefix + # ran, so the bare command word that follows needs the same + # ``\n`` anchor a projected path spelling would get. + prefix_consumed = True + continue + deobfuscated = _deobfuscate_shell_word_for_detection(word) or word + if _ENV_ASSIGNMENT_RE.fullmatch(deobfuscated): + prefix_consumed = True + continue # VAR=value prefix: data, keep walking + if wrapper_kind and deobfuscated.startswith("-"): + skip_next_wrapper_arg = _wrapper_option_consumes_next_word( + wrapper_kind, deobfuscated + ) + # A wrapper option is itself a prefix word, whether or not it + # owns an operand: ``sudo -E rm ...`` needs the same anchor as + # ``sudo -u root rm ...`` in front of the bare command word + # that follows. + prefix_consumed = True + continue + basename = _projected_executable_basename(word) + effective = (basename or deobfuscated).lower() + if basename is not None and word_start not in replacements: + replacements[word_start] = (word_end, basename) + elif ( + basename is None + and prefix_consumed + and word_start not in replacements + ): + # A prefix ran, so this bare command word needs the same ``\n`` + # anchor a projected path spelling would get. + replacements[word_start] = (word_end, word) + if effective in _COMMAND_WRAPPER_WORDS: + wrapper_kind = effective if effective in {"sudo", "env"} else None + continue # wrapper (bare or path-spelled): walk to the command + break + if not replacements: + return None + projected = command + for word_start in sorted(replacements, reverse=True): + word_end, basename = replacements[word_start] + projected = projected[:word_start] + "\n" + basename + projected[word_end:] + return projected + + def _command_detection_variants(command: str): # Mask quoted newlines BEFORE normalization: normalization strips # backslash-escapes (\" -> ") and empty-string pairs (""), which would @@ -2241,6 +2792,16 @@ def _command_detection_variants(command: str): if marked_payload != payload and marked_payload not in seen: seen.add(marked_payload) yield marked_payload + # A payload's own executable can be path-spelled too + # (`bash -c '/sbin/shutdown -h now'`), so it needs the same + # basename projection as the outer command. POSIX paths + # survive the normalization the payload came through; + # backslash Windows paths inside payloads stay out of reach + # until normalization preserves separators (#71919). + projected_payload = _project_path_spelled_executables(payload) + if projected_payload is not None and projected_payload not in seen: + seen.add(projected_payload) + yield projected_payload pending.append(payload) # Subshell `(cmd)` and brace-group `{ cmd; }` openers put `cmd` at a real # command position, but the flat `_CMDPOS`-anchored patterns can't see it: @@ -2257,6 +2818,21 @@ def _command_detection_variants(command: str): if marked != grep_safe and marked not in seen: seen.add(marked) yield marked + # Absolute-path spellings bypass every _CMDPOS-anchored rule the same + # way subshell openers did: the anchor class knows wrappers but not + # paths, so `/sbin/shutdown` or `C:\Windows\System32\shutdown.exe` + # never reaches `shutdown\b`. Project command-position executables to + # `\n` from the RAW command (normalization strips backslashes + # and fuses Windows paths before a basename could be recovered), then + # feed the projection through the standard normalize/grep-safe pipe. + projected = _project_path_spelled_executables(command) + if projected is not None: + projected_variant, _ = _grep_safe_detection_variant( + _normalize_command_for_detection(projected) + ) + if projected_variant not in seen: + seen.add(projected_variant) + yield projected_variant # Shell quoting/escaping can spell a dangerous executable name in pieces # (for example r\m or r''m). Keep that deobfuscation scoped to command # words so similarly shaped arguments do not become false positives.