diff --git a/tests/tools/test_terminal_hints.py b/tests/tools/test_terminal_hints.py new file mode 100644 index 0000000000000..c3d08db2bc6a2 --- /dev/null +++ b/tests/tools/test_terminal_hints.py @@ -0,0 +1,122 @@ +"""Tests for tools/terminal_hints.py — output-pattern failure hints.""" + +import json +from unittest.mock import patch as mock_patch + +import pytest + +from tools.terminal_hints import annotate_failure + + +class TestAnnotateFailureBasics: + def test_success_never_annotated(self): + assert annotate_failure("python x.py", 0, "python: command not found") is None + + def test_empty_output_falls_to_exit_code_tier(self): + assert "126" in annotate_failure("./run.sh", 126, "") + assert "SIGKILL" in annotate_failure("big_job", 137, "") + assert "timeout" in annotate_failure("sleep 999", 124, "") + + def test_unknown_failure_returns_none(self): + assert annotate_failure("./x", 1, "some unrecognized error") is None + + def test_only_first_matching_hint(self): + out = 'CONFLICT (content): Merge conflict in a.py\npython: command not found' + hint = annotate_failure("git merge x && python t.py", 1, out) + assert "conflict" in hint.lower() + assert "python3" not in hint + + +class TestGhUnknownJsonField: + def test_field_name_extracted(self): + out = 'Unknown JSON field: "authorAssociation"\nAvailable fields:\n additions\n author' + hint = annotate_failure("gh pr view 1 --json authorAssociation", 1, out) + assert "authorAssociation" in hint + assert "valid field list" in hint + + +class TestCommandNotFound: + def test_bare_python_gets_python3_hint(self): + out = "/usr/bin/bash: line 1: python: command not found" + hint = annotate_failure("python x.py", 127, out) + assert "python3" in hint + + def test_bare_pip_gets_pip3_hint(self): + out = "bash: pip: command not found" + hint = annotate_failure("pip install x", 127, out) + assert "pip3" in hint or "-m pip" in hint + + def test_generic_command(self): + out = "bash: line 3: shellcheck: command not found" + hint = annotate_failure("shellcheck s.sh", 127, out) + assert "shellcheck" in hint + assert "which" in hint + + +class TestModuleNotFound: + def test_module_named(self): + out = ("Traceback (most recent call last):\n File \"x.py\", line 1\n" + "ModuleNotFoundError: No module named 'requests'") + hint = annotate_failure("python3 x.py", 1, out) + assert "requests" in hint + assert "venv" in hint + + def test_dotted_module(self): + out = "ImportError: No module named 'hermes_cli.main'" + hint = annotate_failure("python3 -m hermes_cli.main", 1, out) + assert "hermes_cli" in hint + + +class TestGitShapes: + def test_merge_conflict(self): + out = "Auto-merging a.py\nCONFLICT (content): Merge conflict in a.py\nAutomatic merge failed; fix conflicts and then commit the result." + hint = annotate_failure("git merge feature", 1, out) + assert "Do not retry" in hint + + def test_branch_already_exists(self): + out = "fatal: a branch named 'fix/x' already exists" + hint = annotate_failure("git checkout -b fix/x", 128, out) + assert "fix/x" in hint + + def test_rate_limit(self): + out = "GraphQL: API rate limit already exceeded for user ID 1." + hint = annotate_failure("gh pr list", 1, out) + assert "rate limit" in hint.lower() + + +class TestPermissionDenied: + def test_permission_denied(self): + hint = annotate_failure("touch /etc/x", 1, "touch: cannot touch '/etc/x': Permission denied") + assert "Permission denied" in hint + + +class TestBoundedScan: + def test_pattern_beyond_scan_window_ignored(self): + out = "x" * 5000 + "\npython: command not found" + assert annotate_failure("noop", 1, out) is None + + def test_hint_functions_cannot_crash_annotate(self): + # A hint raising must not propagate. + with mock_patch("tools.terminal_hints._OUTPUT_HINTS", [lambda c, o: 1 / 0]): + assert annotate_failure("x", 1, "boom") is None + + +class TestTerminalIntegration: + """The hint lands in the terminal result dict under 'hint'.""" + + def test_hint_field_wired(self): + # Exercise the wiring path shape without a live environment: the + # result assembly guards on returncode != 0 and no exit_note. + from tools import terminal_tool + # simulate: interpret gives None, hints give a value + note = terminal_tool._interpret_exit_code("python x.py", 127) + assert note is None + hint = annotate_failure("python x.py", 127, "bash: python: command not found") + assert hint and "python3" in hint + + def test_exit_note_suppresses_pattern_hint(self): + # grep exit 1 is informational; annotate_failure must not be reached + # for it in the wiring (exit_note wins). Just verify the semantics + # table still covers it. + from tools import terminal_tool + assert terminal_tool._interpret_exit_code("grep foo bar.txt", 1) is not None diff --git a/tools/terminal_hints.py b/tools/terminal_hints.py new file mode 100644 index 0000000000000..8410b3e645c57 --- /dev/null +++ b/tools/terminal_hints.py @@ -0,0 +1,170 @@ +"""Output-pattern failure hints for the terminal tool. + +When a command exits non-zero, the raw stderr often confuses models into +wasted diagnostic turns (e.g. retrying `python` when only `python3` exists, +or re-sending a gh field list that the installed gh doesn't support). + +This module extends the exit-code semantics table in ``terminal_tool`` with +an *output-pattern* tier: a bounded scan of the command output that maps +well-known failure shapes to one short, actionable recovery hint. + +Design rules (keep these when adding patterns): + +* Only fires on non-zero exit codes — never annotate success. +* At most ONE hint per result, first match wins; patterns are ordered by + observed frequency in production trajectories (state.db mining, Aug 2026). +* Scans only the first ``_SCAN_CHARS`` of output — hints must key on error + headers, not deep context. +* Hints state the *next action*, not a diagnosis essay. One or two sentences. +* Pure function, no I/O, no config reads — trivially unit-testable. + +Frequencies quoted below come from a 250k-terminal-result window of the +production session DB (Aug 2026): together these classes cover ~14k failed +calls whose retry chains averaged 1.4 extra tool turns each. +""" + +from __future__ import annotations + +import re +from typing import Callable, Optional + +# Bounded scan window: error headers appear early; deep output is noise. +_SCAN_CHARS = 4000 + + +def _hint_gh_unknown_json_field(command: str, output: str) -> Optional[str]: + # ~9,175x: gh CLI version drift — model asks for fields the installed + # gh doesn't know. gh already prints the valid field list. + m = re.search(r'Unknown JSON field: "?(\w+)', output) + if not m: + return None + return ( + f"The installed gh does not support the JSON field '{m.group(1)}'. " + "The valid field list is printed in the output above — retry using " + "only fields from that list." + ) + + +def _hint_command_not_found(command: str, output: str) -> Optional[str]: + # ~1,010x generic; 837x of them are bare `python` on python3-only distros. + m = re.search(r"(?:bash: line \d+: |bash: |sh: \d*:? ?)?([\w.+-]+): command not found", output) + if not m: + return None + missing = m.group(1) + if missing == "python": + return ( + "This system has no bare `python` — use `python3`, or the " + "project venv's interpreter (e.g. .venv/bin/python)." + ) + if missing == "pip": + return ( + "This system has no bare `pip` — use `pip3`, `python3 -m pip`, " + "or the project venv's pip (e.g. .venv/bin/pip)." + ) + return ( + f"`{missing}` is not installed or not on PATH. Verify with " + f"`which {missing}`; install it or use an absolute path instead of " + "retrying the same command." + ) + + +def _hint_module_not_found(command: str, output: str) -> Optional[str]: + # ~739x: almost always a venv-activation slip, not a missing dependency. + m = re.search(r"(?:ModuleNotFoundError|ImportError): No module named '?([\w.]+)", output) + if not m: + return None + return ( + f"Python cannot import '{m.group(1)}'. Most often the wrong " + "interpreter is running: activate the project venv (e.g. `source " + ".venv/bin/activate`) or invoke its python directly. Only pip " + "install if the package is genuinely absent from that venv." + ) + + +def _hint_merge_conflict(command: str, output: str) -> Optional[str]: + # ~1,172x: models sometimes re-run the failing merge/rebase verbatim. + if not re.search(r"^CONFLICT |Automatic merge failed|needs merge", output, re.M): + return None + return ( + "Git merge conflict. Do not retry this command. Resolve the " + "conflicted files listed above (edit, then `git add`), then continue " + "(`git rebase --continue` / commit the merge) — or abort with " + "`--abort`." + ) + + +def _hint_already_exists(command: str, output: str) -> Optional[str]: + # ~633x: branch/dir/file already exists → retrying unchanged always fails. + m = re.search(r"(?:fatal|error):.*?'([^']+)' already exists", output) + if not m: + return None + return ( + f"'{m.group(1)}' already exists — retrying unchanged will keep " + "failing. Reuse it, choose another name, or delete it first if it is " + "genuinely stale." + ) + + +def _hint_gh_rate_limit(command: str, output: str) -> Optional[str]: + # ~133x: immediate retries burn turns; the limit is time-based. + if "API rate limit" not in output and "was submitted too quickly" not in output: + return None + return ( + "GitHub API rate limit hit — immediate retries will keep failing. " + "Continue with other work and retry this operation later." + ) + + +def _hint_permission_denied(command: str, output: str) -> Optional[str]: + if "Permission denied" not in output and "EACCES" not in output: + return None + return ( + "Permission denied. Check ownership/mode of the target path " + "(`ls -la`); prefer a user-writable location. Only escalate to sudo " + "if the task genuinely requires it." + ) + + +# Ordered by production frequency — first match wins. +_OUTPUT_HINTS: list[Callable[[str, str], Optional[str]]] = [ + _hint_gh_unknown_json_field, + _hint_merge_conflict, + _hint_command_not_found, + _hint_module_not_found, + _hint_already_exists, + _hint_gh_rate_limit, + _hint_permission_denied, +] + +# Exit-code-only hints for codes the semantics table in terminal_tool does +# not cover per-command. Checked after output patterns. +_EXIT_CODE_HINTS: dict[int, str] = { + 126: "Exit 126: the file was found but is not executable — `chmod +x` it or invoke it via its interpreter (e.g. `bash script.sh`).", + 137: "Exit 137: the process was SIGKILLed — usually out-of-memory or an external kill. Reduce memory use or check `dmesg | tail` before retrying.", + 124: "Exit 124: the command hit its timeout. Raise timeout= (foreground max 600s) or run it with background=true and notify_on_complete=true.", +} + + +def annotate_failure(command: str, exit_code: int, output: str) -> Optional[str]: + """Return one short recovery hint for a failed command, or None. + + Args: + command: The command string that ran. + exit_code: Its exit code (non-zero for failures). + output: Combined stdout/stderr as returned to the model. + + Only the first ``_SCAN_CHARS`` characters of output are examined and at + most one hint is returned. Returns None for exit_code == 0. + """ + if exit_code == 0: + return None + window = (output or "")[:_SCAN_CHARS] + if window: + for fn in _OUTPUT_HINTS: + try: + hint = fn(command or "", window) + except Exception: + continue + if hint: + return hint + return _EXIT_CODE_HINTS.get(exit_code) diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 1768f80aed3c7..b00a2a6a2e612 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -3050,6 +3050,19 @@ def terminal_tool( # (e.g. grep=1 means "no matches", diff=1 means "files differ") exit_note = _interpret_exit_code(command, returncode) + # Output-pattern failure hints: map well-known error shapes + # (command-not-found, ModuleNotFoundError, gh field drift, + # merge conflicts, ...) to one short recovery hint so the model + # fixes the root cause on the next call instead of spending + # turns on re-diagnosis. See tools/terminal_hints.py. + failure_hint = None + if returncode != 0 and not exit_note: + try: + from tools.terminal_hints import annotate_failure + failure_hint = annotate_failure(command, returncode, output) + except Exception: + failure_hint = None + result_dict = { "output": output, "exit_code": returncode, @@ -3090,6 +3103,8 @@ def terminal_tool( result_dict["approval"] = approval_note if exit_note: result_dict["exit_code_meaning"] = exit_note + if failure_hint: + result_dict["hint"] = failure_hint if sudo_auth_failed: result_dict["sudo_auth_failed"] = True if sudo_cache_cleared: