feat(terminal): output-pattern failure hints for common error classes
When a command exits non-zero, scan the first 4KB of output for
well-known failure shapes and attach one short, actionable recovery
hint to the tool result ('hint' field):
- gh 'Unknown JSON field' (9.2k occurrences in a 250k-call window)
- git merge conflicts (1.2k) — stop verbatim retries
- command not found (1.0k), incl. python->python3 and pip->pip3
- ModuleNotFoundError (739) — venv activation guidance
- 'already exists' (633), gh rate limits (133), permission denied
- exit-code-only tier: 124 timeout, 126 not-executable, 137 SIGKILL
Hints are suppressed when the existing exit_code_meaning tier already
explains the code (grep=1 etc). Pattern order = production frequency
from state.db mining; first match wins; pure function, no I/O.
This commit is contained in:
Teknium
parent
5b4d20b524
commit
677473273e
|
|
@ -0,0 +1,122 @@
|
|||
"""Tests for tools/terminal_hints.py — output-pattern failure hints."""
|
||||
|
||||
import json
|
||||
from unittest.mock import patch as mock_patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.terminal_hints import annotate_failure
|
||||
|
||||
|
||||
class TestAnnotateFailureBasics:
|
||||
def test_success_never_annotated(self):
|
||||
assert annotate_failure("python x.py", 0, "python: command not found") is None
|
||||
|
||||
def test_empty_output_falls_to_exit_code_tier(self):
|
||||
assert "126" in annotate_failure("./run.sh", 126, "")
|
||||
assert "SIGKILL" in annotate_failure("big_job", 137, "")
|
||||
assert "timeout" in annotate_failure("sleep 999", 124, "")
|
||||
|
||||
def test_unknown_failure_returns_none(self):
|
||||
assert annotate_failure("./x", 1, "some unrecognized error") is None
|
||||
|
||||
def test_only_first_matching_hint(self):
|
||||
out = 'CONFLICT (content): Merge conflict in a.py\npython: command not found'
|
||||
hint = annotate_failure("git merge x && python t.py", 1, out)
|
||||
assert "conflict" in hint.lower()
|
||||
assert "python3" not in hint
|
||||
|
||||
|
||||
class TestGhUnknownJsonField:
|
||||
def test_field_name_extracted(self):
|
||||
out = 'Unknown JSON field: "authorAssociation"\nAvailable fields:\n additions\n author'
|
||||
hint = annotate_failure("gh pr view 1 --json authorAssociation", 1, out)
|
||||
assert "authorAssociation" in hint
|
||||
assert "valid field list" in hint
|
||||
|
||||
|
||||
class TestCommandNotFound:
|
||||
def test_bare_python_gets_python3_hint(self):
|
||||
out = "/usr/bin/bash: line 1: python: command not found"
|
||||
hint = annotate_failure("python x.py", 127, out)
|
||||
assert "python3" in hint
|
||||
|
||||
def test_bare_pip_gets_pip3_hint(self):
|
||||
out = "bash: pip: command not found"
|
||||
hint = annotate_failure("pip install x", 127, out)
|
||||
assert "pip3" in hint or "-m pip" in hint
|
||||
|
||||
def test_generic_command(self):
|
||||
out = "bash: line 3: shellcheck: command not found"
|
||||
hint = annotate_failure("shellcheck s.sh", 127, out)
|
||||
assert "shellcheck" in hint
|
||||
assert "which" in hint
|
||||
|
||||
|
||||
class TestModuleNotFound:
|
||||
def test_module_named(self):
|
||||
out = ("Traceback (most recent call last):\n File \"x.py\", line 1\n"
|
||||
"ModuleNotFoundError: No module named 'requests'")
|
||||
hint = annotate_failure("python3 x.py", 1, out)
|
||||
assert "requests" in hint
|
||||
assert "venv" in hint
|
||||
|
||||
def test_dotted_module(self):
|
||||
out = "ImportError: No module named 'hermes_cli.main'"
|
||||
hint = annotate_failure("python3 -m hermes_cli.main", 1, out)
|
||||
assert "hermes_cli" in hint
|
||||
|
||||
|
||||
class TestGitShapes:
|
||||
def test_merge_conflict(self):
|
||||
out = "Auto-merging a.py\nCONFLICT (content): Merge conflict in a.py\nAutomatic merge failed; fix conflicts and then commit the result."
|
||||
hint = annotate_failure("git merge feature", 1, out)
|
||||
assert "Do not retry" in hint
|
||||
|
||||
def test_branch_already_exists(self):
|
||||
out = "fatal: a branch named 'fix/x' already exists"
|
||||
hint = annotate_failure("git checkout -b fix/x", 128, out)
|
||||
assert "fix/x" in hint
|
||||
|
||||
def test_rate_limit(self):
|
||||
out = "GraphQL: API rate limit already exceeded for user ID 1."
|
||||
hint = annotate_failure("gh pr list", 1, out)
|
||||
assert "rate limit" in hint.lower()
|
||||
|
||||
|
||||
class TestPermissionDenied:
|
||||
def test_permission_denied(self):
|
||||
hint = annotate_failure("touch /etc/x", 1, "touch: cannot touch '/etc/x': Permission denied")
|
||||
assert "Permission denied" in hint
|
||||
|
||||
|
||||
class TestBoundedScan:
|
||||
def test_pattern_beyond_scan_window_ignored(self):
|
||||
out = "x" * 5000 + "\npython: command not found"
|
||||
assert annotate_failure("noop", 1, out) is None
|
||||
|
||||
def test_hint_functions_cannot_crash_annotate(self):
|
||||
# A hint raising must not propagate.
|
||||
with mock_patch("tools.terminal_hints._OUTPUT_HINTS", [lambda c, o: 1 / 0]):
|
||||
assert annotate_failure("x", 1, "boom") is None
|
||||
|
||||
|
||||
class TestTerminalIntegration:
|
||||
"""The hint lands in the terminal result dict under 'hint'."""
|
||||
|
||||
def test_hint_field_wired(self):
|
||||
# Exercise the wiring path shape without a live environment: the
|
||||
# result assembly guards on returncode != 0 and no exit_note.
|
||||
from tools import terminal_tool
|
||||
# simulate: interpret gives None, hints give a value
|
||||
note = terminal_tool._interpret_exit_code("python x.py", 127)
|
||||
assert note is None
|
||||
hint = annotate_failure("python x.py", 127, "bash: python: command not found")
|
||||
assert hint and "python3" in hint
|
||||
|
||||
def test_exit_note_suppresses_pattern_hint(self):
|
||||
# grep exit 1 is informational; annotate_failure must not be reached
|
||||
# for it in the wiring (exit_note wins). Just verify the semantics
|
||||
# table still covers it.
|
||||
from tools import terminal_tool
|
||||
assert terminal_tool._interpret_exit_code("grep foo bar.txt", 1) is not None
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
"""Output-pattern failure hints for the terminal tool.
|
||||
|
||||
When a command exits non-zero, the raw stderr often confuses models into
|
||||
wasted diagnostic turns (e.g. retrying `python` when only `python3` exists,
|
||||
or re-sending a gh field list that the installed gh doesn't support).
|
||||
|
||||
This module extends the exit-code semantics table in ``terminal_tool`` with
|
||||
an *output-pattern* tier: a bounded scan of the command output that maps
|
||||
well-known failure shapes to one short, actionable recovery hint.
|
||||
|
||||
Design rules (keep these when adding patterns):
|
||||
|
||||
* Only fires on non-zero exit codes — never annotate success.
|
||||
* At most ONE hint per result, first match wins; patterns are ordered by
|
||||
observed frequency in production trajectories (state.db mining, Aug 2026).
|
||||
* Scans only the first ``_SCAN_CHARS`` of output — hints must key on error
|
||||
headers, not deep context.
|
||||
* Hints state the *next action*, not a diagnosis essay. One or two sentences.
|
||||
* Pure function, no I/O, no config reads — trivially unit-testable.
|
||||
|
||||
Frequencies quoted below come from a 250k-terminal-result window of the
|
||||
production session DB (Aug 2026): together these classes cover ~14k failed
|
||||
calls whose retry chains averaged 1.4 extra tool turns each.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Callable, Optional
|
||||
|
||||
# Bounded scan window: error headers appear early; deep output is noise.
|
||||
_SCAN_CHARS = 4000
|
||||
|
||||
|
||||
def _hint_gh_unknown_json_field(command: str, output: str) -> Optional[str]:
|
||||
# ~9,175x: gh CLI version drift — model asks for fields the installed
|
||||
# gh doesn't know. gh already prints the valid field list.
|
||||
m = re.search(r'Unknown JSON field: "?(\w+)', output)
|
||||
if not m:
|
||||
return None
|
||||
return (
|
||||
f"The installed gh does not support the JSON field '{m.group(1)}'. "
|
||||
"The valid field list is printed in the output above — retry using "
|
||||
"only fields from that list."
|
||||
)
|
||||
|
||||
|
||||
def _hint_command_not_found(command: str, output: str) -> Optional[str]:
|
||||
# ~1,010x generic; 837x of them are bare `python` on python3-only distros.
|
||||
m = re.search(r"(?:bash: line \d+: |bash: |sh: \d*:? ?)?([\w.+-]+): command not found", output)
|
||||
if not m:
|
||||
return None
|
||||
missing = m.group(1)
|
||||
if missing == "python":
|
||||
return (
|
||||
"This system has no bare `python` — use `python3`, or the "
|
||||
"project venv's interpreter (e.g. .venv/bin/python)."
|
||||
)
|
||||
if missing == "pip":
|
||||
return (
|
||||
"This system has no bare `pip` — use `pip3`, `python3 -m pip`, "
|
||||
"or the project venv's pip (e.g. .venv/bin/pip)."
|
||||
)
|
||||
return (
|
||||
f"`{missing}` is not installed or not on PATH. Verify with "
|
||||
f"`which {missing}`; install it or use an absolute path instead of "
|
||||
"retrying the same command."
|
||||
)
|
||||
|
||||
|
||||
def _hint_module_not_found(command: str, output: str) -> Optional[str]:
|
||||
# ~739x: almost always a venv-activation slip, not a missing dependency.
|
||||
m = re.search(r"(?:ModuleNotFoundError|ImportError): No module named '?([\w.]+)", output)
|
||||
if not m:
|
||||
return None
|
||||
return (
|
||||
f"Python cannot import '{m.group(1)}'. Most often the wrong "
|
||||
"interpreter is running: activate the project venv (e.g. `source "
|
||||
".venv/bin/activate`) or invoke its python directly. Only pip "
|
||||
"install if the package is genuinely absent from that venv."
|
||||
)
|
||||
|
||||
|
||||
def _hint_merge_conflict(command: str, output: str) -> Optional[str]:
|
||||
# ~1,172x: models sometimes re-run the failing merge/rebase verbatim.
|
||||
if not re.search(r"^CONFLICT |Automatic merge failed|needs merge", output, re.M):
|
||||
return None
|
||||
return (
|
||||
"Git merge conflict. Do not retry this command. Resolve the "
|
||||
"conflicted files listed above (edit, then `git add`), then continue "
|
||||
"(`git rebase --continue` / commit the merge) — or abort with "
|
||||
"`--abort`."
|
||||
)
|
||||
|
||||
|
||||
def _hint_already_exists(command: str, output: str) -> Optional[str]:
|
||||
# ~633x: branch/dir/file already exists → retrying unchanged always fails.
|
||||
m = re.search(r"(?:fatal|error):.*?'([^']+)' already exists", output)
|
||||
if not m:
|
||||
return None
|
||||
return (
|
||||
f"'{m.group(1)}' already exists — retrying unchanged will keep "
|
||||
"failing. Reuse it, choose another name, or delete it first if it is "
|
||||
"genuinely stale."
|
||||
)
|
||||
|
||||
|
||||
def _hint_gh_rate_limit(command: str, output: str) -> Optional[str]:
|
||||
# ~133x: immediate retries burn turns; the limit is time-based.
|
||||
if "API rate limit" not in output and "was submitted too quickly" not in output:
|
||||
return None
|
||||
return (
|
||||
"GitHub API rate limit hit — immediate retries will keep failing. "
|
||||
"Continue with other work and retry this operation later."
|
||||
)
|
||||
|
||||
|
||||
def _hint_permission_denied(command: str, output: str) -> Optional[str]:
|
||||
if "Permission denied" not in output and "EACCES" not in output:
|
||||
return None
|
||||
return (
|
||||
"Permission denied. Check ownership/mode of the target path "
|
||||
"(`ls -la`); prefer a user-writable location. Only escalate to sudo "
|
||||
"if the task genuinely requires it."
|
||||
)
|
||||
|
||||
|
||||
# Ordered by production frequency — first match wins.
|
||||
_OUTPUT_HINTS: list[Callable[[str, str], Optional[str]]] = [
|
||||
_hint_gh_unknown_json_field,
|
||||
_hint_merge_conflict,
|
||||
_hint_command_not_found,
|
||||
_hint_module_not_found,
|
||||
_hint_already_exists,
|
||||
_hint_gh_rate_limit,
|
||||
_hint_permission_denied,
|
||||
]
|
||||
|
||||
# Exit-code-only hints for codes the semantics table in terminal_tool does
|
||||
# not cover per-command. Checked after output patterns.
|
||||
_EXIT_CODE_HINTS: dict[int, str] = {
|
||||
126: "Exit 126: the file was found but is not executable — `chmod +x` it or invoke it via its interpreter (e.g. `bash script.sh`).",
|
||||
137: "Exit 137: the process was SIGKILLed — usually out-of-memory or an external kill. Reduce memory use or check `dmesg | tail` before retrying.",
|
||||
124: "Exit 124: the command hit its timeout. Raise timeout= (foreground max 600s) or run it with background=true and notify_on_complete=true.",
|
||||
}
|
||||
|
||||
|
||||
def annotate_failure(command: str, exit_code: int, output: str) -> Optional[str]:
|
||||
"""Return one short recovery hint for a failed command, or None.
|
||||
|
||||
Args:
|
||||
command: The command string that ran.
|
||||
exit_code: Its exit code (non-zero for failures).
|
||||
output: Combined stdout/stderr as returned to the model.
|
||||
|
||||
Only the first ``_SCAN_CHARS`` characters of output are examined and at
|
||||
most one hint is returned. Returns None for exit_code == 0.
|
||||
"""
|
||||
if exit_code == 0:
|
||||
return None
|
||||
window = (output or "")[:_SCAN_CHARS]
|
||||
if window:
|
||||
for fn in _OUTPUT_HINTS:
|
||||
try:
|
||||
hint = fn(command or "", window)
|
||||
except Exception:
|
||||
continue
|
||||
if hint:
|
||||
return hint
|
||||
return _EXIT_CODE_HINTS.get(exit_code)
|
||||
|
|
@ -3050,6 +3050,19 @@ def terminal_tool(
|
|||
# (e.g. grep=1 means "no matches", diff=1 means "files differ")
|
||||
exit_note = _interpret_exit_code(command, returncode)
|
||||
|
||||
# Output-pattern failure hints: map well-known error shapes
|
||||
# (command-not-found, ModuleNotFoundError, gh field drift,
|
||||
# merge conflicts, ...) to one short recovery hint so the model
|
||||
# fixes the root cause on the next call instead of spending
|
||||
# turns on re-diagnosis. See tools/terminal_hints.py.
|
||||
failure_hint = None
|
||||
if returncode != 0 and not exit_note:
|
||||
try:
|
||||
from tools.terminal_hints import annotate_failure
|
||||
failure_hint = annotate_failure(command, returncode, output)
|
||||
except Exception:
|
||||
failure_hint = None
|
||||
|
||||
result_dict = {
|
||||
"output": output,
|
||||
"exit_code": returncode,
|
||||
|
|
@ -3090,6 +3103,8 @@ def terminal_tool(
|
|||
result_dict["approval"] = approval_note
|
||||
if exit_note:
|
||||
result_dict["exit_code_meaning"] = exit_note
|
||||
if failure_hint:
|
||||
result_dict["hint"] = failure_hint
|
||||
if sudo_auth_failed:
|
||||
result_dict["sudo_auth_failed"] = True
|
||||
if sudo_cache_cleared:
|
||||
|
|
|
|||
Loading…
Reference in New Issue