From 6e041d524439b69ddfec73398816f8a7b01edfa0 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:13:29 -0700 Subject: [PATCH] =?UTF-8?q?feat(goals):=20quality=20gates=20=E2=80=94=20de?= =?UTF-8?q?terministic=20commands=20that=20must=20pass=20before=20/goal=20?= =?UTF-8?q?completes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /goal gate add attaches shell commands to the active goal. Gates run at turn boundary BEFORE the LLM judge: a failing gate skips the judge entirely and feeds its exit code + bounded output tail back as the continuation prompt, so the agent iterates against concrete evidence instead of a prose verdict. - Unchanged-workspace skip: a gate that failed on an identical workspace (git HEAD + status fingerprint) is not re-run — the recorded failure replays and the attempt count advances. - Bounded retries (default 3) + per-gate timeout (default 300s); exhaustion auto-pauses the goal like the turn budget does. - Gates persist in SessionDB.state_meta with the goal (survive /resume and compression rotation); pre-gate goal rows load unchanged. - /goal gate [list|add|remove|clear] on CLI + gateway; 'gate' added to the mid-run control-verb whitelist (gates only run at turn boundary, so editing the list mid-run is safe). Adapted from the quality-gate concept in Prime Intellect's Prime-Agent (--autonomous-gate). --- gateway/run.py | 6 +- gateway/slash_commands.py | 32 +++ hermes_cli/cli_commands_mixin.py | 45 ++- hermes_cli/commands.py | 2 +- hermes_cli/goals.py | 328 +++++++++++++++++++++- tests/hermes_cli/test_goal_gates.py | 224 +++++++++++++++ website/docs/user-guide/features/goals.md | 24 ++ 7 files changed, 656 insertions(+), 5 deletions(-) create mode 100644 tests/hermes_cli/test_goal_gates.py diff --git a/gateway/run.py b/gateway/run.py index b9628c43747fa..0fbd424d778a8 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -14257,11 +14257,13 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _goal_arg = (event.get_command_args() or "").strip().lower() _goal_verb = _goal_arg.split(None, 1)[0] if _goal_arg else "" # Exact-match control verbs (unchanged semantics), plus the - # wait/unwait barrier verbs which take a pid argument. + # wait/unwait barrier verbs which take a pid argument and the + # gate management verb (inspection/mutation of the gate list only — + # gates run at turn boundary, so editing them mid-run is safe). _is_control = ( not _goal_arg or _goal_arg in {"status", "pause", "resume", "clear", "stop", "done", "unwait"} - or _goal_verb == "wait" + or _goal_verb in {"wait", "gate"} ) if _is_control: return await self._handle_goal_command(event) diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index 7b87e435055cb..9ba1a940401ce 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -2680,6 +2680,38 @@ class GatewaySlashCommandsMixin: return "▶ Wait barrier cleared — goal loop resumes." return "No wait barrier set." + # /goal gate ... — manage deterministic quality gates. + if lower == "gate" or lower.startswith("gate "): + gate_arg = args[len("gate"):].strip() + gate_lower = gate_arg.lower() + if not gate_arg or gate_lower == "list": + return mgr.render_gates() + if gate_lower.startswith("add "): + command = gate_arg[len("add"):].strip() + try: + gate = mgr.add_gate(command) + except (RuntimeError, ValueError) as exc: + return f"/goal gate add: {exc}" + return ( + f"⚿ Gate added: $ {gate.command} " + f"({gate.max_retries} retries, {gate.timeout_seconds}s timeout). " + f"It must pass before the goal can complete." + ) + if gate_lower.startswith("remove ") or gate_lower.startswith("rm "): + idx_text = gate_arg.split(None, 1)[1].strip() + try: + removed = mgr.remove_gate(int(idx_text)) + except (RuntimeError, ValueError, IndexError) as exc: + return f"/goal gate remove: {exc}" + return f"✓ Gate removed: $ {removed}" + if gate_lower == "clear": + try: + prev = mgr.clear_gates() + except RuntimeError as exc: + return f"/goal gate clear: {exc}" + return f"✓ Cleared {prev} gate{'s' if prev != 1 else ''}." + return "Usage: /goal gate [list | add | remove | clear]" + # /goal draft → draft a structured completion contract, # then set it. The aux LLM call is sync; run it off the event loop. draft_contract_obj = None diff --git a/hermes_cli/cli_commands_mixin.py b/hermes_cli/cli_commands_mixin.py index 9f9071ea94ef0..8c4507d2ff356 100644 --- a/hermes_cli/cli_commands_mixin.py +++ b/hermes_cli/cli_commands_mixin.py @@ -2369,7 +2369,7 @@ class CLICommandsMixin: print() def _handle_goal_command(self, cmd: str) -> None: - """Dispatch /goal subcommands: set / draft / show / status / pause / resume / clear.""" + """Dispatch /goal subcommands: set / draft / show / gate / status / pause / resume / clear.""" from cli import _DIM, _RST, _cprint parts = (cmd or "").strip().split(None, 1) arg = parts[1].strip() if len(parts) > 1 else "" @@ -2466,6 +2466,49 @@ class CLICommandsMixin: _cprint(f" {_DIM}No wait barrier set.{_RST}") return + # /goal gate ... — manage deterministic quality gates. A gate is a + # shell command that must pass before the judge may declare the goal + # done; a failing gate's output becomes the continuation prompt. + if lower == "gate" or lower.startswith("gate "): + gate_arg = arg[len("gate"):].strip() + gate_lower = gate_arg.lower() + if not gate_arg or gate_lower == "list": + for line in mgr.render_gates().splitlines(): + _cprint(f" {line}") + return + if gate_lower.startswith("add "): + command = gate_arg[len("add"):].strip() + try: + gate = mgr.add_gate(command) + except (RuntimeError, ValueError) as exc: + _cprint(f" /goal gate add: {exc}") + return + _cprint( + f" ⚿ Gate added: $ {gate.command} " + f"({gate.max_retries} retries, {gate.timeout_seconds}s timeout). " + f"It must pass before the goal can complete." + ) + return + if gate_lower.startswith("remove ") or gate_lower.startswith("rm "): + idx_text = gate_arg.split(None, 1)[1].strip() + try: + removed = mgr.remove_gate(int(idx_text)) + except (RuntimeError, ValueError, IndexError) as exc: + _cprint(f" /goal gate remove: {exc}") + return + _cprint(f" ✓ Gate removed: $ {removed}") + return + if gate_lower == "clear": + try: + prev = mgr.clear_gates() + except RuntimeError as exc: + _cprint(f" /goal gate clear: {exc}") + return + _cprint(f" ✓ Cleared {prev} gate{'s' if prev != 1 else ''}.") + return + _cprint(" Usage: /goal gate [list | add | remove | clear]") + return + # Otherwise treat the arg as the goal text. Inline `field: value` # lines (verify:, constraints:, boundaries:, stop when:) are parsed # into a completion contract; the remaining prose is the headline. diff --git a/hermes_cli/commands.py b/hermes_cli/commands.py index a803ee925f30e..a183a8b1043f2 100644 --- a/hermes_cli/commands.py +++ b/hermes_cli/commands.py @@ -157,7 +157,7 @@ COMMAND_REGISTRY: list[CommandDef] = [ CommandDef("steer", "Inject a message after the next tool call without interrupting", "Session", args_hint="", busy_policy="dispatch", busy_handler="steer"), CommandDef("goal", "Set a standing goal Hermes works on across turns until achieved", "Session", - args_hint="[text | draft | show | pause | resume | clear | status | wait | unwait]", + args_hint="[text | draft | show | gate add | pause | resume | clear | status | wait | unwait]", busy_policy="dispatch", busy_handler="goal"), CommandDef("moa", "Run one prompt through the default Mixture of Agents preset, then restore your model", "Session", args_hint="", busy_policy="reject", busy_handler="moa"), diff --git a/hermes_cli/goals.py b/hermes_cli/goals.py index be99bbdc9e83e..4762710cd314b 100644 --- a/hermes_cli/goals.py +++ b/hermes_cli/goals.py @@ -29,9 +29,12 @@ Nothing in this module touches the agent's system prompt or toolset. from __future__ import annotations +import hashlib import json import logging +import os import re +import subprocess import time from dataclasses import dataclass, field, asdict from datetime import datetime, timezone @@ -72,6 +75,17 @@ DEFAULT_MAX_CONSECUTIVE_PARSE_FAILURES = 3 # run until the turn budget, wasting every turn on an unreachable judge. DEFAULT_MAX_CONSECUTIVE_TRANSPORT_FAILURES = 5 +# Quality gates: deterministic shell commands that must pass before the goal +# judge may declare the goal done. Defaults mirror the bounded-autonomy +# pattern (per-gate retry limit + timeout, bounded output fed back to the +# agent). A failed gate short-circuits the judge — its output IS the +# continuation prompt, so the agent works on concrete evidence instead of a +# vibe check. +DEFAULT_GATE_TIMEOUT_SECONDS = 300 +DEFAULT_GATE_MAX_RETRIES = 3 +# Bounded tail of a failed gate's combined stdout/stderr fed back to the agent. +_GATE_OUTPUT_TAIL_CHARS = 3000 + CONTINUATION_PROMPT_TEMPLATE = ( "[Continuing toward your standing goal]\n" @@ -114,6 +128,25 @@ CONTINUATION_PROMPT_WITH_SUBGOALS_TEMPLATE = ( ) +# Fed back when a quality gate fails: the gate's bounded output is the +# evidence the agent must repair against. Deterministic — no judge involved. +CONTINUATION_PROMPT_GATE_FAILED_TEMPLATE = ( + "[Continuing toward your standing goal — a quality gate failed]\n" + "Goal: {goal}\n\n" + "The quality gate command below must pass before this goal can be " + "declared done, and it just failed (attempt {attempt}/{max_retries}):\n" + " $ {command}\n" + "Exit code: {exit_code}\n" + "Output (tail):\n" + "```\n" + "{output}\n" + "```\n\n" + "Fix the underlying problem so this gate passes, then re-run it to " + "confirm. Do not declare the goal complete while any gate fails. If the " + "gate itself is wrong or cannot pass, say so clearly and stop." +) + + JUDGE_SYSTEM_PROMPT = ( "You are a strict judge evaluating whether an autonomous agent has " "achieved a user's stated goal. You receive the goal text, the agent's " @@ -385,6 +418,114 @@ def parse_contract(text: str) -> Tuple[str, GoalContract]: return headline, contract +# ────────────────────────────────────────────────────────────────────── +# Quality gates +# ────────────────────────────────────────────────────────────────────── + + +@dataclass +class GoalGate: + """A deterministic shell command that must pass before a goal can be done. + + Gates run at turn boundary BEFORE the LLM judge. A failing gate + short-circuits judging entirely: its bounded output becomes the + continuation prompt, so the agent iterates against concrete evidence. + Only when every gate passes does the judge get to decide DONE. + + ``attempts`` counts failed runs; when it exceeds ``max_retries`` the goal + auto-pauses (mirrors the turn-budget pause) instead of spinning. A gate + that failed on an unchanged workspace is not re-run — the recorded + failure is replayed and the attempt count advances, so a stuck agent + can't burn wall-clock re-running the same red suite. + """ + + command: str + timeout_seconds: int = DEFAULT_GATE_TIMEOUT_SECONDS + max_retries: int = DEFAULT_GATE_MAX_RETRIES + attempts: int = 0 + last_exit_code: Optional[int] = None + last_output_tail: str = "" + # Workspace fingerprint at the time of the last FAILED run — used to skip + # re-running an identical gate when nothing changed since it failed. + last_failed_fingerprint: str = "" + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: Optional[Dict[str, Any]]) -> "GoalGate": + if not isinstance(data, dict): + return cls(command="") + return cls( + command=str(data.get("command") or ""), + timeout_seconds=int(data.get("timeout_seconds", DEFAULT_GATE_TIMEOUT_SECONDS) or DEFAULT_GATE_TIMEOUT_SECONDS), + max_retries=int(data.get("max_retries", DEFAULT_GATE_MAX_RETRIES) or DEFAULT_GATE_MAX_RETRIES), + attempts=int(data.get("attempts", 0) or 0), + last_exit_code=(int(data["last_exit_code"]) if data.get("last_exit_code") is not None else None), + last_output_tail=str(data.get("last_output_tail") or ""), + last_failed_fingerprint=str(data.get("last_failed_fingerprint") or ""), + ) + + +def workspace_fingerprint(cwd: Optional[str] = None) -> str: + """Cheap workspace change fingerprint for unchanged-gate skip. + + Uses ``git status --porcelain`` + ``git rev-parse HEAD`` when inside a git + repo (covers tracked edits, stages, and commits). Outside git, returns + an empty string — an empty fingerprint never matches, so gates simply + always re-run (safe fallback, no behavior regression for non-repo work). + """ + workdir = cwd or os.getcwd() + try: + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, text=True, timeout=10, cwd=workdir, + ) + if head.returncode != 0: + return "" + status = subprocess.run( + ["git", "status", "--porcelain"], + capture_output=True, text=True, timeout=30, cwd=workdir, + ) + if status.returncode != 0: + return "" + blob = head.stdout.strip() + "\n" + status.stdout + return hashlib.sha256(blob.encode("utf-8", "replace")).hexdigest() + except Exception: + return "" + + +def run_gate(gate: GoalGate, *, cwd: Optional[str] = None) -> Tuple[bool, int, str]: + """Run one gate command. Returns ``(passed, exit_code, output_tail)``. + + The command runs through the shell in ``cwd`` (default: process cwd) with + a hard timeout; on timeout the process is killed and treated as failed + with exit code -1. Output is the combined stdout+stderr tail, bounded to + ``_GATE_OUTPUT_TAIL_CHARS``. + """ + try: + proc = subprocess.run( + gate.command, + shell=True, + capture_output=True, + text=True, + timeout=max(1, int(gate.timeout_seconds)), + cwd=cwd or None, + ) + combined = (proc.stdout or "") + (("\n" + proc.stderr) if proc.stderr else "") + tail = combined[-_GATE_OUTPUT_TAIL_CHARS:] + return proc.returncode == 0, proc.returncode, tail + except subprocess.TimeoutExpired as exc: + out = "" + for chunk in (exc.stdout, exc.stderr): + if chunk: + out += chunk if isinstance(chunk, str) else chunk.decode("utf-8", "replace") + tail = (out + f"\n[gate timed out after {gate.timeout_seconds}s]")[-_GATE_OUTPUT_TAIL_CHARS:] + return False, -1, tail + except Exception as exc: + return False, -1, f"[gate could not run: {type(exc).__name__}: {exc}]" + + # ────────────────────────────────────────────────────────────────────── # Dataclass # ────────────────────────────────────────────────────────────────────── @@ -442,6 +583,10 @@ class GoalState: # constraints / boundaries / stop_when). Empty by default; a goal with # no contract behaves exactly like the original free-form goal. contract: GoalContract = field(default_factory=GoalContract) + # Quality gates (/goal gate add ): deterministic shell commands that + # must ALL pass before the judge may declare the goal done. Empty by + # default — a goal with no gates behaves exactly as before. + gates: List[GoalGate] = field(default_factory=list) def to_json(self) -> str: data = asdict(self) @@ -474,6 +619,11 @@ class GoalState: waiting_reason=data.get("waiting_reason"), waiting_since=float(data.get("waiting_since", 0.0) or 0.0), contract=GoalContract.from_dict(data.get("contract")), + gates=[ + GoalGate.from_dict(g) + for g in (data.get("gates") or []) + if isinstance(g, dict) and str(g.get("command") or "").strip() + ], ) # --- contract helpers ------------------------------------------------- @@ -1120,7 +1270,8 @@ class GoalManager: turns = f"{s.turns_used}/{s.max_turns} turns" sub = f", {len(s.subgoals)} subgoal{'s' if len(s.subgoals) != 1 else ''}" if s.subgoals else "" con = ", contract" if self.has_contract() else "" - meta = f"{turns}{sub}{con}" + gat = f", {len(s.gates)} gate{'s' if len(s.gates) != 1 else ''}" if s.gates else "" + meta = f"{turns}{sub}{con}{gat}" if s.status == "active": if s.waiting_on_session and _session_waiting(s.waiting_on_session): wr = s.waiting_reason or f"session {s.waiting_on_session}" @@ -1262,6 +1413,154 @@ class GoalManager: return "(no subgoals — use /subgoal to add criteria)" return self._state.render_subgoals_block() + # --- /goal gate quality gates --------------------------------------- + + def add_gate( + self, + command: str, + *, + timeout_seconds: Optional[int] = None, + max_retries: Optional[int] = None, + ) -> GoalGate: + """Append a quality-gate command to the active goal. + + Requires ``has_goal()``; raises ``RuntimeError`` otherwise. Returns + the created gate so callers can echo it back. + """ + if self._state is None or not self.has_goal(): + raise RuntimeError("no active goal") + command = (command or "").strip() + if not command: + raise ValueError("gate command is empty") + gate = GoalGate( + command=command, + timeout_seconds=int(timeout_seconds) if timeout_seconds else DEFAULT_GATE_TIMEOUT_SECONDS, + max_retries=int(max_retries) if max_retries else DEFAULT_GATE_MAX_RETRIES, + ) + self._state.gates.append(gate) + save_goal(self.session_id, self._state) + return gate + + def remove_gate(self, index_1based: int) -> str: + """Remove a gate by 1-based index. Returns the removed command.""" + if self._state is None or not self.has_goal(): + raise RuntimeError("no active goal") + idx = int(index_1based) - 1 + if idx < 0 or idx >= len(self._state.gates): + raise IndexError(f"index out of range (1..{len(self._state.gates)})") + removed = self._state.gates.pop(idx) + save_goal(self.session_id, self._state) + return removed.command + + def clear_gates(self) -> int: + """Remove all gates. Returns the previous count.""" + if self._state is None or not self.has_goal(): + raise RuntimeError("no active goal") + prev = len(self._state.gates) + self._state.gates = [] + save_goal(self.session_id, self._state) + return prev + + def render_gates(self) -> str: + """Public helper for the /goal gate slash command.""" + if self._state is None: + return "(no active goal)" + if not self._state.gates: + return "(no quality gates — use /goal gate add to require one)" + lines = [] + for i, g in enumerate(self._state.gates, start=1): + status = "" + if g.last_exit_code is not None: + status = " ✓ passing" if g.last_exit_code == 0 else ( + f" ✗ failing (exit {g.last_exit_code}, attempt {g.attempts}/{g.max_retries})" + ) + lines.append(f"- {i}. $ {g.command}{status}") + return "\n".join(lines) + + def _check_gates(self) -> Optional[Dict[str, Any]]: + """Run quality gates in order; return a decision dict on failure. + + Returns ``None`` when there are no gates or every gate passes — + the caller then proceeds to the LLM judge. On the first failing + gate, returns a full ``evaluate_after_turn``-shaped decision dict: + either a continuation carrying the gate's output (attempts left) + or an auto-pause (retries exhausted). + + An unchanged workspace since the last failure of the same gate is + NOT re-run — the recorded failure is replayed and the attempt count + advances, so a stalled agent can't spin re-running an identical red + suite (mirrors Prime-Agent's unchanged-gate rule). + """ + state = self._state + if state is None or not state.gates: + return None + + fingerprint = workspace_fingerprint() + for gate in state.gates: + unchanged = ( + bool(fingerprint) + and gate.last_exit_code not in (None, 0) + and gate.last_failed_fingerprint == fingerprint + ) + if unchanged: + passed, exit_code, tail = False, int(gate.last_exit_code or -1), gate.last_output_tail + else: + passed, exit_code, tail = run_gate(gate) + gate.last_exit_code = exit_code + gate.last_output_tail = tail + if passed: + gate.attempts = 0 + gate.last_failed_fingerprint = "" + continue + + gate.attempts += 1 + gate.last_failed_fingerprint = fingerprint + skipped_note = " (workspace unchanged since last failure — not re-run)" if unchanged else "" + + if gate.attempts > gate.max_retries: + state.status = "paused" + state.paused_reason = ( + f"quality gate exhausted {gate.attempts - 1} retries: $ {gate.command}" + ) + save_goal(self.session_id, state) + return { + "status": "paused", + "should_continue": False, + "continuation_prompt": None, + "verdict": "gate_failed", + "reason": f"gate exhausted retries: $ {gate.command}", + "message": ( + f"⏸ Goal paused — quality gate still failing after " + f"{gate.max_retries} retries: $ {gate.command} " + f"(exit {exit_code}). Fix it manually or /goal gate remove it, " + f"then /goal resume." + ), + } + + save_goal(self.session_id, state) + prompt = CONTINUATION_PROMPT_GATE_FAILED_TEMPLATE.format( + goal=state.goal, + command=gate.command, + exit_code=exit_code, + attempt=gate.attempts, + max_retries=gate.max_retries, + output=tail or "(no output)", + ) + return { + "status": "active", + "should_continue": True, + "continuation_prompt": prompt, + "verdict": "gate_failed", + "reason": f"gate failed (exit {exit_code}): $ {gate.command}", + "message": ( + f"✗ Quality gate failed ({state.turns_used}/{state.max_turns} turns, " + f"attempt {gate.attempts}/{gate.max_retries}){skipped_note}: $ {gate.command}" + ), + } + + save_goal(self.session_id, state) + return None + # --- /goal wait barrier ------------------------------------------- def wait_on(self, pid: int, reason: str = "") -> GoalState: @@ -1443,6 +1742,30 @@ class GoalManager: state.turns_used += 1 state.last_turn_at = time.time() + # Quality gates run BEFORE the LLM judge: a failing gate is + # deterministic evidence the goal is not done, so the judge call is + # skipped entirely and the gate's output drives the next turn. Gate + # continuations respect the same turn budget as judge continuations. + gate_decision = self._check_gates() + if gate_decision is not None: + if gate_decision.get("should_continue") and state.turns_used >= state.max_turns: + state.status = "paused" + state.paused_reason = f"turn budget exhausted ({state.turns_used}/{state.max_turns})" + save_goal(self.session_id, state) + return { + "status": "paused", + "should_continue": False, + "continuation_prompt": None, + "verdict": "gate_failed", + "reason": gate_decision.get("reason", ""), + "message": ( + f"⏸ Goal paused — {state.turns_used}/{state.max_turns} turns used " + f"(a quality gate is still failing). " + "Use /goal resume to keep going, or /goal clear to stop." + ), + } + return gate_decision + verdict, reason, parse_failed, wait_directive, transport_failed = judge_goal( state.goal, last_response, @@ -1785,9 +2108,12 @@ def run_kanban_goal_loop( __all__ = [ "GoalState", "GoalContract", + "GoalGate", "GoalManager", "parse_contract", "draft_contract", + "run_gate", + "workspace_fingerprint", "CONTINUATION_PROMPT_TEMPLATE", "CONTINUATION_PROMPT_WITH_SUBGOALS_TEMPLATE", "CONTINUATION_PROMPT_WITH_CONTRACT_TEMPLATE", diff --git a/tests/hermes_cli/test_goal_gates.py b/tests/hermes_cli/test_goal_gates.py new file mode 100644 index 0000000000000..d90a63e0fb13d --- /dev/null +++ b/tests/hermes_cli/test_goal_gates.py @@ -0,0 +1,224 @@ +"""Tests for /goal quality gates (GoalGate, run_gate, GoalManager gate flow).""" + +import json +import time +from unittest.mock import patch + +import pytest + +from hermes_cli.goals import ( + DEFAULT_GATE_MAX_RETRIES, + DEFAULT_GATE_TIMEOUT_SECONDS, + GoalGate, + GoalManager, + GoalState, + run_gate, + save_goal, + load_goal, +) + + +# ────────────────────────────────────────────────────────────────────── +# GoalGate serialization +# ────────────────────────────────────────────────────────────────────── + + +def test_gate_roundtrip_through_goalstate_json(): + state = GoalState(goal="ship it", status="active") + state.gates.append(GoalGate(command="echo ok", timeout_seconds=42, max_retries=7)) + raw = state.to_json() + loaded = GoalState.from_json(raw) + assert len(loaded.gates) == 1 + g = loaded.gates[0] + assert g.command == "echo ok" + assert g.timeout_seconds == 42 + assert g.max_retries == 7 + assert g.attempts == 0 + + +def test_gate_from_dict_defaults_and_garbage(): + g = GoalGate.from_dict({"command": "true"}) + assert g.timeout_seconds == DEFAULT_GATE_TIMEOUT_SECONDS + assert g.max_retries == DEFAULT_GATE_MAX_RETRIES + assert GoalGate.from_dict(None).command == "" + assert GoalGate.from_dict("nonsense").command == "" + + +def test_old_state_rows_without_gates_load_clean(): + """Backwards compatibility: pre-gates state_meta rows load with no gates.""" + old = {"goal": "legacy", "status": "active"} + state = GoalState.from_json(json.dumps(old)) + assert state.gates == [] + + +# ────────────────────────────────────────────────────────────────────── +# run_gate +# ────────────────────────────────────────────────────────────────────── + + +def test_run_gate_pass(): + passed, code, out = run_gate(GoalGate(command="echo hello")) + assert passed is True + assert code == 0 + assert "hello" in out + + +def test_run_gate_fail_captures_output(): + passed, code, out = run_gate(GoalGate(command="echo broken >&2; exit 3")) + assert passed is False + assert code == 3 + assert "broken" in out + + +def test_run_gate_timeout(): + passed, code, out = run_gate(GoalGate(command="sleep 5", timeout_seconds=1)) + assert passed is False + assert code == -1 + assert "timed out" in out + + +# ────────────────────────────────────────────────────────────────────── +# GoalManager gate management +# ────────────────────────────────────────────────────────────────────── + + +def _mgr_with_goal(session_id="gate-test-sid"): + mgr = GoalManager(session_id=session_id) + mgr.set("test goal") + return mgr + + +def test_add_remove_clear_gates(): + mgr = _mgr_with_goal("gate-mgmt-sid") + mgr.add_gate("echo one") + mgr.add_gate("echo two") + assert len(mgr.state.gates) == 2 + assert "echo one" in mgr.render_gates() + + removed = mgr.remove_gate(1) + assert removed == "echo one" + assert len(mgr.state.gates) == 1 + + assert mgr.clear_gates() == 1 + assert mgr.state.gates == [] + + +def test_add_gate_requires_active_goal(): + mgr = GoalManager(session_id="gate-nogoal-sid") + with pytest.raises(RuntimeError): + mgr.add_gate("echo nope") + + +def test_gates_persist_and_reload(): + mgr = _mgr_with_goal("gate-persist-sid") + mgr.add_gate("echo persisted") + reloaded = GoalManager(session_id="gate-persist-sid") + assert len(reloaded.state.gates) == 1 + assert reloaded.state.gates[0].command == "echo persisted" + + +def test_status_line_mentions_gates(): + mgr = _mgr_with_goal("gate-status-sid") + mgr.add_gate("echo g") + assert "1 gate" in mgr.status_line() + + +# ────────────────────────────────────────────────────────────────────── +# evaluate_after_turn integration +# ────────────────────────────────────────────────────────────────────── + + +def test_failing_gate_short_circuits_judge(): + mgr = _mgr_with_goal("gate-fail-sid") + mgr.add_gate("exit 5") + with patch("hermes_cli.goals.judge_goal") as mock_judge, \ + patch("hermes_cli.goals.workspace_fingerprint", return_value=""): + decision = mgr.evaluate_after_turn("I think it's done!") + mock_judge.assert_not_called() + assert decision["verdict"] == "gate_failed" + assert decision["should_continue"] is True + assert "exit 5" in decision["continuation_prompt"] + assert "quality gate" in decision["continuation_prompt"].lower() + + +def test_passing_gates_fall_through_to_judge(): + mgr = _mgr_with_goal("gate-pass-sid") + mgr.add_gate("true") + with patch( + "hermes_cli.goals.judge_goal", + return_value=("done", "all good", False, None, False), + ) as mock_judge: + decision = mgr.evaluate_after_turn("finished") + mock_judge.assert_called_once() + assert decision["verdict"] == "done" + # Passing run resets attempt bookkeeping. + assert mgr.state.gates[0].attempts == 0 + assert mgr.state.gates[0].last_exit_code == 0 + + +def test_gate_retry_exhaustion_pauses_goal(): + mgr = _mgr_with_goal("gate-exhaust-sid") + mgr.add_gate("exit 1") + mgr.state.gates[0].max_retries = 2 + with patch("hermes_cli.goals.judge_goal") as mock_judge, \ + patch("hermes_cli.goals.workspace_fingerprint", return_value=""): + d1 = mgr.evaluate_after_turn("attempt one") + d2 = mgr.evaluate_after_turn("attempt two") + d3 = mgr.evaluate_after_turn("attempt three") + mock_judge.assert_not_called() + assert d1["should_continue"] is True + assert d2["should_continue"] is True + assert d3["status"] == "paused" + assert d3["should_continue"] is False + assert mgr.state.status == "paused" + assert "gate" in (mgr.state.paused_reason or "") + + +def test_unchanged_workspace_skips_rerun(): + mgr = _mgr_with_goal("gate-unchanged-sid") + mgr.add_gate("exit 1") + with patch("hermes_cli.goals.workspace_fingerprint", return_value="fp-1"), \ + patch("hermes_cli.goals.judge_goal"): + mgr.evaluate_after_turn("turn 1") + # Second turn, same fingerprint — run_gate must NOT run again. + with patch("hermes_cli.goals.run_gate") as mock_run: + d2 = mgr.evaluate_after_turn("turn 2") + mock_run.assert_not_called() + assert d2["verdict"] == "gate_failed" + assert "unchanged" in d2["message"] + + +def test_changed_workspace_reruns_gate(): + mgr = _mgr_with_goal("gate-changed-sid") + mgr.add_gate("exit 1") + with patch("hermes_cli.goals.judge_goal"): + with patch("hermes_cli.goals.workspace_fingerprint", return_value="fp-1"): + mgr.evaluate_after_turn("turn 1") + with patch("hermes_cli.goals.workspace_fingerprint", return_value="fp-2"), \ + patch("hermes_cli.goals.run_gate", return_value=(False, 1, "still red")) as mock_run: + mgr.evaluate_after_turn("turn 2") + mock_run.assert_called_once() + + +def test_gate_continuation_respects_turn_budget(): + mgr = GoalManager(session_id="gate-budget-sid", default_max_turns=1) + mgr.set("budget goal") + mgr.add_gate("exit 1") + with patch("hermes_cli.goals.judge_goal"), \ + patch("hermes_cli.goals.workspace_fingerprint", return_value=""): + decision = mgr.evaluate_after_turn("only turn") + assert decision["status"] == "paused" + assert decision["should_continue"] is False + assert "turns used" in decision["message"] + + +def test_no_gates_behaves_exactly_as_before(): + mgr = _mgr_with_goal("gate-none-sid") + with patch( + "hermes_cli.goals.judge_goal", + return_value=("continue", "keep going", False, None, False), + ) as mock_judge: + decision = mgr.evaluate_after_turn("wip") + mock_judge.assert_called_once() + assert decision["verdict"] == "continue" + assert decision["should_continue"] is True diff --git a/website/docs/user-guide/features/goals.md b/website/docs/user-guide/features/goals.md index bbb1306c3b8d9..b4a9f31585840 100644 --- a/website/docs/user-guide/features/goals.md +++ b/website/docs/user-guide/features/goals.md @@ -66,6 +66,10 @@ What you'll see: | `/goal clear` | Drop the goal entirely. | | `/goal wait [reason]` | Park the loop on a background process — it stops re-poking the agent every turn while the process runs, and auto-resumes when it exits. | | `/goal unwait` | Drop the wait barrier and resume the loop immediately. | +| `/goal gate add ` | Add a **quality gate**: a shell command that must pass before the goal can be judged done. See [Quality gates](#quality-gates). | +| `/goal gate` or `/goal gate list` | List the goal's gates and their pass/fail state. | +| `/goal gate remove ` | Remove the Nth gate (1-based). | +| `/goal gate clear` | Remove all gates. | Works identically on the CLI and every gateway platform (Telegram, Discord, Slack, Matrix, Signal, WhatsApp, SMS, iMessage, Webhook, API server, and the web dashboard). @@ -124,6 +128,26 @@ Subgoals are persisted alongside the goal in `SessionDB.state_meta`, so they sur Use this when you start a loop ("fix the failing tests") and notice partway through that you also want it to "and add a regression test for the bug you just patched" — `/subgoal add a regression test` tightens the success criteria without breaking the running loop. +## Quality gates + +A completion contract makes the judge stricter, but the judge is still an LLM reading prose. A **quality gate** is stronger: a deterministic shell command that must exit 0 before the goal can complete at all. Inspired by Prime-Agent's bounded autonomous mode (`--autonomous-gate`). + +``` +/goal Fix the flaky session tests +/goal gate add scripts/run_tests.sh tests/hermes_cli/test_goals.py +``` + +How it works, each turn: + +1. **Gates run before the judge.** If any gate fails, the judge is *not called* — a red gate is deterministic evidence the goal isn't done. The gate's exit code and output tail (last ~3 KB) become the continuation prompt, so the agent iterates against the actual failure instead of a vibe. +2. **All gates pass → normal judging.** The LLM judge then decides done/continue/wait exactly as before. +3. **Unchanged workspace → no re-run.** If a gate failed and nothing changed in the workspace since (tracked via a git fingerprint of HEAD + working-tree status), the gate is not re-run — the recorded failure is replayed and the attempt count advances. A stuck agent can't burn wall-clock re-running an identical red suite. Outside a git repo, gates simply always re-run. +4. **Retries are bounded.** Each gate defaults to 3 retries and a 5-minute timeout. When a gate exhausts its retries the goal auto-pauses (like the turn budget) with a message telling you to fix it manually, remove the gate, or `/goal resume`. + +Gates persist with the goal in `SessionDB.state_meta` (they survive `/resume` and context compression), and gate management (`/goal gate …`) is safe mid-run on the gateway — gates only run at turn boundary. + +Gates and contracts compose: use a contract to shape *what the agent aims for*, and gates to make *"done" mechanically checkable*. When both are set, gates run first. + ## Parking on a background process: automatic, with a manual override Some goals are gated on something that takes minutes and runs on its own — CI on a pushed PR, a long build, a test matrix, a deploy, a rate-limit cooldown. Without help, the goal loop would re-poke the agent every turn into "is it done yet?" busy-work while it waits.