diff --git a/agent/verification_evidence.py b/agent/verification_evidence.py index 2c8d1f85efa9e..3d5af6b81c25d 100644 --- a/agent/verification_evidence.py +++ b/agent/verification_evidence.py @@ -477,7 +477,56 @@ def record_terminal_result( ) if evidence is None: return None + return _insert_evidence(evidence) + +def record_verify_run( + *, + root: str | Path, + session_id: str | None = None, + ok: bool, + command: str = "hermes verify", + scope: str = "full", + output: str = "", +) -> Optional[dict[str, Any]]: + """Record a completed ``hermes verify`` run as verification evidence. + + Explicit CLI-side write: unlike :func:`record_terminal_result` there is + nothing to classify — the caller (the ``hermes verify`` command) already + knows the run was a verification pass and whether it succeeded. A passing + run marks the workspace ``passed`` for the verify-on-stop guard exactly + like a passing canonical test command would; a failing run records the + failure so the guard keeps asking for a fix. + + ``root`` is re-resolved through :func:`agent.coding_context.project_facts_for` + so the recorded workspace root matches what :func:`verification_status` + derives when the stop guard later looks the evidence up. + """ + try: + from agent.coding_context import project_facts_for + + facts = project_facts_for(root) + except Exception: + facts = None + + resolved = str(Path(root).resolve()) + evidence = VerificationEvidence( + command=command, + canonical_command="hermes verify", + kind="verify", + scope=scope if scope in {"full", "targeted"} else "full", + status="passed" if ok else "failed", + exit_code=0 if ok else 1, + cwd=resolved, + root=str((facts or {}).get("root") or resolved), + session_id=str(session_id or "default"), + output_summary=_summarize_output(output), + ) + return _insert_evidence(evidence) + + +def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]: + """Insert a classified evidence row and repoint the workspace state.""" created_at = _utc_now() with _DB_LOCK: with _transaction() as conn: diff --git a/agent/verification_stop.py b/agent/verification_stop.py index dc66944883c9d..2a47c4d13a963 100644 --- a/agent/verification_stop.py +++ b/agent/verification_stop.py @@ -183,6 +183,30 @@ def _format_changed_paths(paths: list[str]) -> str: return "\n".join(lines) +def _workspace_has_runnable_recipe(root: Any) -> bool: + """Whether the workspace has a runtime verify recipe ``hermes verify`` can run. + + True when a saved ``.hermes/environment.json`` manifest exists, or when + cheap static detection (:func:`agent.verify.recipes.detect_recipe`) finds a + recipe with a start command. Deliberately fail-silent and cheap — this only + decorates the nudge text; it must never break or slow the nudge path. + """ + if not root: + return False + try: + root_path = Path(str(root)) + from agent.verify.environment import manifest_path + + if manifest_path(root_path).is_file(): + return True + from agent.verify.recipes import detect_recipe + + recipe = detect_recipe(root_path) + return bool(recipe is not None and recipe.start) + except Exception: + return False + + def _status_detail(status: dict[str, Any]) -> str: state = str(status.get("status") or "unverified") evidence = status.get("evidence") if isinstance(status.get("evidence"), dict) else None @@ -248,16 +272,31 @@ def build_verify_on_stop_nudge( + (", ..." if len(verify_commands) > 3 else "") + "), read any failure, repair the code, and summarize what passed." ) + if _workspace_has_runnable_recipe(facts.get("root")): + command_instruction += ( + " For a full check including a runtime boot (build + test + " + "start + readiness), prefer `hermes verify --json` — a passing " + "run records verification evidence for this workspace." + ) else: temp_dir = os.path.realpath(tempfile.gettempdir()) - command_instruction = ( - "No canonical test/lint/build command was detected. Create a focused " - f"temporary verification script under `{temp_dir}` using an OS-safe " - "`tempfile` path with a `hermes-verify-` filename prefix, run it " - "against the changed behavior, clean it up when possible, and " - "summarize it explicitly as ad-hoc verification rather than suite " - "green." - ) + if _workspace_has_runnable_recipe(facts.get("root")): + command_instruction = ( + "No canonical test/lint/build command was detected, but the " + "project has a runnable verification recipe. Run `hermes verify " + "--json` (detect -> build -> test -> boot -> readiness poll); a " + "passing run records verification evidence for this workspace. " + "Read any failure, repair the code, and summarize what passed." + ) + else: + command_instruction = ( + "No canonical test/lint/build command was detected. Create a focused " + f"temporary verification script under `{temp_dir}` using an OS-safe " + "`tempfile` path with a `hermes-verify-` filename prefix, run it " + "against the changed behavior, clean it up when possible, and " + "summarize it explicitly as ad-hoc verification rather than suite " + "green." + ) return ( "[System: You edited code in this turn, but the workspace does not have " diff --git a/agent/verify/recipes.py b/agent/verify/recipes.py index a500742073b42..66578cb4fc2c1 100644 --- a/agent/verify/recipes.py +++ b/agent/verify/recipes.py @@ -5,6 +5,21 @@ Mirrors grok's detection order and command choices: Node (lockfile-based package-manager choice + framework detection), Python (Django / FastAPI / generic, with uv/poetry/pipenv awareness), Go, Rust, Java (Maven/Gradle), Makefile fallback, plus a docker-compose recipe. + +Layer ownership vs :mod:`agent.coding_context`: + +- ``agent.coding_context.detect_project_facts`` owns the *cheap prompt-time + facts* — manifests, package managers, and test/lint/build verify commands + surfaced in the system-prompt workspace snapshot, the verify-on-stop nudge, + and the desktop verify UI. Its output must stay byte-stable and fast; do + not push runtime detection into it. +- This module owns the *deep runtime recipe* — framework identification, + bootstrap/build/test command inference, and crucially the start command, + port, and readiness path that let ``hermes verify`` boot the app and prove + it serves HTTP. The ``hermes verify`` CLI merges any project-facts verify + commands the recipe missed into its test list (see + ``hermes_cli.verify_cmd._merge_project_facts_commands``) so the two layers + extend rather than contradict each other. """ from __future__ import annotations diff --git a/hermes_cli/verify_cmd.py b/hermes_cli/verify_cmd.py index 612e1a010546e..aae05029e39de 100644 --- a/hermes_cli/verify_cmd.py +++ b/hermes_cli/verify_cmd.py @@ -4,11 +4,16 @@ Scoped port of superagent-ai/grok-cli's verify subsystem entrypoint. Statically detects the project kind (or loads the saved manifest at ``.hermes/environment.json``), then runs bootstrap/build/test phases and an optional background start + readiness poll, printing an evidence summary. + +Completed runs are recorded into the coding verification evidence ledger +(:mod:`agent.verification_evidence`), so a passing ``hermes verify`` satisfies +the verify-on-stop guard the same way a passing canonical test command does. """ from __future__ import annotations import json +import os import sys from pathlib import Path @@ -38,6 +43,9 @@ def run_verify_command(args) -> int: print(message, file=sys.stderr) return 1 + if source == "detected": + _merge_project_facts_commands(root, recipe) + if args.port: recipe.port = args.port @@ -65,6 +73,8 @@ def run_verify_command(args) -> int: port_override=args.port, ) + _record_evidence(root, recipe, result, partial=bool(phases or args.skip_start)) + if args.json: payload = result.to_dict() payload["source"] = source @@ -75,6 +85,69 @@ def run_verify_command(args) -> int: return 0 if result.ok else 1 +def _merge_project_facts_commands(root: Path, recipe) -> None: + """Fold ``detect_project_facts`` verify commands into a detected recipe. + + Layer ownership: ``agent.coding_context`` owns the cheap prompt-time facts + (test/lint/build commands surfaced in the workspace snapshot and the + verify-on-stop nudge); ``agent.verify.recipes`` owns the deep runtime + recipe (framework, start command, port, readiness). When the two disagree + the runtime recipe must not *lose* commands the prompt layer already + promised the model — e.g. ``scripts/run_tests.sh`` or a ``pytest`` config + the recipe detector doesn't know about — so any project-facts verify + command not already covered is appended to the recipe's test list. + + Never applied to a saved manifest (the user-edited manifest is the source + of truth) and never raises: this is a best-effort union. + """ + try: + from agent.coding_context import detect_project_facts + + facts_commands = list(detect_project_facts(root).verify_commands) + except Exception: + return + existing = {c.strip() for c in (*recipe.bootstrap, *recipe.build, *recipe.test) if c} + for command in facts_commands: + command = command.strip() + if command and command not in existing: + recipe.test.append(command) + existing.add(command) + + +def _record_evidence(root: Path, recipe, result, *, partial: bool) -> None: + """Record the completed run into the verification evidence ledger. + + Best-effort and fail-silent: a ledger problem must never change the CLI's + exit code or output. ``partial`` (an explicit ``--phase`` subset or + ``--skip-start``) downgrades the scope to ``targeted`` so a partial pass + is never presented as a full workspace green. + """ + try: + from agent.verification_evidence import record_verify_run + + tails: list[str] = [] + for p in result.phases: + if p.output_tail: + tails.append(f"[{p.phase}] {p.command}\n{p.output_tail}") + if result.readiness is not None: + r = result.readiness + readiness_line = ( + f"[start] {recipe.start} -> " + + (f"ready (HTTP {r.status_code})" if r.ready else f"not ready ({r.error or 'timeout'})") + ) + tails.append(readiness_line) + record_verify_run( + root=root, + session_id=os.environ.get("HERMES_SESSION_ID"), + ok=result.ok, + command="hermes verify", + scope="targeted" if partial else "full", + output="\n".join(tails), + ) + except Exception: + pass + + def _print_human_report(recipe, source, result) -> None: print(f"Recipe: {recipe.name} ({recipe.kind}) — source: {source}") print() diff --git a/tests/verify/test_ledger_and_nudge_integration.py b/tests/verify/test_ledger_and_nudge_integration.py new file mode 100644 index 0000000000000..cb9654e533137 --- /dev/null +++ b/tests/verify/test_ledger_and_nudge_integration.py @@ -0,0 +1,232 @@ +"""Integration of the verify subsystem with the existing verification stack. + +Covers the closed loop the rescoped PR is about: + +- ``hermes verify`` records into the evidence ledger (pass and fail), +- a passing run satisfies the verify-on-stop guard, +- the verify-on-stop nudge names ``hermes verify --json`` when the workspace + has a runnable recipe (start command or saved manifest), +- the CLI's detect path merges ``detect_project_facts`` verify commands the + recipe missed. +""" + +import argparse +import json + +import pytest + +from agent.verification_evidence import ( + mark_workspace_edited, + record_verify_run, + verification_status, +) +from agent.verification_stop import build_verify_on_stop_nudge +from hermes_cli.verify_cmd import run_verify_command + + +def make_args(path, **overrides): + defaults = dict( + path=str(path), + detect_only=False, + save=False, + skip_start=False, + phase=None, + port=None, + timeout=60.0, + ready_timeout=5.0, + json=True, + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + +@pytest.fixture +def hermes_home(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes-home")) + monkeypatch.delenv("HERMES_SESSION_ID", raising=False) + return tmp_path + + +def _workspace(tmp_path, *, scripts=None, manifest_recipe=None): + """A marker-rooted workspace (package.json) with an optional saved recipe.""" + project = tmp_path / "project" + project.mkdir() + (project / "package.json").write_text( + json.dumps({"scripts": scripts} if scripts else {}), encoding="utf-8" + ) + if manifest_recipe is not None: + hermes_dir = project / ".hermes" + hermes_dir.mkdir() + (hermes_dir / "environment.json").write_text( + json.dumps({"version": 1, "recipe": manifest_recipe}), encoding="utf-8" + ) + return project + + +# --------------------------------------------------------------------------- +# ledger recording +# --------------------------------------------------------------------------- + + +def test_record_verify_run_marks_workspace_passed(hermes_home): + project = _workspace(hermes_home) + event = record_verify_run(root=project, session_id="s1", ok=True, output="all green") + assert event is not None + assert event["status"] == "passed" + assert event["kind"] == "verify" + status = verification_status(session_id="s1", cwd=project) + assert status["status"] == "passed" + assert status["evidence"]["canonical_command"] == "hermes verify" + + +def test_record_verify_run_records_failure(hermes_home): + project = _workspace(hermes_home) + record_verify_run(root=project, session_id="s1", ok=False, output="boom") + status = verification_status(session_id="s1", cwd=project) + assert status["status"] == "failed" + + +def test_cli_passing_run_writes_ledger_evidence(hermes_home, capsys): + project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]}) + code = run_verify_command(make_args(project)) + assert code == 0 + assert json.loads(capsys.readouterr().out)["ok"] is True + status = verification_status(session_id=None, cwd=project) + assert status["status"] == "passed" + assert status["evidence"]["scope"] == "full" + + +def test_cli_failing_run_writes_failed_evidence(hermes_home, capsys): + project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["false"]}) + code = run_verify_command(make_args(project)) + assert code == 1 + status = verification_status(session_id=None, cwd=project) + assert status["status"] == "failed" + + +def test_cli_partial_run_records_targeted_scope(hermes_home, capsys): + # --skip-start / --phase subsets must never present as full workspace green. + project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]}) + code = run_verify_command(make_args(project, skip_start=True)) + assert code == 0 + status = verification_status(session_id=None, cwd=project) + assert status["evidence"]["scope"] == "targeted" + + +def test_cli_run_uses_hermes_session_id_env(hermes_home, capsys, monkeypatch): + monkeypatch.setenv("HERMES_SESSION_ID", "sess-42") + project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]}) + run_verify_command(make_args(project)) + assert verification_status(session_id="sess-42", cwd=project)["status"] == "passed" + + +# --------------------------------------------------------------------------- +# closed loop: edit -> stop guard nudge -> hermes verify -> guard satisfied +# --------------------------------------------------------------------------- + + +def test_passing_verify_run_satisfies_stop_guard(hermes_home, capsys): + project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]}) + changed = str(project / "src" / "app.ts") + mark_workspace_edited(session_id="default", cwd=project, paths=[changed]) + assert build_verify_on_stop_nudge(session_id="default", changed_paths=[changed]) is not None + + assert run_verify_command(make_args(project)) == 0 + + assert build_verify_on_stop_nudge(session_id="default", changed_paths=[changed]) is None + + +# --------------------------------------------------------------------------- +# nudge wording: recipe-aware `hermes verify --json` suggestion +# --------------------------------------------------------------------------- + + +def test_nudge_mentions_hermes_verify_when_recipe_has_start(hermes_home): + project = _workspace(hermes_home, scripts={"test": "vitest", "dev": "vite"}) + changed = str(project / "src" / "app.ts") + mark_workspace_edited(session_id="s1", cwd=project, paths=[changed]) + nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed]) + assert nudge is not None + assert "hermes verify --json" in nudge + # The cheap verify commands are still listed first. + assert "npm run test" in nudge + + +def test_nudge_mentions_hermes_verify_when_manifest_exists(hermes_home): + # No start script, but a saved .hermes/environment.json qualifies. + project = _workspace( + hermes_home, + scripts={"test": "vitest"}, + manifest_recipe={"name": "Fake", "test": ["echo ok"]}, + ) + changed = str(project / "src" / "app.ts") + mark_workspace_edited(session_id="s1", cwd=project, paths=[changed]) + nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed]) + assert nudge is not None + assert "hermes verify --json" in nudge + + +def test_nudge_keeps_plain_wording_without_recipe_start(hermes_home): + # Verify commands but no start script and no manifest: today's wording. + project = _workspace(hermes_home, scripts={"test": "vitest"}) + changed = str(project / "src" / "app.ts") + mark_workspace_edited(session_id="s1", cwd=project, paths=[changed]) + nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed]) + assert nudge is not None + assert "hermes verify" not in nudge + + +def test_nudge_recipe_detection_failure_is_silent(hermes_home, monkeypatch): + # A broken recipe detector must never break the nudge path. + import agent.verify.recipes as recipes + + def boom(_root): + raise RuntimeError("detector exploded") + + monkeypatch.setattr(recipes, "detect_recipe", boom) + project = _workspace(hermes_home, scripts={"test": "vitest", "dev": "vite"}) + changed = str(project / "src" / "app.ts") + mark_workspace_edited(session_id="s1", cwd=project, paths=[changed]) + nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed]) + assert nudge is not None + assert "hermes verify" not in nudge + + +# --------------------------------------------------------------------------- +# detection unification: project-facts commands merged into detected recipes +# --------------------------------------------------------------------------- + + +def test_detect_path_merges_project_facts_commands(hermes_home, capsys): + project = _workspace(hermes_home) # package.json with no scripts + scripts_dir = project / "scripts" + scripts_dir.mkdir() + (scripts_dir / "run_tests.sh").write_text("#!/bin/sh\n", encoding="utf-8") + (project / "pytest.ini").write_text("[pytest]\n", encoding="utf-8") + + code = run_verify_command(make_args(project, detect_only=True)) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["source"] == "detected" + tests = payload["recipe"]["test"] + assert "scripts/run_tests.sh" in tests + assert "pytest" in tests + + +def test_manifest_recipe_is_not_merged(hermes_home, capsys): + # A saved manifest is the user-edited source of truth; leave it alone. + project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]}) + (project / "pytest.ini").write_text("[pytest]\n", encoding="utf-8") + code = run_verify_command(make_args(project, detect_only=True)) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["source"] == "manifest" + assert payload["recipe"]["test"] == ["echo ok"] + + +def test_merge_skips_commands_recipe_already_has(hermes_home, capsys): + project = _workspace(hermes_home, scripts={"test": "vitest"}) + code = run_verify_command(make_args(project, detect_only=True)) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["recipe"]["test"].count("npm run test") == 1