Integrate verify subsystem with the existing verification stack

Rescope: hermes verify fills only the runtime-smoke gap and plugs into
the pieces Hermes already has instead of standing beside them.

- agent/verification_evidence.py: record_verify_run() — explicit ledger
  write for hermes verify results (shared _insert_evidence factored out
  of record_terminal_result). Passing runs mark the workspace passed
  like scripts/run_tests.sh; failures are recorded; --phase/--skip-start
  runs are recorded as targeted scope.
- hermes_cli/verify_cmd.py: record results into the ledger on completion
  (fail-silent, HERMES_SESSION_ID attribution); on the detect path merge
  detect_project_facts verify commands the recipe missed into the
  recipe's test list (never applied to a saved manifest).
- agent/verification_stop.py: recipe-aware nudge — when the workspace
  has a runnable recipe (start command or .hermes/environment.json),
  suggest hermes verify --json as the preferred full check; cheap,
  try/except-guarded detection that can never break the nudge path.
- agent/verify/recipes.py: document layer ownership (coding_context =
  cheap prompt facts; verify/recipes = deep runtime recipe).
- tests/verify/test_ledger_and_nudge_integration.py: 17 tests covering
  ledger pass/fail recording, the closed edit->nudge->verify->satisfied
  loop, recipe-aware nudge wording + fail-silence, and the facts merge.
This commit is contained in:
Teknium 2026-08-07 09:42:21 -07:00
parent cc1acfb229
commit fa1a5c0485
5 changed files with 416 additions and 8 deletions

View File

@ -477,7 +477,56 @@ def record_terminal_result(
)
if evidence is None:
return None
return _insert_evidence(evidence)
def record_verify_run(
*,
root: str | Path,
session_id: str | None = None,
ok: bool,
command: str = "hermes verify",
scope: str = "full",
output: str = "",
) -> Optional[dict[str, Any]]:
"""Record a completed ``hermes verify`` run as verification evidence.
Explicit CLI-side write: unlike :func:`record_terminal_result` there is
nothing to classify the caller (the ``hermes verify`` command) already
knows the run was a verification pass and whether it succeeded. A passing
run marks the workspace ``passed`` for the verify-on-stop guard exactly
like a passing canonical test command would; a failing run records the
failure so the guard keeps asking for a fix.
``root`` is re-resolved through :func:`agent.coding_context.project_facts_for`
so the recorded workspace root matches what :func:`verification_status`
derives when the stop guard later looks the evidence up.
"""
try:
from agent.coding_context import project_facts_for
facts = project_facts_for(root)
except Exception:
facts = None
resolved = str(Path(root).resolve())
evidence = VerificationEvidence(
command=command,
canonical_command="hermes verify",
kind="verify",
scope=scope if scope in {"full", "targeted"} else "full",
status="passed" if ok else "failed",
exit_code=0 if ok else 1,
cwd=resolved,
root=str((facts or {}).get("root") or resolved),
session_id=str(session_id or "default"),
output_summary=_summarize_output(output),
)
return _insert_evidence(evidence)
def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]:
"""Insert a classified evidence row and repoint the workspace state."""
created_at = _utc_now()
with _DB_LOCK:
with _transaction() as conn:

View File

@ -183,6 +183,30 @@ def _format_changed_paths(paths: list[str]) -> str:
return "\n".join(lines)
def _workspace_has_runnable_recipe(root: Any) -> bool:
"""Whether the workspace has a runtime verify recipe ``hermes verify`` can run.
True when a saved ``.hermes/environment.json`` manifest exists, or when
cheap static detection (:func:`agent.verify.recipes.detect_recipe`) finds a
recipe with a start command. Deliberately fail-silent and cheap this only
decorates the nudge text; it must never break or slow the nudge path.
"""
if not root:
return False
try:
root_path = Path(str(root))
from agent.verify.environment import manifest_path
if manifest_path(root_path).is_file():
return True
from agent.verify.recipes import detect_recipe
recipe = detect_recipe(root_path)
return bool(recipe is not None and recipe.start)
except Exception:
return False
def _status_detail(status: dict[str, Any]) -> str:
state = str(status.get("status") or "unverified")
evidence = status.get("evidence") if isinstance(status.get("evidence"), dict) else None
@ -248,16 +272,31 @@ def build_verify_on_stop_nudge(
+ (", ..." if len(verify_commands) > 3 else "")
+ "), read any failure, repair the code, and summarize what passed."
)
if _workspace_has_runnable_recipe(facts.get("root")):
command_instruction += (
" For a full check including a runtime boot (build + test + "
"start + readiness), prefer `hermes verify --json` — a passing "
"run records verification evidence for this workspace."
)
else:
temp_dir = os.path.realpath(tempfile.gettempdir())
command_instruction = (
"No canonical test/lint/build command was detected. Create a focused "
f"temporary verification script under `{temp_dir}` using an OS-safe "
"`tempfile` path with a `hermes-verify-` filename prefix, run it "
"against the changed behavior, clean it up when possible, and "
"summarize it explicitly as ad-hoc verification rather than suite "
"green."
)
if _workspace_has_runnable_recipe(facts.get("root")):
command_instruction = (
"No canonical test/lint/build command was detected, but the "
"project has a runnable verification recipe. Run `hermes verify "
"--json` (detect -> build -> test -> boot -> readiness poll); a "
"passing run records verification evidence for this workspace. "
"Read any failure, repair the code, and summarize what passed."
)
else:
command_instruction = (
"No canonical test/lint/build command was detected. Create a focused "
f"temporary verification script under `{temp_dir}` using an OS-safe "
"`tempfile` path with a `hermes-verify-` filename prefix, run it "
"against the changed behavior, clean it up when possible, and "
"summarize it explicitly as ad-hoc verification rather than suite "
"green."
)
return (
"[System: You edited code in this turn, but the workspace does not have "

View File

@ -5,6 +5,21 @@ Mirrors grok's detection order and command choices: Node (lockfile-based
package-manager choice + framework detection), Python (Django / FastAPI /
generic, with uv/poetry/pipenv awareness), Go, Rust, Java (Maven/Gradle),
Makefile fallback, plus a docker-compose recipe.
Layer ownership vs :mod:`agent.coding_context`:
- ``agent.coding_context.detect_project_facts`` owns the *cheap prompt-time
facts* manifests, package managers, and test/lint/build verify commands
surfaced in the system-prompt workspace snapshot, the verify-on-stop nudge,
and the desktop verify UI. Its output must stay byte-stable and fast; do
not push runtime detection into it.
- This module owns the *deep runtime recipe* framework identification,
bootstrap/build/test command inference, and crucially the start command,
port, and readiness path that let ``hermes verify`` boot the app and prove
it serves HTTP. The ``hermes verify`` CLI merges any project-facts verify
commands the recipe missed into its test list (see
``hermes_cli.verify_cmd._merge_project_facts_commands``) so the two layers
extend rather than contradict each other.
"""
from __future__ import annotations

View File

@ -4,11 +4,16 @@ Scoped port of superagent-ai/grok-cli's verify subsystem entrypoint.
Statically detects the project kind (or loads the saved manifest at
``.hermes/environment.json``), then runs bootstrap/build/test phases and an
optional background start + readiness poll, printing an evidence summary.
Completed runs are recorded into the coding verification evidence ledger
(:mod:`agent.verification_evidence`), so a passing ``hermes verify`` satisfies
the verify-on-stop guard the same way a passing canonical test command does.
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
@ -38,6 +43,9 @@ def run_verify_command(args) -> int:
print(message, file=sys.stderr)
return 1
if source == "detected":
_merge_project_facts_commands(root, recipe)
if args.port:
recipe.port = args.port
@ -65,6 +73,8 @@ def run_verify_command(args) -> int:
port_override=args.port,
)
_record_evidence(root, recipe, result, partial=bool(phases or args.skip_start))
if args.json:
payload = result.to_dict()
payload["source"] = source
@ -75,6 +85,69 @@ def run_verify_command(args) -> int:
return 0 if result.ok else 1
def _merge_project_facts_commands(root: Path, recipe) -> None:
"""Fold ``detect_project_facts`` verify commands into a detected recipe.
Layer ownership: ``agent.coding_context`` owns the cheap prompt-time facts
(test/lint/build commands surfaced in the workspace snapshot and the
verify-on-stop nudge); ``agent.verify.recipes`` owns the deep runtime
recipe (framework, start command, port, readiness). When the two disagree
the runtime recipe must not *lose* commands the prompt layer already
promised the model e.g. ``scripts/run_tests.sh`` or a ``pytest`` config
the recipe detector doesn't know about — so any project-facts verify
command not already covered is appended to the recipe's test list.
Never applied to a saved manifest (the user-edited manifest is the source
of truth) and never raises: this is a best-effort union.
"""
try:
from agent.coding_context import detect_project_facts
facts_commands = list(detect_project_facts(root).verify_commands)
except Exception:
return
existing = {c.strip() for c in (*recipe.bootstrap, *recipe.build, *recipe.test) if c}
for command in facts_commands:
command = command.strip()
if command and command not in existing:
recipe.test.append(command)
existing.add(command)
def _record_evidence(root: Path, recipe, result, *, partial: bool) -> None:
"""Record the completed run into the verification evidence ledger.
Best-effort and fail-silent: a ledger problem must never change the CLI's
exit code or output. ``partial`` (an explicit ``--phase`` subset or
``--skip-start``) downgrades the scope to ``targeted`` so a partial pass
is never presented as a full workspace green.
"""
try:
from agent.verification_evidence import record_verify_run
tails: list[str] = []
for p in result.phases:
if p.output_tail:
tails.append(f"[{p.phase}] {p.command}\n{p.output_tail}")
if result.readiness is not None:
r = result.readiness
readiness_line = (
f"[start] {recipe.start} -> "
+ (f"ready (HTTP {r.status_code})" if r.ready else f"not ready ({r.error or 'timeout'})")
)
tails.append(readiness_line)
record_verify_run(
root=root,
session_id=os.environ.get("HERMES_SESSION_ID"),
ok=result.ok,
command="hermes verify",
scope="targeted" if partial else "full",
output="\n".join(tails),
)
except Exception:
pass
def _print_human_report(recipe, source, result) -> None:
print(f"Recipe: {recipe.name} ({recipe.kind}) — source: {source}")
print()

View File

@ -0,0 +1,232 @@
"""Integration of the verify subsystem with the existing verification stack.
Covers the closed loop the rescoped PR is about:
- ``hermes verify`` records into the evidence ledger (pass and fail),
- a passing run satisfies the verify-on-stop guard,
- the verify-on-stop nudge names ``hermes verify --json`` when the workspace
has a runnable recipe (start command or saved manifest),
- the CLI's detect path merges ``detect_project_facts`` verify commands the
recipe missed.
"""
import argparse
import json
import pytest
from agent.verification_evidence import (
mark_workspace_edited,
record_verify_run,
verification_status,
)
from agent.verification_stop import build_verify_on_stop_nudge
from hermes_cli.verify_cmd import run_verify_command
def make_args(path, **overrides):
defaults = dict(
path=str(path),
detect_only=False,
save=False,
skip_start=False,
phase=None,
port=None,
timeout=60.0,
ready_timeout=5.0,
json=True,
)
defaults.update(overrides)
return argparse.Namespace(**defaults)
@pytest.fixture
def hermes_home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes-home"))
monkeypatch.delenv("HERMES_SESSION_ID", raising=False)
return tmp_path
def _workspace(tmp_path, *, scripts=None, manifest_recipe=None):
"""A marker-rooted workspace (package.json) with an optional saved recipe."""
project = tmp_path / "project"
project.mkdir()
(project / "package.json").write_text(
json.dumps({"scripts": scripts} if scripts else {}), encoding="utf-8"
)
if manifest_recipe is not None:
hermes_dir = project / ".hermes"
hermes_dir.mkdir()
(hermes_dir / "environment.json").write_text(
json.dumps({"version": 1, "recipe": manifest_recipe}), encoding="utf-8"
)
return project
# ---------------------------------------------------------------------------
# ledger recording
# ---------------------------------------------------------------------------
def test_record_verify_run_marks_workspace_passed(hermes_home):
project = _workspace(hermes_home)
event = record_verify_run(root=project, session_id="s1", ok=True, output="all green")
assert event is not None
assert event["status"] == "passed"
assert event["kind"] == "verify"
status = verification_status(session_id="s1", cwd=project)
assert status["status"] == "passed"
assert status["evidence"]["canonical_command"] == "hermes verify"
def test_record_verify_run_records_failure(hermes_home):
project = _workspace(hermes_home)
record_verify_run(root=project, session_id="s1", ok=False, output="boom")
status = verification_status(session_id="s1", cwd=project)
assert status["status"] == "failed"
def test_cli_passing_run_writes_ledger_evidence(hermes_home, capsys):
project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]})
code = run_verify_command(make_args(project))
assert code == 0
assert json.loads(capsys.readouterr().out)["ok"] is True
status = verification_status(session_id=None, cwd=project)
assert status["status"] == "passed"
assert status["evidence"]["scope"] == "full"
def test_cli_failing_run_writes_failed_evidence(hermes_home, capsys):
project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["false"]})
code = run_verify_command(make_args(project))
assert code == 1
status = verification_status(session_id=None, cwd=project)
assert status["status"] == "failed"
def test_cli_partial_run_records_targeted_scope(hermes_home, capsys):
# --skip-start / --phase subsets must never present as full workspace green.
project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]})
code = run_verify_command(make_args(project, skip_start=True))
assert code == 0
status = verification_status(session_id=None, cwd=project)
assert status["evidence"]["scope"] == "targeted"
def test_cli_run_uses_hermes_session_id_env(hermes_home, capsys, monkeypatch):
monkeypatch.setenv("HERMES_SESSION_ID", "sess-42")
project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]})
run_verify_command(make_args(project))
assert verification_status(session_id="sess-42", cwd=project)["status"] == "passed"
# ---------------------------------------------------------------------------
# closed loop: edit -> stop guard nudge -> hermes verify -> guard satisfied
# ---------------------------------------------------------------------------
def test_passing_verify_run_satisfies_stop_guard(hermes_home, capsys):
project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]})
changed = str(project / "src" / "app.ts")
mark_workspace_edited(session_id="default", cwd=project, paths=[changed])
assert build_verify_on_stop_nudge(session_id="default", changed_paths=[changed]) is not None
assert run_verify_command(make_args(project)) == 0
assert build_verify_on_stop_nudge(session_id="default", changed_paths=[changed]) is None
# ---------------------------------------------------------------------------
# nudge wording: recipe-aware `hermes verify --json` suggestion
# ---------------------------------------------------------------------------
def test_nudge_mentions_hermes_verify_when_recipe_has_start(hermes_home):
project = _workspace(hermes_home, scripts={"test": "vitest", "dev": "vite"})
changed = str(project / "src" / "app.ts")
mark_workspace_edited(session_id="s1", cwd=project, paths=[changed])
nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
assert nudge is not None
assert "hermes verify --json" in nudge
# The cheap verify commands are still listed first.
assert "npm run test" in nudge
def test_nudge_mentions_hermes_verify_when_manifest_exists(hermes_home):
# No start script, but a saved .hermes/environment.json qualifies.
project = _workspace(
hermes_home,
scripts={"test": "vitest"},
manifest_recipe={"name": "Fake", "test": ["echo ok"]},
)
changed = str(project / "src" / "app.ts")
mark_workspace_edited(session_id="s1", cwd=project, paths=[changed])
nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
assert nudge is not None
assert "hermes verify --json" in nudge
def test_nudge_keeps_plain_wording_without_recipe_start(hermes_home):
# Verify commands but no start script and no manifest: today's wording.
project = _workspace(hermes_home, scripts={"test": "vitest"})
changed = str(project / "src" / "app.ts")
mark_workspace_edited(session_id="s1", cwd=project, paths=[changed])
nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
assert nudge is not None
assert "hermes verify" not in nudge
def test_nudge_recipe_detection_failure_is_silent(hermes_home, monkeypatch):
# A broken recipe detector must never break the nudge path.
import agent.verify.recipes as recipes
def boom(_root):
raise RuntimeError("detector exploded")
monkeypatch.setattr(recipes, "detect_recipe", boom)
project = _workspace(hermes_home, scripts={"test": "vitest", "dev": "vite"})
changed = str(project / "src" / "app.ts")
mark_workspace_edited(session_id="s1", cwd=project, paths=[changed])
nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
assert nudge is not None
assert "hermes verify" not in nudge
# ---------------------------------------------------------------------------
# detection unification: project-facts commands merged into detected recipes
# ---------------------------------------------------------------------------
def test_detect_path_merges_project_facts_commands(hermes_home, capsys):
project = _workspace(hermes_home) # package.json with no scripts
scripts_dir = project / "scripts"
scripts_dir.mkdir()
(scripts_dir / "run_tests.sh").write_text("#!/bin/sh\n", encoding="utf-8")
(project / "pytest.ini").write_text("[pytest]\n", encoding="utf-8")
code = run_verify_command(make_args(project, detect_only=True))
assert code == 0
payload = json.loads(capsys.readouterr().out)
assert payload["source"] == "detected"
tests = payload["recipe"]["test"]
assert "scripts/run_tests.sh" in tests
assert "pytest" in tests
def test_manifest_recipe_is_not_merged(hermes_home, capsys):
# A saved manifest is the user-edited source of truth; leave it alone.
project = _workspace(hermes_home, manifest_recipe={"name": "Fake", "test": ["echo ok"]})
(project / "pytest.ini").write_text("[pytest]\n", encoding="utf-8")
code = run_verify_command(make_args(project, detect_only=True))
assert code == 0
payload = json.loads(capsys.readouterr().out)
assert payload["source"] == "manifest"
assert payload["recipe"]["test"] == ["echo ok"]
def test_merge_skips_commands_recipe_already_has(hermes_home, capsys):
project = _workspace(hermes_home, scripts={"test": "vitest"})
code = run_verify_command(make_args(project, detect_only=True))
assert code == 0
payload = json.loads(capsys.readouterr().out)
assert payload["recipe"]["test"].count("npm run test") == 1