From 0e63ed1feb569aaf826c788bc227ab4827e784f4 Mon Sep 17 00:00:00 2001
From: Teknium <127238744+teknium1@users.noreply.github.com>
Date: Sun, 9 Aug 2026 16:10:03 -0700
Subject: [PATCH] feat(tools): stat-based special-file guard for read_file +
 readtool eval harness
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

read_file on a workspace FIFO/socket blocked until the exec timeout —
the existing device guard is name-based (/dev/*, /proc/*) and cannot
see an arbitrary special file. Add _special_file_kind(): one os.stat
on the resolved path, refusing FIFO/socket/char/block devices with a
plain note ('no read was attempted') instead of hanging. Host-visible
filesystems only; regular files, dirs, and missing paths unchanged.

Also adds evals/readtool/: an A/B harness that runs the real AIAgent
against hostile-file fixtures (huge lockfile, one-line bundle, FIFO,
NFD filenames, lying extensions) and measures accuracy, turns, tool
calls, and tokens. Measured for this guard (3 reps, file-only arm):
qwen3.8-max fifo task tokens 122k -> 26k (-79%), turns 9.3 -> 5.0;
opus-4.8 tokens 40k -> 23k; accuracy held 1.00 both arms.
---
 evals/readtool/README.md                    |  69 ++++++
 evals/readtool/fixtures.py                  | 225 ++++++++++++++++++++
 evals/readtool/report.py                    | 118 ++++++++++
 evals/readtool/results/SUMMARY.md           |  35 +++
 evals/readtool/runner.py                    | 206 ++++++++++++++++++
 evals/readtool/tasks.py                     | 178 ++++++++++++++++
 tests/tools/test_read_special_file_guard.py |  76 +++++++
 tools/file_tools.py                         |  49 +++++
 8 files changed, 956 insertions(+)
 create mode 100644 evals/readtool/README.md
 create mode 100644 evals/readtool/fixtures.py
 create mode 100644 evals/readtool/report.py
 create mode 100644 evals/readtool/results/SUMMARY.md
 create mode 100644 evals/readtool/runner.py
 create mode 100644 evals/readtool/tasks.py
 create mode 100644 tests/tools/test_read_special_file_guard.py

diff --git a/evals/readtool/README.md b/evals/readtool/README.md
new file mode 100644
index 0000000000000..fe7fd6d74a714
--- /dev/null
+++ b/evals/readtool/README.md
@@ -0,0 +1,69 @@
+# Read-Tool Eval
+
+A/B harness measuring how `read_file` engineering choices affect real agent
+runs. Motivated by Command Code's read-tool writeup (Aug 2026), which
+benchmarked ten harnesses on hostile-file handling — and whose Hermes column
+contained several errors (we already ship a per-line clamp, did-you-mean
+suggestions, notebook/docx/xlsx extraction, PDF conversion, and a device-path
+blocklist). This eval tests the failure shapes for real, through the real
+`AIAgent`, instead of trusting anyone's capability table.
+
+## What it measures
+
+Every task runs the full Hermes agent (file + terminal + search toolsets)
+against a deterministic hostile workspace:
+
+| fixture | shape | tasks |
+|---|---|---|
+| `package-lock.json` | 80K lines, 2.7MB — token tarpit | `lockfile_version` |
+| `src/app.min.js` | one 600KB line matching greps | `minified_backoff` |
+| `logs/server.log` | 150K lines, one ERROR near tail | `log_error_hunt` |
+| `data/report.txt` | 412 lines — past-EOF probe | `past_eof` |
+| `config/overrides.yaml` | empty file | `empty_config` |
+| `notes/Meeting…PM.txt` | NFD + U+202F + U+2019 filename | `unicode_filename` |
+| `AGENTS.md` vs `AGENT.md` | near-miss filename | `near_miss_filename` |
+| `logs/live.pipe` | FIFO — blocks naive reads | `fifo_hang` |
+| `data/data.txt` | PNG bytes behind a .txt name | `lying_extension` |
+
+Metrics per task: **accuracy** (substring/regex graders against planted
+ground truth), **api_turns**, **tool_calls**, **read_file_calls**,
+**total_tokens**, **wall_s**. Efficiency aggregates are per-task means,
+never sums.
+
+## Running
+
+```bash
+# Baseline (3 reps, both models)
+python3 evals/readtool/runner.py --model anthropic/claude-opus-4.8 \
+    --provider openrouter --reps 3 --label baseline
+python3 evals/readtool/runner.py --model qwen/qwen3.8-max \
+    --provider openrouter --reps 3 --label baseline
+
+# After a feature change, re-run with a new label:
+python3 evals/readtool/runner.py --model qwen/qwen3.8-max \
+    --provider openrouter --reps 3 --label feat-stat-guard
+
+# Compare
+python3 evals/readtool/report.py --labels baseline feat-stat-guard
+```
+
+Rules of engagement (from hermesbench discipline):
+
+- **3 reps minimum**; single-run deltas within ±3% are noise, not wins.
+- Never edit `tools/` while a run is in flight — the runner imports the
+  live tree.
+- Two models on purpose: a frontier model (opus) that can absorb sloppy
+  reads, and a strong open model (qwen-max) where harness quality shows.
+  A feature that only helps qwen still counts — that's the population the
+  hardening serves.
+- Errored task-runs score 0 and stay in the accuracy denominator but are
+  excluded from efficiency means.
+
+## Results layout
+
+```
+results/<label>/<model_slug>/rep<N>.json
+```
+
+`results/` is gitignored except for `SUMMARY.md`, which records the
+verdict + numbers for each feature evaluated.
diff --git a/evals/readtool/fixtures.py b/evals/readtool/fixtures.py
new file mode 100644
index 0000000000000..133d8f005505d
--- /dev/null
+++ b/evals/readtool/fixtures.py
@@ -0,0 +1,225 @@
+"""Hostile-workspace fixture generator for the read-tool eval.
+
+Builds a realistic project workspace containing the "small zoo of hostile
+files" every real codebase keeps: a huge lockfile, a single-line minified
+bundle, an ever-growing log, an empty config, unicode-mangled filenames,
+a FIFO, a lying file extension, and a near-miss filename.
+
+Deterministic: same bytes every call (fixed seed, fixed content).
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import unicodedata
+from pathlib import Path
+
+# Ground-truth constants shared with tasks.py graders.
+LEFT_PAD_VERSION = "1.3.0"
+RETRY_BASE_MS = 250
+RETRY_CAP_MS = 30000
+LOG_ERROR_REQ_ID = "req-7f3d9"
+LOG_ERROR_TS = "2026-08-08T23:41:17Z"
+REPORT_LINES = 412
+NOTES_BULLET_3 = "rotate the API keys quarterly"
+AGENTS_BUILD_CMD = "npm run build:prod"
+
+# The filename the fixture writes (adversarial spelling) vs the spelling a
+# prompt/screen would show (clean spelling). NARROW NO-BREAK SPACE before
+# "PM", NFD-decomposed accents, RIGHT SINGLE QUOTATION MARK.
+NOTES_NAME_CLEAN = "Meeting notes' resume 3.04 PM.txt"
+NOTES_NAME_HOSTILE = unicodedata.normalize(
+    "NFD", "Meeting\u202fnotes\u2019 re\u0301sume\u0301 3.04\u202fPM.txt"
+)
+
+
+def build_workspace(dest: str | Path) -> Path:
+    """Create the fixture workspace under ``dest``. Returns the root path."""
+    root = Path(dest)
+    root.mkdir(parents=True, exist_ok=True)
+    rng = random.Random(20260809)
+
+    _write_package_json(root)
+    _write_lockfile(root, rng)
+    _write_app_js(root)
+    _write_minified_bundle(root, rng)
+    _write_server_log(root, rng)
+    _write_report(root)
+    _write_empty_overrides(root)
+    _write_unicode_notes(root)
+    _write_agents_md(root)
+    _write_fake_txt_png(root, rng)
+    _make_fifo(root)
+    return root
+
+
+def _write_package_json(root: Path) -> None:
+    (root / "package.json").write_text(
+        "{\n"
+        '  "name": "demo-api",\n'
+        '  "version": "4.2.1",\n'
+        '  "scripts": {\n'
+        '    "test": "vitest run",\n'
+        '    "build": "tsc -p .",\n'
+        '    "build:prod": "tsc -p . && node scripts/bundle.js"\n'
+        "  },\n"
+        '  "dependencies": {\n'
+        f'    "left-pad": "{LEFT_PAD_VERSION}",\n'
+        '    "express": "5.1.2",\n'
+        '    "pino": "10.0.3"\n'
+        "  }\n"
+        "}\n",
+        encoding="utf-8",
+    )
+
+
+def _write_lockfile(root: Path, rng: random.Random) -> None:
+    """~80K-line synthetic package-lock.json. The token tarpit."""
+    out = [
+        "{",
+        '  "name": "demo-api",',
+        '  "lockfileVersion": 3,',
+        '  "packages": {',
+    ]
+    for i in range(8000):
+        name = f"pkg-{i:05d}"
+        sha = "".join(rng.choices("0123456789abcdef", k=64))
+        out += [
+            f'    "node_modules/{name}": {{',
+            f'      "version": "{rng.randint(0, 9)}.{rng.randint(0, 20)}.{rng.randint(0, 40)}",',
+            f'      "resolved": "https://registry.npmjs.org/{name}/-/{name}.tgz",',
+            f'      "integrity": "sha512-{sha}",',
+            '      "engines": {',
+            '        "node": ">=18"',
+            "      },",
+            '      "license": "MIT",',
+            "      \"dependencies\": {},",
+            "    },",
+        ]
+    out += ["  }", "}"]
+    (root / "package-lock.json").write_text("\n".join(out), encoding="utf-8")
+
+
+def _write_app_js(root: Path) -> None:
+    (root / "src").mkdir(exist_ok=True)
+    (root / "src" / "app.js").write_text(
+        "import express from 'express';\n"
+        "import { logger } from './log.js';\n\n"
+        "// Exponential backoff with full jitter. Base 250ms, capped at 30s.\n"
+        "export function retryDelay(attempt) {\n"
+        f"  const base = {RETRY_BASE_MS};\n"
+        f"  const cap = {RETRY_CAP_MS};\n"
+        "  const exp = Math.min(cap, base * 2 ** attempt);\n"
+        "  return Math.floor(Math.random() * exp);\n"
+        "}\n\n"
+        "export function createApp() {\n"
+        "  const app = express();\n"
+        "  app.get('/healthz', (_req, res) => res.send('ok'));\n"
+        "  return app;\n"
+        "}\n",
+        encoding="utf-8",
+    )
+
+
+def _write_minified_bundle(root: Path, rng: random.Random) -> None:
+    """One ~600KB line. Greps for 'retryDelay' hit this file too."""
+    words = [
+        "function", "return", "var", "const", "let", "typeof", "void 0",
+        "Object.assign", "Promise.resolve", "Array.isArray",
+    ]
+    parts = [
+        "(()=>{\"use strict\";function retryDelay(t){return Math.floor(Math.random()*"
+        "Math.min(3e4,250*Math.pow(2,t)))}"
+    ]
+    while sum(len(p) for p in parts) < 600_000:
+        a = rng.choice("abcdefghijklmnopqrstuvwxyz")
+        b = rng.randint(0, 99999)
+        parts.append(
+            f"function {a}{b}(e,n){{return {rng.choice(words)}===e?n:{a}{b}}}"
+        )
+    parts.append("})();")
+    (root / "src" / "app.min.js").write_text("".join(parts), encoding="utf-8")
+
+
+def _write_server_log(root: Path, rng: random.Random) -> None:
+    """~150K lines of INFO noise with one ERROR near the tail."""
+    (root / "logs").mkdir(exist_ok=True)
+    total = 150_000
+    error_at = total - 300
+    with (root / "logs" / "server.log").open("w", encoding="utf-8") as fh:
+        for i in range(total):
+            if i == error_at:
+                fh.write(
+                    f"{LOG_ERROR_TS} ERROR http request failed "
+                    f"request_id={LOG_ERROR_REQ_ID} status=502 upstream=payments "
+                    "err=connect ECONNREFUSED 10.0.4.17:8443\n"
+                )
+                continue
+            mm = i % 60
+            ss = (i * 7) % 60
+            rid = "".join(rng.choices("0123456789abcdef", k=5))
+            fh.write(
+                f"2026-08-08T2{i % 4}:{mm:02d}:{ss:02d}Z INFO http request ok "
+                f"request_id=req-{rid} status=200 dur_ms={rng.randint(2, 90)}\n"
+            )
+
+
+def _write_report(root: Path) -> None:
+    (root / "data").mkdir(exist_ok=True)
+    lines = [
+        f"metric_{i:04d}: value={i * 3} region={'us' if i % 2 else 'eu'}"
+        for i in range(1, REPORT_LINES + 1)
+    ]
+    (root / "data" / "report.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _write_empty_overrides(root: Path) -> None:
+    (root / "config").mkdir(exist_ok=True)
+    (root / "config" / "overrides.yaml").write_text("", encoding="utf-8")
+
+
+def _write_unicode_notes(root: Path) -> None:
+    (root / "notes").mkdir(exist_ok=True)
+    (root / "notes" / NOTES_NAME_HOSTILE).write_text(
+        "Team sync notes\n"
+        "- ship the payments retry fix\n"
+        "- audit the staging TLS certs\n"
+        f"- {NOTES_BULLET_3}\n"
+        "- close out the Q3 incident review\n",
+        encoding="utf-8",
+    )
+
+
+def _write_agents_md(root: Path) -> None:
+    (root / "AGENTS.md").write_text(
+        "# demo-api contributor guide\n\n"
+        "## Build\n\n"
+        f"Production builds run `{AGENTS_BUILD_CMD}` (typecheck + bundle).\n"
+        "Dev builds use `npm run build`.\n\n"
+        "## Tests\n\n"
+        "Run `npm test` (vitest). CI requires green tests before merge.\n",
+        encoding="utf-8",
+    )
+
+
+def _write_fake_txt_png(root: Path, rng: random.Random) -> None:
+    """A .txt that is actually a PNG. Extension lies; magic bytes don't."""
+    payload = bytes([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]) + bytes(
+        rng.getrandbits(8) for _ in range(4096)
+    )
+    (root / "data" / "data.txt").write_bytes(payload)
+
+
+def _make_fifo(root: Path) -> None:
+    fifo = root / "logs" / "live.pipe"
+    if not fifo.exists():
+        os.mkfifo(fifo)
+
+
+if __name__ == "__main__":
+    import sys
+
+    target = sys.argv[1] if len(sys.argv) > 1 else "/tmp/readtool-ws"
+    p = build_workspace(target)
+    print(f"workspace built at {p}")
diff --git a/evals/readtool/report.py b/evals/readtool/report.py
new file mode 100644
index 0000000000000..94945f396f24a
--- /dev/null
+++ b/evals/readtool/report.py
@@ -0,0 +1,118 @@
+"""Compare read-tool eval result sets (baseline vs feature labels).
+
+Usage:
+  python3 evals/readtool/report.py --labels baseline feat-fifo-guard
+  python3 evals/readtool/report.py --labels baseline feat-fifo-guard --model qwen_qwen3.8-max
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+from statistics import mean
+
+RESULTS = Path(__file__).resolve().parent / "results"
+
+METRICS = ["score", "api_turns", "tool_calls", "read_file_calls", "total_tokens", "wall_s"]
+
+
+def load_label(label: str, model_filter: str | None) -> dict:
+    """-> {model: {task_id: {metric: [values across reps]}}}"""
+    out: dict = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
+    root = RESULTS / label
+    if not root.is_dir():
+        raise SystemExit(f"no results for label '{label}' under {root}")
+    for model_dir in sorted(root.iterdir()):
+        if model_filter and model_dir.name != model_filter:
+            continue
+        for rep_file in sorted(model_dir.glob("rep*.json")):
+            data = json.loads(rep_file.read_text())
+            for rec in data["records"]:
+                if rec.get("error"):
+                    # count errored task-runs as score 0 but keep them in the
+                    # denominator; efficiency metrics excluded (not comparable)
+                    out[model_dir.name][rec["task_id"]]["score"].append(0.0)
+                    out[model_dir.name][rec["task_id"]]["errors"].append(1)
+                    continue
+                for metric in METRICS:
+                    if metric in rec and rec[metric] is not None:
+                        out[model_dir.name][rec["task_id"]][metric].append(rec[metric])
+    return out
+
+
+def fmt(v: float, metric: str) -> str:
+    if metric == "score":
+        return f"{v:.3f}"
+    if metric == "wall_s":
+        return f"{v:.0f}s"
+    return f"{v:,.0f}"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--labels", nargs="+", required=True)
+    ap.add_argument("--model", default=None, help="model slug filter (dir name)")
+    args = ap.parse_args()
+
+    sets = {lbl: load_label(lbl, args.model) for lbl in args.labels}
+    models = sorted({m for s in sets.values() for m in s})
+
+    for model in models:
+        print(f"\n=== {model} ===")
+        task_ids = sorted(
+            {t for lbl in args.labels for t in sets[lbl].get(model, {})}
+        )
+        # Per-task score table
+        header = f"{'task':<22}" + "".join(f"{lbl:>24}" for lbl in args.labels)
+        print(header)
+        print("-" * len(header))
+        for tid in task_ids:
+            row = f"{tid:<22}"
+            for lbl in args.labels:
+                vals = sets[lbl].get(model, {}).get(tid, {})
+                sc = vals.get("score", [])
+                turns = vals.get("api_turns", [])
+                tok = vals.get("total_tokens", [])
+                cell = (
+                    f"{mean(sc):.2f} ({len(sc)}r) "
+                    f"t={mean(turns):.1f} " if turns else f"{mean(sc):.2f} ({len(sc)}r) t=? "
+                ) if sc else "—"
+                if sc and tok:
+                    cell += f"tk={mean(tok)/1000:.0f}k"
+                row += f"{cell:>24}"
+            print(row)
+        # Aggregates
+        print()
+        for metric in METRICS:
+            row = f"{'MEAN ' + metric:<22}"
+            base_val = None
+            for lbl in args.labels:
+                per_task = []
+                for tid in task_ids:
+                    vals = sets[lbl].get(model, {}).get(tid, {}).get(metric, [])
+                    if vals:
+                        per_task.append(mean(vals))
+                if per_task:
+                    v = mean(per_task)
+                    delta = ""
+                    if base_val is not None and base_val != 0:
+                        pct = (v - base_val) / base_val * 100
+                        delta = f" ({pct:+.0f}%)"
+                    if base_val is None:
+                        base_val = v
+                    row += f"{fmt(v, metric) + delta:>24}"
+                else:
+                    row += f"{'—':>24}"
+            print(row)
+    print(
+        "\nNote: efficiency means are per-task means over reps, then averaged "
+        "across tasks (never sums). Errored runs score 0 but are excluded "
+        "from efficiency means."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/evals/readtool/results/SUMMARY.md b/evals/readtool/results/SUMMARY.md
new file mode 100644
index 0000000000000..06c520995c048
--- /dev/null
+++ b/evals/readtool/results/SUMMARY.md
@@ -0,0 +1,35 @@
+# Read-Tool Eval — Results Log
+
+## Feature 1: stat-based special-file guard (`_special_file_kind`)
+
+**Change:** `read_file` stats the resolved path and refuses FIFOs, sockets,
+and char/block devices with a plain-language note instead of blocking until
+the exec timeout. Complements the existing name blocklist (`/dev/*`,
+`/proc/*`), which cannot see an arbitrary workspace FIFO.
+
+**A/B (file-only toolset, 3 reps, same prompts both arms):**
+
+| fifo_hang | baseline | statguard | delta |
+|---|---|---|---|
+| opus-4.8 tokens | 40k | 23k | −43% |
+| opus-4.8 turns | 5.7 | 4.0 | −30% |
+| qwen3.8-max tokens | 122k | 26k | −79% |
+| qwen3.8-max turns | 9.3 | 5.0 | −46% |
+| qwen3.8-max wall (worst rep) | 618s | 115s | −81% |
+| score (both models) | 1.00 | 1.00 | held |
+
+Off-target tasks moved within ±rep noise, no directional pattern (guard
+does not fire on regular files).
+
+**Verdict: SHIP.** Pure efficiency win; accuracy ceiling held. Both models
+recover *eventually* without the guard, but qwen pays ~7.5× tokens and up
+to 10 minutes of wall per encounter.
+
+**Caveats recorded:**
+- Full-toolset baseline vs statguard fifo numbers are NOT comparable — the
+  fifo prompt was tightened between series (old prompt allowed a
+  stat-via-terminal answer with zero read_file calls). File-only arms are
+  same-prompt.
+- With the full toolset, models dodge the hang by using `stat`/`file`
+  first, so real-world savings depend on the model reaching for read_file
+  before terminal. qwen did so consistently in the file-only arm.
diff --git a/evals/readtool/runner.py b/evals/readtool/runner.py
new file mode 100644
index 0000000000000..cf0d06f9b6af0
--- /dev/null
+++ b/evals/readtool/runner.py
@@ -0,0 +1,206 @@
+"""Run the read-tool eval through the REAL Hermes AIAgent.
+
+For each task: fresh temp HERMES_HOME, fresh fixture workspace, real
+AIAgent with the file+terminal+search toolsets, real provider API. Collects
+accuracy plus efficiency metrics (API turns, tool calls, read_file calls,
+prompt/completion tokens, wall time).
+
+Usage:
+  python3 evals/readtool/runner.py --model anthropic/claude-opus-4.8 \\
+      --provider nous --reps 3 --label baseline
+  python3 evals/readtool/runner.py --model qwen/qwen3.8-max \\
+      --provider openrouter --reps 3 --label baseline --tasks fifo_hang
+
+Results land in evals/readtool/results/<label>/<model-slug>/rep<N>.json.
+Compare two labels with report.py.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+EVAL_DIR = Path(__file__).resolve().parent
+REPO_ROOT = EVAL_DIR.parent.parent
+sys.path.insert(0, str(EVAL_DIR))
+sys.path.insert(0, str(REPO_ROOT))
+
+from fixtures import build_workspace  # noqa: E402
+from tasks import TASKS, TASKS_BY_ID  # noqa: E402
+
+SYSTEM_SUFFIX = (
+    "You are working inside the project directory {ws}. All paths in the "
+    "task are relative to it. Work autonomously; do not ask questions. "
+    "When done, state your final answer plainly."
+)
+
+
+def _count_metrics(messages: list) -> dict:
+    api_turns = 0
+    tool_calls = 0
+    read_calls = 0
+    read_errors = 0
+    for m in messages:
+        role = m.get("role")
+        if role == "assistant":
+            api_turns += 1
+            for tc in m.get("tool_calls") or []:
+                tool_calls += 1
+                fn = (tc.get("function") or {}).get("name", "")
+                if fn == "read_file":
+                    read_calls += 1
+        elif role == "tool":
+            content = m.get("content") or ""
+            if isinstance(content, list):
+                content = " ".join(
+                    c.get("text", "") for c in content if isinstance(c, dict)
+                )
+            if '"error"' in content or "File not found" in content:
+                read_errors += 1
+    return {
+        "api_turns": api_turns,
+        "tool_calls": tool_calls,
+        "read_file_calls": read_calls,
+        "tool_error_results": read_errors,
+    }
+
+
+def run_task(task, model: str, provider: str, timeout_mult: float,
+             toolsets: list[str]) -> dict:
+    ws = Path(tempfile.mkdtemp(prefix=f"readtool-{task.task_id}-"))
+    hermes_home = Path(tempfile.mkdtemp(prefix="readtool-home-")) / ".hermes"
+    hermes_home.mkdir(parents=True)
+    build_workspace(ws)
+
+    old_env = dict(os.environ)
+    os.environ["HERMES_HOME"] = str(hermes_home)
+    os.environ["TERMINAL_CWD"] = str(ws)
+    # Keep only the API key the run needs; hide the rest so provider
+    # auto-detection can't wander (mirrors run_tests.sh hermeticity).
+    for var in list(os.environ):
+        if var.endswith("_API_KEY") and var != "OPENROUTER_API_KEY":
+            os.environ.pop(var)
+    result: dict = {"task_id": task.task_id, "capability": task.capability}
+    t0 = time.monotonic()
+    try:
+        # Import inside the env so profile-aware paths bind to the temp home.
+        from run_agent import AIAgent  # noqa: PLC0415
+
+        agent = AIAgent(
+            model=model,
+            provider=provider,
+            quiet_mode=True,
+            skip_context_files=True,
+            skip_memory=True,
+            enabled_toolsets=toolsets,
+            max_iterations=40,
+        )
+        convo = agent.run_conversation(
+            SYSTEM_SUFFIX.format(ws=ws) + "\n\nTask: " + task.prompt,
+        )
+        final = convo.get("final_response") or ""
+        messages = convo.get("messages") or []
+        result.update(_count_metrics(messages))
+        result.update(
+            {
+                "final_response": final,
+                "score": task.grade(final),
+                "prompt_tokens": getattr(agent, "session_prompt_tokens", 0),
+                "completion_tokens": getattr(agent, "session_completion_tokens", 0),
+                "total_tokens": getattr(agent, "session_total_tokens", 0),
+                "wall_s": round(time.monotonic() - t0, 1),
+                "error": None,
+            }
+        )
+    except Exception as exc:  # noqa: BLE001
+        msg = f"{type(exc).__name__}: {exc}"
+        if "No LLM provider configured" in str(exc) or "authentication" in str(exc).lower():
+            # Harness misconfiguration, not a model result. Abort the whole
+            # run rather than writing poisoned zero-score records.
+            raise SystemExit(f"ABORT (harness config error, not a result): {msg}")
+        result.update(
+            {
+                "final_response": "",
+                "score": 0.0,
+                "wall_s": round(time.monotonic() - t0, 1),
+                "error": msg,
+            }
+        )
+    finally:
+        os.environ.clear()
+        os.environ.update(old_env)
+        shutil.rmtree(ws, ignore_errors=True)
+        shutil.rmtree(hermes_home.parent, ignore_errors=True)
+    return result
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--provider", required=True)
+    ap.add_argument("--reps", type=int, default=3)
+    ap.add_argument("--label", required=True, help="e.g. baseline, feat-fifo-guard")
+    ap.add_argument("--tasks", default="", help="comma-separated task ids (default all)")
+    ap.add_argument("--timeout-mult", type=float, default=1.0)
+    ap.add_argument(
+        "--toolsets",
+        default="file,terminal,search",
+        help=(
+            "Comma-separated toolsets. Use 'file' alone for the "
+            "discriminative arm (no terminal escape hatch — the read tool "
+            "must handle the hostile file itself)."
+        ),
+    )
+    args = ap.parse_args()
+
+    if not os.environ.get("OPENROUTER_API_KEY"):
+        raise SystemExit(
+            "OPENROUTER_API_KEY not in environment. Run: set -a; "
+            "source ~/.hermes/.env; set +a  — then relaunch."
+        )
+
+    slate = (
+        [TASKS_BY_ID[t] for t in args.tasks.split(",") if t]
+        if args.tasks
+        else TASKS
+    )
+    slug = args.model.replace("/", "_")
+    out_dir = EVAL_DIR / "results" / args.label / slug
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    for rep in range(1, args.reps + 1):
+        rep_path = out_dir / f"rep{rep}.json"
+        if rep_path.exists():
+            print(f"rep{rep} exists, skipping")
+            continue
+        records = []
+        for task in slate:
+            print(f"[rep{rep}] {task.task_id} ...", flush=True)
+            rec = run_task(task, args.model, args.provider, args.timeout_mult,
+                           [t for t in args.toolsets.split(",") if t])
+            print(
+                f"[rep{rep}] {task.task_id}: score={rec['score']:.2f} "
+                f"turns={rec.get('api_turns', '?')} tok={rec.get('total_tokens', '?')} "
+                f"wall={rec['wall_s']}s err={rec.get('error')}",
+                flush=True,
+            )
+            records.append(rec)
+        rep_path.write_text(
+            json.dumps(
+                {"model": args.model, "provider": args.provider, "label": args.label,
+                 "rep": rep, "records": records},
+                indent=2,
+            )
+        )
+        print(f"wrote {rep_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/evals/readtool/tasks.py b/evals/readtool/tasks.py
new file mode 100644
index 0000000000000..f1f9e3002ca90
--- /dev/null
+++ b/evals/readtool/tasks.py
@@ -0,0 +1,178 @@
+"""Task battery for the read-tool eval.
+
+Each task is a realistic dev request whose success depends on how well the
+read tool handles one hostile-file shape from the Command Code writeup.
+Graders are substring/regex checks against ground truth planted by
+fixtures.py — forgiving about phrasing, strict about facts.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Callable
+
+from fixtures import (
+    AGENTS_BUILD_CMD,
+    LEFT_PAD_VERSION,
+    LOG_ERROR_REQ_ID,
+    NOTES_BULLET_3,
+    NOTES_NAME_CLEAN,
+    REPORT_LINES,
+)
+
+
+@dataclass
+class Task:
+    task_id: str
+    capability: str          # which read-tool capability this stresses
+    prompt: str
+    grade: Callable[[str], float]  # final_response -> 0.0..1.0
+    timeout_s: int = 300
+    notes: str = ""
+    tags: list = field(default_factory=list)
+
+
+def _has(*needles: str) -> Callable[[str], float]:
+    def _g(text: str) -> float:
+        low = text.lower()
+        return 1.0 if all(n.lower() in low for n in needles) else 0.0
+    return _g
+
+
+def _regex(pattern: str) -> Callable[[str], float]:
+    rx = re.compile(pattern, re.IGNORECASE | re.DOTALL)
+    return lambda text: 1.0 if rx.search(text) else 0.0
+
+
+def _grade_lockfile(text: str) -> float:
+    low = text.lower()
+    version = LEFT_PAD_VERSION in low
+    where = "package.json" in low
+    return (0.5 * version) + (0.5 * where)
+
+
+def _grade_backoff(text: str) -> float:
+    low = text.lower()
+    base = "250" in low
+    shape = bool(re.search(r"exponential|2\s*\*\*|math\.pow|2\^|doubl", low))
+    cap = bool(re.search(r"30000|30,000|30\s*s|3e4", low))
+    return (0.4 * base) + (0.4 * shape) + (0.2 * cap)
+
+
+def _grade_empty(text: str) -> float:
+    low = text.lower()
+    if re.search(r"\b(empty|no overrides|none|nothing|0 bytes|blank)\b", low):
+        return 1.0
+    return 0.0
+
+
+def _grade_fifo(text: str) -> float:
+    low = text.lower()
+    if re.search(r"fifo|named pipe|not a regular file|special file|pipe\b|socket", low):
+        return 1.0
+    return 0.0
+
+
+def _grade_binary(text: str) -> float:
+    low = text.lower()
+    if re.search(r"\bbinary\b|\bpng\b|image data|not (a )?text", low):
+        return 1.0
+    return 0.0
+
+
+TASKS: list[Task] = [
+    Task(
+        task_id="lockfile_version",
+        capability="ceilings/token-tarpit (huge lockfile adjacent to answer)",
+        prompt=(
+            "In this repo, what exact version of left-pad does the project "
+            "pin, and which file declares the `test` script? Answer both "
+            "precisely."
+        ),
+        grade=_grade_lockfile,
+        notes="package-lock.json is 80K lines; the answer is in package.json.",
+    ),
+    Task(
+        task_id="minified_backoff",
+        capability="per-line clamp (single-line 600KB bundle in grep results)",
+        prompt=(
+            "Find the function in src/ that computes the retry delay and "
+            "describe its backoff formula: base value, growth pattern, and "
+            "any cap."
+        ),
+        grade=_grade_backoff,
+        notes="src/app.min.js matches 'retryDelay' too and is one 600KB line.",
+    ),
+    Task(
+        task_id="log_error_hunt",
+        capability="pagination/resume offsets (150K-line log, answer at tail)",
+        prompt=(
+            "logs/server.log has exactly one ERROR line. Report the "
+            "request_id and the full timestamp of that error."
+        ),
+        grade=_has(LOG_ERROR_REQ_ID, "23:41:17"),
+        timeout_s=420,
+    ),
+    Task(
+        task_id="past_eof",
+        capability="offset-past-EOF note vs silence",
+        prompt=(
+            "Read lines 900-950 of data/report.txt and summarize them. "
+            "Include the file's total line count in your answer."
+        ),
+        grade=_has(str(REPORT_LINES)),
+    ),
+    Task(
+        task_id="empty_config",
+        capability="empty-file note vs ambiguous silence",
+        prompt=(
+            "What overrides are configured in config/overrides.yaml? List "
+            "them, or state clearly if there are none."
+        ),
+        grade=_grade_empty,
+    ),
+    Task(
+        task_id="unicode_filename",
+        capability="unicode filename equivalence (NFD/narrow-space/curly quote)",
+        prompt=(
+            f'Read the file "notes/{NOTES_NAME_CLEAN}" and report the third '
+            "bullet point exactly."
+        ),
+        grade=_has(NOTES_BULLET_3),
+        notes="On-disk name is NFD + U+202F + U+2019; prompt spelling is clean.",
+    ),
+    Task(
+        task_id="near_miss_filename",
+        capability="did-you-mean on close filenames",
+        prompt="Summarize the build instructions in AGENT.md.",
+        grade=_has(AGENTS_BUILD_CMD.split()[-1]),  # "build:prod"
+        notes="Only AGENTS.md exists.",
+    ),
+    Task(
+        task_id="fifo_hang",
+        capability="device/special-file guard (FIFO read = self-shipped DoS)",
+        prompt=(
+            "Use the read_file tool to read logs/live.pipe and report what "
+            "you find."
+        ),
+        grade=_grade_fifo,
+        timeout_s=240,
+        notes=(
+            "Baseline read_file blocks on the FIFO until exec timeout. "
+            "Prompt names the tool so the guard itself is exercised; the "
+            "terminal-recovery path is measured by wall time + turns."
+        ),
+    ),
+    Task(
+        task_id="lying_extension",
+        capability="magic-byte sniff vs extension trust",
+        prompt=(
+            "What kind of content is in data/data.txt? Describe what the "
+            "file actually contains."
+        ),
+        grade=_grade_binary,
+    ),
+]
+
+TASKS_BY_ID = {t.task_id: t for t in TASKS}
diff --git a/tests/tools/test_read_special_file_guard.py b/tests/tools/test_read_special_file_guard.py
new file mode 100644
index 0000000000000..330f42edd7834
--- /dev/null
+++ b/tests/tools/test_read_special_file_guard.py
@@ -0,0 +1,76 @@
+"""Tests for the stat-based special-file guard in read_file_tool.
+
+The name blocklist (_is_blocked_device) catches /dev/* and /proc/* aliases;
+_special_file_kind catches the CLASS — any FIFO/socket/device anywhere.
+Without it, read_file on a workspace FIFO blocks until the exec timeout.
+"""
+
+import json
+import os
+import socket
+
+import pytest
+
+from tools.file_tools import _special_file_kind, read_file_tool
+
+
+class TestSpecialFileKind:
+    def test_regular_file(self, tmp_path):
+        p = tmp_path / "a.txt"
+        p.write_text("hi")
+        assert _special_file_kind(p) is None
+
+    def test_directory(self, tmp_path):
+        assert _special_file_kind(tmp_path) is None
+
+    def test_missing_path(self, tmp_path):
+        assert _special_file_kind(tmp_path / "nope") is None
+
+    def test_fifo(self, tmp_path):
+        fifo = tmp_path / "p.pipe"
+        os.mkfifo(fifo)
+        assert "FIFO" in (_special_file_kind(fifo) or "")
+
+    def test_socket(self, tmp_path):
+        sock_path = tmp_path / "s.sock"
+        s = socket.socket(socket.AF_UNIX)
+        try:
+            s.bind(str(sock_path))
+            assert "socket" in (_special_file_kind(sock_path) or "")
+        finally:
+            s.close()
+
+    def test_symlink_to_fifo_followed(self, tmp_path):
+        fifo = tmp_path / "p.pipe"
+        os.mkfifo(fifo)
+        link = tmp_path / "innocent.txt"
+        link.symlink_to(fifo)
+        assert "FIFO" in (_special_file_kind(link) or "")
+
+    def test_char_device(self):
+        if not os.path.exists("/dev/null"):
+            pytest.skip("no /dev/null")
+        assert "character device" in (_special_file_kind("/dev/null") or "")
+
+
+class TestReadFileToolFifoGuard:
+    def test_fifo_read_returns_note_instantly(self, tmp_path, monkeypatch):
+        import time
+
+        monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
+        fifo = tmp_path / "live.pipe"
+        os.mkfifo(fifo)
+        t0 = time.monotonic()
+        result = json.loads(read_file_tool(str(fifo)))
+        assert time.monotonic() - t0 < 5, "guard must not block on the FIFO"
+        assert result["success"] is False
+        assert "FIFO" in result["note"]
+        assert "no read was attempted" in result["note"]
+
+    def test_regular_file_unaffected(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
+        f = tmp_path / "ok.txt"
+        f.write_text("alpha\nbeta\n")
+        result = json.loads(read_file_tool(str(f)))
+        assert result.get("success", True) is not False
+        assert "alpha" in result.get("content", "")
diff --git a/tools/file_tools.py b/tools/file_tools.py
index deb55134f542a..2349c41623590 100644
--- a/tools/file_tools.py
+++ b/tools/file_tools.py
@@ -1515,6 +1515,38 @@ def clear_file_ops_cache(task_id: str = None):
             _file_ops_cache.clear()
 
 
+def _special_file_kind(path) -> str | None:
+    """Return a human name for non-regular file types that block reads.
+
+    Stat-based sibling of the name-based ``_is_blocked_device`` guard: a
+    FIFO at ``logs/live.pipe`` or a socket in a workspace hangs ``read_file``
+    just as hard as ``/dev/zero``, but carries no recognizable name. Only
+    called for host-visible filesystems (see ``_file_ops_uses_host_paths``);
+    remote backends cannot be statted from here.
+
+    Returns None for regular files, missing paths, and anything unstattable
+    (those flow to the normal read path and its own error handling).
+    """
+    import stat as _stat
+
+    try:
+        st = os.stat(os.fspath(path))  # follows symlinks, matching a real read
+    except OSError:
+        return None
+    mode = st.st_mode
+    if _stat.S_ISREG(mode) or _stat.S_ISDIR(mode):
+        return None
+    if _stat.S_ISFIFO(mode):
+        return "a FIFO (named pipe)"
+    if _stat.S_ISSOCK(mode):
+        return "a socket"
+    if _stat.S_ISCHR(mode):
+        return "a character device"
+    if _stat.S_ISBLK(mode):
+        return "a block device"
+    return "a special (non-regular) file"
+
+
 def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = "default") -> str:
     """Read a file with pagination and line numbers."""
     try:
@@ -1532,6 +1564,23 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str =
 
         _resolved = _resolve_path_for_task(path, task_id)
 
+        # ── Special-file type guard (stat-based) ──────────────────────
+        # The name blocklist above catches /dev/* and /proc/* aliases; this
+        # catches the class — any FIFO/socket/device wherever it lives. A
+        # read on a FIFO blocks until the exec timeout: a self-shipped DoS.
+        if _file_ops_uses_host_paths(_get_file_ops(task_id)):
+            kind = _special_file_kind(_resolved)
+            if kind is not None:
+                return json.dumps({
+                    "success": False,
+                    "note": (
+                        f"'{path}' is {kind}, not a regular file — reading "
+                        "it would block indefinitely, so no read was "
+                        "attempted. Use terminal utilities if you need to "
+                        "interact with it."
+                    ),
+                })
+
         # ── Structured-document extraction ────────────────────────────
         # Try before the binary-extension guard so .docx/.xlsx can render as text.
         # Malformed documents fall through to the normal path/binary guard.