"""v0.56.0 — `soup diagnose` post-training model report card.

Six failure-mode probes (forgetting / refusal / format / mode_collapse /
memorization / contamination) + FailureReport + SVG badge + CLI + train
--diagnose-gate. All probes are tested with caller-supplied generator
closures so the suite is GPU-free.
"""

from __future__ import annotations

import dataclasses
import json
import os
import re
from pathlib import Path

import pytest
from typer.testing import CliRunner

from soup_cli import __version__
from soup_cli.cli import app
from soup_cli.utils.diagnose import (
    FAILURE_MODES,
    FailureReport,
    FailureScore,
    classify_score,
    compose_report,
    overall_verdict,
)
from soup_cli.utils.diagnose.badge import render_badge_svg
from soup_cli.utils.diagnose.contamination import score_contamination
from soup_cli.utils.diagnose.forgetting import score_forgetting
from soup_cli.utils.diagnose.format import (
    is_valid_json,
    is_valid_tool_call,
    matches_regex,
    score_format,
)
from soup_cli.utils.diagnose.memorization import score_memorization, split_prefix
from soup_cli.utils.diagnose.mode_collapse import score_mode_collapse
from soup_cli.utils.diagnose.refusal import looks_like_refusal, score_refusal
from soup_cli.utils.diagnose.report import THRESHOLDS
from soup_cli.utils.diagnose.report import classify_score as classify_v2
from soup_cli.utils.diagnose.runner import (
    build_report,
    write_report,
)
from soup_cli.utils.diagnose.runner import (
    diagnose as diagnose_sdk,
)

runner = CliRunner()
# Capture project root at import time so source-grep tests survive the
# tmp_path os.chdir calls earlier in the suite.
_PROJECT_ROOT = Path(__file__).resolve().parent.parent

# Rich's CliRunner output carries ANSI escapes on CI; strip before
# substring assertions because Rich wraps long-option strings — e.g.
# `--badge` is rendered as `-\x1b[0m\x1b[1;36m-badge`, breaking a naive
# `"--badge" in result.output` check (v0.55.0 CI fix policy).
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")


def _strip_ansi(text: str) -> str:
    return _ANSI_RE.sub("", text or "")


# --- report dataclasses + classify_score ----------------------------------


class TestClassifyScore:
    @pytest.mark.parametrize(
        "score,expected",
        [(1.0, "OK"), (0.85, "OK"), (0.84, "MINOR"), (0.60, "MINOR"),
         (0.59, "MAJOR"), (0.0, "MAJOR")],
    )
    def test_thresholds(self, score: float, expected: str) -> None:
        assert classify_score(score) == expected
        assert classify_v2(score) == expected

    @pytest.mark.parametrize("bad", [True, False])
    def test_rejects_bool(self, bad: object) -> None:
        with pytest.raises(TypeError):
            classify_score(bad)

    def test_rejects_nan(self) -> None:
        with pytest.raises(ValueError, match="finite"):
            classify_score(float("nan"))

    def test_rejects_inf(self) -> None:
        with pytest.raises(ValueError, match="finite"):
            classify_score(float("inf"))

    @pytest.mark.parametrize("bad", [-0.01, 1.01, 2.0, -1.0])
    def test_rejects_out_of_range(self, bad: float) -> None:
        with pytest.raises(ValueError, match=r"\[0, 1\]"):
            classify_score(bad)

    def test_rejects_non_numeric(self) -> None:
        with pytest.raises(TypeError):
            classify_score("0.5")  # type: ignore[arg-type]

    def test_thresholds_proxy_keys(self) -> None:
        assert THRESHOLDS["ok"] == 0.85
        assert THRESHOLDS["minor"] == 0.60
        with pytest.raises(TypeError):
            THRESHOLDS["ok"] = 0.5  # type: ignore[index]


class TestFailureScore:
    def test_frozen(self) -> None:
        sc = FailureScore(mode="forgetting", score=1.0, verdict="OK", evidence="x")
        with pytest.raises(dataclasses.FrozenInstanceError):
            sc.score = 0.5  # type: ignore[misc]

    def test_unknown_mode_rejected(self) -> None:
        with pytest.raises(ValueError, match="unknown failure mode"):
            FailureScore(mode="not_a_mode", score=1.0, verdict="OK", evidence="x")

    def test_verdict_must_match_score(self) -> None:
        with pytest.raises(ValueError, match="disagrees"):
            FailureScore(
                mode="forgetting", score=0.10, verdict="OK", evidence="x"
            )

    def test_evidence_null_byte(self) -> None:
        with pytest.raises(ValueError, match="null"):
            FailureScore(
                mode="forgetting", score=1.0, verdict="OK", evidence="x\x00y"
            )

    def test_evidence_oversize(self) -> None:
        with pytest.raises(ValueError, match="too long"):
            FailureScore(
                mode="forgetting", score=1.0, verdict="OK", evidence="a" * 5000
            )

    def test_evidence_must_be_str(self) -> None:
        with pytest.raises(TypeError):
            FailureScore(
                mode="forgetting", score=1.0, verdict="OK", evidence=123  # type: ignore[arg-type]
            )


class TestFailureReport:
    def _scores(self) -> dict:
        return {
            mode: FailureScore(mode=mode, score=1.0, verdict="OK", evidence="ok")
            for mode in FAILURE_MODES
        }

    def test_compose_and_overall(self) -> None:
        report = compose_report(
            run_id="r1", base="b", adapter="a", scores=self._scores()
        )
        assert report.overall == "OK"
        assert set(report.scores.keys()) == set(FAILURE_MODES)

    def test_overall_major_wins(self) -> None:
        scores = self._scores()
        scores["refusal"] = FailureScore(
            mode="refusal", score=0.10, verdict="MAJOR", evidence="bad"
        )
        report = compose_report(run_id="r1", base="b", adapter="a", scores=scores)
        assert report.overall == "MAJOR"

    def test_overall_minor_promotes(self) -> None:
        scores = self._scores()
        scores["format"] = FailureScore(
            mode="format", score=0.70, verdict="MINOR", evidence="meh"
        )
        report = compose_report(run_id="r1", base="b", adapter="a", scores=scores)
        assert report.overall == "MINOR"

    def test_unknown_mode_key_rejected(self) -> None:
        scores = self._scores()
        bad = FailureScore(mode="forgetting", score=1.0, verdict="OK", evidence="x")
        # Build a dict whose KEY says "alien" but value is a real mode.
        scores_bad = dict(scores)
        scores_bad["alien"] = bad
        with pytest.raises(ValueError, match="unknown failure mode key"):
            compose_report(run_id="r1", base="b", adapter="a", scores=scores_bad)

    def test_score_mode_mismatch_rejected(self) -> None:
        scores = self._scores()
        scores["forgetting"] = FailureScore(
            mode="refusal", score=1.0, verdict="OK", evidence="x"
        )
        with pytest.raises(ValueError, match="mismatch"):
            compose_report(run_id="r1", base="b", adapter="a", scores=scores)

    def test_null_byte_in_run_id_rejected(self) -> None:
        with pytest.raises(ValueError, match="null"):
            compose_report(run_id="r1\x00", base="b", adapter="a", scores=self._scores())

    def test_oversize_base_rejected(self) -> None:
        with pytest.raises(ValueError, match="too long"):
            compose_report(
                run_id="r1", base="x" * 1000, adapter="a", scores=self._scores()
            )

    def test_frozen(self) -> None:
        report = compose_report(
            run_id="r1", base="b", adapter="a", scores=self._scores()
        )
        with pytest.raises(dataclasses.FrozenInstanceError):
            report.overall = "MAJOR"  # type: ignore[misc]

    def test_to_dict_serialisable(self) -> None:
        report = compose_report(
            run_id="r1", base="b", adapter="a", scores=self._scores()
        )
        payload = report.to_dict()
        # Round-trip through json + allow_nan=False.
        json.dumps(payload, allow_nan=False)
        assert payload["overall"] == "OK"
        assert set(payload["scores"]) == set(FAILURE_MODES)

    def test_scores_immutable(self) -> None:
        report = compose_report(
            run_id="r1", base="b", adapter="a", scores=self._scores()
        )
        with pytest.raises(TypeError):
            report.scores["forgetting"] = "evil"  # type: ignore[index]


class TestOverallVerdict:
    def test_empty_ok(self) -> None:
        assert overall_verdict({}) == "OK"

    def test_non_mapping_rejected(self) -> None:
        with pytest.raises(TypeError):
            overall_verdict([])  # type: ignore[arg-type]

    def test_non_score_entry_rejected(self) -> None:
        with pytest.raises(TypeError):
            overall_verdict({"forgetting": "OK"})  # type: ignore[dict-item]


# --- 6 probe modules -------------------------------------------------------


class TestForgetting:
    def test_no_shared_tasks_returns_ok(self) -> None:
        score = score_forgetting({"mmlu": 0.5}, {"hellaswag": 0.5})
        assert score.verdict == "OK"
        assert "no shared tasks" in score.evidence

    def test_perfect_preservation(self) -> None:
        score = score_forgetting({"mmlu": 0.5}, {"mmlu": 0.5})
        assert score.verdict == "OK"
        assert score.score == pytest.approx(1.0)

    def test_within_tolerance(self) -> None:
        score = score_forgetting({"mmlu": 0.5}, {"mmlu": 0.49}, tolerance=0.02)
        assert score.verdict == "OK"

    def test_major_regression(self) -> None:
        score = score_forgetting({"mmlu": 0.9}, {"mmlu": 0.1}, tolerance=0.01)
        assert score.verdict == "MAJOR"
        assert "mmlu" in score.evidence

    def test_bool_tolerance_rejected(self) -> None:
        with pytest.raises(TypeError):
            score_forgetting({"mmlu": 0.5}, {"mmlu": 0.5}, tolerance=True)

    def test_tolerance_out_of_range(self) -> None:
        with pytest.raises(ValueError, match=r"\[0, 1\]"):
            score_forgetting({"mmlu": 0.5}, {"mmlu": 0.5}, tolerance=2.0)

    def test_non_mapping_rejected(self) -> None:
        with pytest.raises(TypeError):
            score_forgetting([], {"mmlu": 0.5})  # type: ignore[arg-type]


class TestRefusal:
    def test_refusal_detector(self) -> None:
        assert looks_like_refusal("I cannot help with that.")
        assert looks_like_refusal("Sorry, but I am unable to comply.")
        assert looks_like_refusal("As an AI language model, I refuse to answer.")
        assert not looks_like_refusal("Sure! Here is the answer.")
        assert not looks_like_refusal(None)  # type: ignore[arg-type]
        assert not looks_like_refusal("ok\x00bad")

    def test_perfect_match(self) -> None:
        # Both base and adapter refuse on every harmful prompt and answer
        # every benign prompt — zero delta.
        base = lambda p: "I cannot help" if "harm" in p else "Sure!"  # noqa: E731
        adapter = base
        score = score_refusal(["harm1", "harm2"], ["ben1", "ben2"], base, adapter)
        assert score.verdict == "OK"
        assert score.score == pytest.approx(1.0)

    def test_major_drift(self) -> None:
        base = lambda p: "I cannot help"  # noqa: E731
        adapter = lambda p: "Sure!"  # noqa: E731
        score = score_refusal(["x"], ["y"], base, adapter)
        # Both harmful and benign deltas = 1.0 → score 0.0
        assert score.verdict == "MAJOR"

    def test_generator_must_return_str(self) -> None:
        with pytest.raises(TypeError):
            score_refusal(["x"], [], lambda p: 42, lambda p: "ok")  # type: ignore[return-value]


class TestFormat:
    def test_json_valid_detector(self) -> None:
        assert is_valid_json('{"a": 1}')
        assert not is_valid_json("{not json}")
        assert not is_valid_json("x\x00")
        assert not is_valid_json(None)  # type: ignore[arg-type]

    def test_regex_detector(self) -> None:
        assert matches_regex("abc123", r"\d+")
        assert not matches_regex("abc", r"\d+")
        assert not matches_regex("ok", "(?:")  # bad regex → False

    def test_tool_call_detector(self) -> None:
        assert is_valid_tool_call('{"tool_calls": [{"name": "x"}]}')
        assert not is_valid_tool_call('{"tool_calls": []}')
        assert not is_valid_tool_call('{"other": 1}')

    def test_score_json(self) -> None:
        score = score_format(["p1", "p2"], lambda p: '{"x": 1}', kind="json")
        assert score.verdict == "OK"
        assert score.score == pytest.approx(1.0)

    def test_score_regex_requires_pattern(self) -> None:
        with pytest.raises(ValueError, match="regex_pattern"):
            score_format(["p"], lambda p: "x", kind="regex")

    def test_unknown_kind_rejected(self) -> None:
        with pytest.raises(ValueError, match="kind must be"):
            score_format(["p"], lambda p: "x", kind="alien")

    def test_empty_prompts_ok(self) -> None:
        score = score_format([], lambda p: "x", kind="json")
        assert score.verdict == "OK"
        assert "nothing to check" in score.evidence

    def test_major_when_all_invalid(self) -> None:
        score = score_format(["p"] * 5, lambda p: "not json", kind="json")
        assert score.verdict == "MAJOR"


class TestModeCollapse:
    def test_diverse_outputs_ok(self) -> None:
        templates = [
            "the quick brown fox jumps over lazy dogs",
            "I prefer my coffee strong with two sugars",
            "yesterday morning a strange parcel arrived early",
            "rocket launches require months of careful preparation",
        ]

        def multi(_prompt: str, k: int) -> list:
            return templates[:k]

        score = score_mode_collapse(["p"], multi, k=4, ngram_n=2)
        assert score.verdict == "OK"

    def test_collapsed_outputs_major(self) -> None:
        multi = lambda p, k: ["same exact reply here"] * k  # noqa: E731
        score = score_mode_collapse(["p"], multi, k=4)
        assert score.verdict == "MAJOR"

    def test_k_must_be_two_plus(self) -> None:
        with pytest.raises(ValueError, match=r"k must be in"):
            score_mode_collapse(["p"], lambda p, k: ["x"], k=1)

    def test_bool_k_rejected(self) -> None:
        with pytest.raises(TypeError):
            score_mode_collapse(["p"], lambda p, k: ["x", "y"], k=True)

    def test_must_be_callable(self) -> None:
        with pytest.raises(TypeError):
            score_mode_collapse(["p"], "not_callable")  # type: ignore[arg-type]

    def test_generator_must_return_sequence(self) -> None:
        with pytest.raises(TypeError):
            score_mode_collapse(
                ["p"], lambda p, k: "string_not_seq"
            )  # type: ignore[return-value]

    def test_empty_prompts(self) -> None:
        score = score_mode_collapse([], lambda p, k: ["x", "y"], k=2)
        assert score.verdict == "OK"


class TestMemorization:
    def test_split_prefix(self) -> None:
        prefix, suffix = split_prefix("one two three four", fraction=0.5)
        assert prefix.split() == ["one", "two"]
        assert suffix.split() == ["three", "four"]

    def test_split_empty(self) -> None:
        assert split_prefix("") == ("", "")

    def test_no_memorization(self) -> None:
        rows = [{"text": "alpha beta gamma delta epsilon"}]
        gen = lambda p: "completely unrelated reply here"  # noqa: E731
        score = score_memorization(rows, gen, prefix_fraction=0.4)
        assert score.verdict == "OK"

    def test_full_memorization(self) -> None:
        rows = [{"text": "alpha beta gamma delta epsilon zeta"}]
        # Generator echoes the suffix verbatim → MAJOR.
        gen = lambda p: "gamma delta epsilon zeta"  # noqa: E731
        score = score_memorization(rows, gen, prefix_fraction=0.4, echo_threshold=0.5)
        assert score.verdict == "MAJOR"

    def test_skips_rows_without_text(self) -> None:
        rows = [{"not_text": "x"}, "not_a_dict"]
        score = score_memorization(rows, lambda p: "x")
        assert score.verdict == "OK"
        assert "no rows" in score.evidence

    def test_too_many_rows_rejected(self) -> None:
        with pytest.raises(ValueError, match="too many"):
            score_memorization([{"text": "x"}] * 5_001, lambda p: "x")


class TestContamination:
    def test_clean(self) -> None:
        training = [{"text": "unique training content alpha beta gamma"}]
        benchmark = ["totally different benchmark text here"]
        score = score_contamination(training, benchmark, n=3, threshold=0.5)
        assert score.verdict == "OK"

    def test_contaminated(self) -> None:
        text = "alpha beta gamma delta epsilon zeta eta theta iota kappa"
        training = [{"text": text}]
        benchmark = [text]  # identical
        score = score_contamination(training, benchmark, n=3, threshold=0.5)
        assert score.verdict == "MAJOR"

    def test_empty_benchmark_ok(self) -> None:
        score = score_contamination([{"text": "x"}], [])
        assert score.verdict == "OK"

    def test_benchmark_dict_rows(self) -> None:
        text = "alpha beta gamma delta epsilon zeta eta theta iota kappa"
        score = score_contamination(
            [{"text": text}], [{"text": text}], n=3, threshold=0.5
        )
        assert score.verdict == "MAJOR"

    def test_bool_n_rejected(self) -> None:
        with pytest.raises(TypeError):
            score_contamination([{"text": "x"}], ["y"], n=True)

    def test_oversize_training_rejected(self) -> None:
        with pytest.raises(ValueError, match="too many training"):
            score_contamination([{"text": "x"}] * 100_001, [{"text": "y"}])

    def test_no_scannable_training(self) -> None:
        score = score_contamination([{"not_text": "x"}], ["y"], n=3)
        assert score.verdict == "OK"


# --- runner + write_report -------------------------------------------------


class TestRunner:
    def test_build_report_fills_missing(self) -> None:
        scores = {
            "forgetting": FailureScore(
                mode="forgetting", score=1.0, verdict="OK", evidence="x"
            )
        }
        report = build_report(
            run_id="r1", base="b", adapter="a", scores=scores
        )
        assert set(report.scores.keys()) == set(FAILURE_MODES)
        for mode in FAILURE_MODES:
            if mode != "forgetting":
                assert "probe not run" in report.scores[mode].evidence

    def test_scores_type_validated(self) -> None:
        with pytest.raises(TypeError):
            build_report(
                run_id="r1", base="b", adapter="a",
                scores={"forgetting": "not a score"},  # type: ignore[dict-item]
            )

    def test_diagnose_sdk(self) -> None:
        report = diagnose_sdk(run_id="r1", base="b", adapter="a")
        assert isinstance(report, FailureReport)
        assert report.overall == "OK"

    def test_write_report_atomic(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        report = diagnose_sdk(run_id="r1", base="b", adapter="a")
        path = tmp_path / "diagnose.json"
        result = write_report(report, str(path))
        assert os.path.exists(result)
        with open(result, encoding="utf-8") as handle:
            payload = json.load(handle)
        assert payload["run_id"] == "r1"
        assert set(payload["scores"]) == set(FAILURE_MODES)

    def test_write_report_outside_cwd_rejected(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        report = diagnose_sdk(run_id="r1", base="b", adapter="a")
        outside = os.path.realpath(os.path.join(tmp_path, "..", "evil.json"))
        with pytest.raises(ValueError, match="cwd"):
            write_report(report, outside)

    @pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics")
    def test_write_report_symlink_rejected(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        target = tmp_path / "real.json"
        target.write_text("{}", encoding="utf-8")
        link = tmp_path / "link.json"
        os.symlink(target, link)
        report = diagnose_sdk(run_id="r1", base="b", adapter="a")
        with pytest.raises(ValueError, match="symlink"):
            write_report(report, str(link))


# --- badge SVG -------------------------------------------------------------


class TestBadge:
    def test_renders_svg(self) -> None:
        report = diagnose_sdk(run_id="r1", base="b", adapter="my-adapter")
        svg = render_badge_svg(report)
        assert svg.startswith("<svg ")
        assert "OK" in svg
        assert "my-adapter" in svg
        # All 6 modes appear as labels (with underscores → spaces).
        for mode in FAILURE_MODES:
            assert mode.replace("_", " ") in svg

    def test_escapes_user_text(self) -> None:
        report = compose_report(
            run_id="r1", base="b", adapter='<script>alert(1)</script>',
            scores={
                mode: FailureScore(mode=mode, score=1.0, verdict="OK", evidence="x")
                for mode in FAILURE_MODES
            },
        )
        svg = render_badge_svg(report)
        assert "<script>" not in svg
        assert "&lt;script&gt;" in svg

    def test_requires_report_type(self) -> None:
        with pytest.raises(TypeError):
            render_badge_svg({"not": "a report"})  # type: ignore[arg-type]


# --- registry artifact kind -----------------------------------------------


class TestRegistryKind:
    def test_diagnose_report_in_valid_kinds(self) -> None:
        from soup_cli.registry.store import _VALID_KINDS

        assert "diagnose_report" in _VALID_KINDS


# --- CLI -------------------------------------------------------------------


class TestCli:
    def test_diagnose_in_help(self) -> None:
        result = runner.invoke(app, ["--help"])
        assert result.exit_code == 0
        assert "diagnose" in _strip_ansi(result.output)

    def test_diagnose_help(self) -> None:
        result = runner.invoke(app, ["diagnose", "--help"])
        assert result.exit_code == 0
        assert "--badge" in _strip_ansi(result.output)
        assert "--output" in _strip_ansi(result.output)
        assert "--evidence" in _strip_ansi(result.output)

    def test_diagnose_neutral_run(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        result = runner.invoke(app, ["diagnose", "myrun"])
        assert result.exit_code == 0, (result.output, repr(result.exception))
        for mode in FAILURE_MODES:
            assert mode in _strip_ansi(result.output)

    def test_diagnose_writes_output(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        out = tmp_path / "diag.json"
        result = runner.invoke(
            app, ["diagnose", "myrun", "--output", str(out)]
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        with open(out, encoding="utf-8") as handle:
            payload = json.load(handle)
        assert payload["overall"] == "OK"

    def test_diagnose_badge_svg(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        badge = tmp_path / "diag.svg"
        result = runner.invoke(
            app, ["diagnose", "myrun", "--badge", str(badge)]
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))
        svg = badge.read_text(encoding="utf-8")
        assert svg.startswith("<svg")

    def test_diagnose_evidence_with_major_exit_2(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        evidence = tmp_path / "ev.json"
        evidence.write_text(
            json.dumps(
                {
                    "scores": {
                        "refusal": {
                            "score": 0.1,
                            "verdict": "MAJOR",
                            "evidence": "broke safety",
                        }
                    }
                }
            ),
            encoding="utf-8",
        )
        result = runner.invoke(
            app, ["diagnose", "myrun", "--evidence", str(evidence)]
        )
        # Exit 2 on MAJOR overall.
        assert result.exit_code == 2, (result.output, repr(result.exception))
        assert "MAJOR" in _strip_ansi(result.output)

    def test_diagnose_evidence_outside_cwd(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        outside = os.path.realpath(os.path.join(tmp_path, "..", "evil.json"))
        result = runner.invoke(
            app, ["diagnose", "myrun", "--evidence", outside]
        )
        assert result.exit_code == 1

    def test_diagnose_attach_to_registry_without_output_warns(
        self, tmp_path: Path
    ) -> None:
        os.chdir(tmp_path)
        result = runner.invoke(
            app, ["diagnose", "myrun", "--attach-to-registry", "abc"]
        )
        assert result.exit_code == 0
        assert "needs --output" in _strip_ansi(result.output)

    def test_diagnose_run_id_oversize(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        result = runner.invoke(app, ["diagnose", "x" * 600])
        assert result.exit_code != 0

    def test_diagnose_run_id_null_byte(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        result = runner.invoke(app, ["diagnose", "x\x00y"])
        assert result.exit_code != 0


# --- train --diagnose-gate ------------------------------------------------


class TestTrainDiagnoseGate:
    def test_help_lists_flag(self) -> None:
        result = runner.invoke(app, ["train", "--help"])
        assert result.exit_code == 0
        assert "--diagnose-gate" in _strip_ansi(result.output)

    def test_run_diagnose_gate_helper_major_exits(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        evidence = tmp_path / "ev.json"
        evidence.write_text(
            json.dumps(
                {
                    "scores": {
                        "format": {
                            "score": 0.05,
                            "verdict": "MAJOR",
                            "evidence": "json broken",
                        }
                    }
                }
            ),
            encoding="utf-8",
        )
        import typer

        from soup_cli.commands.train import _run_diagnose_gate

        with pytest.raises(typer.Exit) as excinfo:
            _run_diagnose_gate(str(evidence), "run1", "base", "adapter")
        assert excinfo.value.exit_code == 2

    def test_run_diagnose_gate_helper_ok_passes(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        evidence = tmp_path / "ev.json"
        evidence.write_text(
            json.dumps(
                {
                    "scores": {
                        "format": {
                            "score": 1.0,
                            "verdict": "OK",
                            "evidence": "all valid",
                        }
                    }
                }
            ),
            encoding="utf-8",
        )
        from soup_cli.commands.train import _run_diagnose_gate
        _run_diagnose_gate(str(evidence), "run1", "base", "adapter")

    def test_diagnose_gate_skips_nonzero_local_rank(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from soup_cli.commands.train import _should_run_diagnose_gate_on_rank

        monkeypatch.setenv("LOCAL_RANK", "1")
        assert _should_run_diagnose_gate_on_rank() is False

    def test_diagnose_gate_runs_on_rank_zero(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from soup_cli.commands.train import _should_run_diagnose_gate_on_rank

        monkeypatch.setenv("LOCAL_RANK", "0")
        assert _should_run_diagnose_gate_on_rank() is True

    def test_diagnose_gate_handles_malformed_local_rank(
        self, monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        """Garbage LOCAL_RANK falls back to True -- safer to over-run than skip."""
        from soup_cli.commands.train import _should_run_diagnose_gate_on_rank

        monkeypatch.setenv("LOCAL_RANK", "not-an-int")
        assert _should_run_diagnose_gate_on_rank() is True

    def test_run_diagnose_gate_rejects_non_dict_payload(
        self, tmp_path: Path
    ) -> None:
        os.chdir(tmp_path)
        evidence = tmp_path / "ev.json"
        evidence.write_text("[]", encoding="utf-8")
        from soup_cli.commands.train import _run_diagnose_gate

        with pytest.raises(ValueError, match="JSON object"):
            _run_diagnose_gate(str(evidence), "run1", "base", "adapter")


# --- source-grep regression guards ----------------------------------------


class TestSourceWiring:
    def test_cli_registers_diagnose(self) -> None:
        source = (_PROJECT_ROOT / "src" / "soup_cli" / "cli.py").read_text(encoding="utf-8")
        assert 'name="diagnose"' in source
        assert "_diagnose_cmd" in source

    def test_no_top_level_heavy_imports(self) -> None:
        # All probe + report modules must be torch/transformers-free
        # (citation.py added in v0.71.10 #202).
        probe_modules = [
            "src/soup_cli/utils/diagnose/__init__.py",
            "src/soup_cli/utils/diagnose/report.py",
            "src/soup_cli/utils/diagnose/_common.py",
            "src/soup_cli/utils/diagnose/forgetting.py",
            "src/soup_cli/utils/diagnose/refusal.py",
            "src/soup_cli/utils/diagnose/format.py",
            "src/soup_cli/utils/diagnose/mode_collapse.py",
            "src/soup_cli/utils/diagnose/memorization.py",
            "src/soup_cli/utils/diagnose/contamination.py",
            "src/soup_cli/utils/diagnose/citation.py",
            "src/soup_cli/utils/diagnose/badge.py",
            "src/soup_cli/utils/diagnose/runner.py",
        ]
        for relative in probe_modules:
            source = (_PROJECT_ROOT / relative).read_text(encoding="utf-8")
            for forbidden in ("\nimport torch", "\nimport transformers",
                              "\nfrom torch ", "\nfrom transformers "):
                assert forbidden not in source, (
                    f"{relative} carries a top-level {forbidden!r} import"
                )

    def test_version_at_or_above_0_56_0(self) -> None:
        # v0.57.0 widened this from exact-match to floor-check (matches
        # the v0.51.0 / v0.54.0 floor-test idiom — every subsequent release
        # would otherwise have to edit this single line).
        assert __version__ >= "0.56.0"

    def test_pyproject_version(self) -> None:
        import re

        text = (_PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8")
        # Floor check — v0.56.0 was the first release that needed this guard.
        # We accept any v0.<minor>.<patch> >= 0.56.0 so future releases (v0.6x,
        # v0.7x, ...) don't have to edit this line.
        m = re.search(r'^version = "(0\.\d+\.\d+)"', text, re.MULTILINE)
        assert m, "pyproject.toml missing version line"
        major, minor, _patch = m.group(1).split(".")
        assert int(major) == 0 and int(minor) >= 56, m.group(1)


# --- review-fix coverage --------------------------------------------------


class TestReviewFixCoverage:
    """Tests added per python / security / code / tdd review wave."""

    # python-review HIGH — write_report uses realpath, not abspath.
    def test_write_report_uses_realpath(self) -> None:
        source = (
            _PROJECT_ROOT / "src" / "soup_cli" / "utils" / "diagnose" / "runner.py"
        ).read_text(
            encoding="utf-8"
        )
        assert "os.path.realpath(path)" in source
        assert "os.path.abspath" not in source

    # code-review HIGH — neutral_score is the single source of truth.
    def test_neutral_score_centralised(self) -> None:
        from soup_cli.utils.diagnose.runner import neutral_score

        sc = neutral_score("forgetting", "skipped")
        assert sc.mode == "forgetting"
        assert sc.verdict == "OK"
        assert "skipped" in sc.evidence

    # code-review HIGH — sys.exit replaced with typer.Exit.
    def test_diagnose_uses_typer_exit_not_sys_exit(self) -> None:
        source = (_PROJECT_ROOT / "src" / "soup_cli" / "commands" / "diagnose.py").read_text(
            encoding="utf-8"
        )
        assert "sys.exit(" not in source
        assert "raise typer.Exit(code=2)" in source

    # security-review HIGH — atomic badge write with TOCTOU guard.
    def test_badge_write_atomic_and_symlink_safe(self) -> None:
        source = (_PROJECT_ROOT / "src" / "soup_cli" / "commands" / "diagnose.py").read_text(
            encoding="utf-8"
        )
        assert "_write_badge" in source
        assert "tempfile.mkstemp" in source
        assert "os.replace" in source

    @pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics")
    def test_badge_symlink_rejected(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        target = tmp_path / "real.svg"
        target.write_text("<svg/>", encoding="utf-8")
        link = tmp_path / "link.svg"
        os.symlink(target, link)
        from soup_cli.commands.diagnose import _write_badge

        with pytest.raises(ValueError, match="symlink"):
            _write_badge(str(link), "<svg/>")

    # security-review HIGH — evidence file size cap (16 MiB).
    def test_evidence_size_cap(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        # Use a tiny file but monkeypatch getsize to exceed the cap.
        ev = tmp_path / "ev.json"
        ev.write_text("{}", encoding="utf-8")
        from unittest.mock import patch

        from soup_cli.commands.diagnose import _load_evidence

        with patch("os.path.getsize", return_value=20 * 1024 * 1024):
            with pytest.raises(Exception, match="exceeds"):
                _load_evidence(str(ev))

    def test_train_diagnose_gate_size_cap(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        ev = tmp_path / "ev.json"
        ev.write_text("{}", encoding="utf-8")
        from unittest.mock import patch

        from soup_cli.commands.train import _run_diagnose_gate

        with patch("os.path.getsize", return_value=20 * 1024 * 1024):
            with pytest.raises(ValueError, match="exceeds 16 MiB"):
                _run_diagnose_gate(str(ev), "run1", "base", "adapter")

    # security-review MEDIUM — ReDoS probe in matches_regex.
    def test_matches_regex_invalid_pattern_rejected(self) -> None:
        # Invalid regex syntax → False (probe wrapped in try/except).
        assert not matches_regex("abc", "(?:")
        # Sane patterns still pass.
        assert matches_regex("abc123", r"\d+")
        # Source-grep — confirm the ReDoS probe wiring exists.
        source = (
            _PROJECT_ROOT / "src" / "soup_cli" / "utils" / "diagnose" / "format.py"
        ).read_text(
            encoding="utf-8"
        )
        assert 'compiled.search("a" * 128)' in source

    # security-review MEDIUM — looks_like_refusal caps input length.
    def test_refusal_input_capped(self) -> None:
        from soup_cli.utils.diagnose.refusal import _MAX_REFUSAL_SCAN

        assert _MAX_REFUSAL_SCAN == 8192
        # An adversarially huge input should still resolve quickly.
        huge = "ok " * 50_000
        assert isinstance(looks_like_refusal(huge), bool)

    # security-review MEDIUM — extras null-byte rejected.
    def test_evidence_extras_null_byte_rejected(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        ev = tmp_path / "ev.json"
        ev.write_text(
            json.dumps({"extras": {"key": "value\x00bad"}}),
            encoding="utf-8",
        )
        result = runner.invoke(
            app, ["diagnose", "myrun", "--evidence", str(ev)]
        )
        assert result.exit_code == 1
        assert "null bytes" in _strip_ansi(result.output)

    # python-review MEDIUM — math.isfinite on forgetting tolerance.
    def test_forgetting_nan_tolerance_rejected(self) -> None:
        with pytest.raises(ValueError, match="finite"):
            score_forgetting({"mmlu": 0.5}, {"mmlu": 0.5}, tolerance=float("nan"))

    def test_forgetting_inf_tolerance_rejected(self) -> None:
        with pytest.raises(ValueError, match="finite"):
            score_forgetting({"mmlu": 0.5}, {"mmlu": 0.5}, tolerance=float("inf"))

    # python-review MEDIUM — contamination combined-complexity cap.
    def test_contamination_combined_cap(self) -> None:
        from unittest.mock import MagicMock

        # Fake sequences whose len() advertises huge sizes.
        big_train = MagicMock()
        big_train.__len__ = lambda self: 1_000_001
        big_train.__iter__ = lambda self: iter([])
        big_bench = MagicMock()
        big_bench.__len__ = lambda self: 1001
        big_bench.__iter__ = lambda self: iter([])
        # We need real Sequence subclass; use a list trick instead.
        # Build two small lists but stub the product check.
        # Simpler: just verify the cap exists in source.
        source = (
            _PROJECT_ROOT / "src" / "soup_cli" / "utils" / "diagnose" / "contamination.py"
        ).read_text(
            encoding="utf-8"
        )
        assert "combined-complexity cap" in source
        assert "1_000_000_000" in source

    # code-review MEDIUM — _VALID_KINDS is frozenset.
    def test_format_valid_kinds_frozenset(self) -> None:
        from soup_cli.utils.diagnose.format import _VALID_KINDS

        assert isinstance(_VALID_KINDS, frozenset)

    # code-review MEDIUM — tokenize delegates to _eval_text.
    def test_tokenize_delegates(self) -> None:
        from soup_cli.utils._eval_text import tokenize as shared
        from soup_cli.utils.diagnose._common import tokenize as local

        # Both produce the same token sequence.
        text = "Hello World 123"
        assert list(local(text)) == list(shared(text))

    # code-review HIGH — extract_row_text shared across modules.
    def test_extract_row_text(self) -> None:
        from soup_cli.utils.diagnose._common import extract_row_text

        assert extract_row_text({"text": "x"}) == "x"
        assert extract_row_text({"content": "y"}) == "y"
        assert extract_row_text({"prompt": "z"}) == "z"
        assert extract_row_text({"instruction": "i"}) == "i"
        assert extract_row_text({"messages": [{"content": "hi"}, {"content": "bye"}]}) == "hi\nbye"
        assert extract_row_text({"not_text": "x"}) == ""
        assert extract_row_text("not_dict") == ""

    # tdd-review HIGH — write_report TypeError on non-FailureReport.
    def test_write_report_rejects_non_report(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        with pytest.raises(TypeError, match="FailureReport"):
            write_report({"not": "a report"}, str(tmp_path / "x.json"))  # type: ignore[arg-type]

    # tdd-review HIGH — split_prefix rejects bad fraction.
    def test_split_prefix_rejects_bad_fraction(self) -> None:
        with pytest.raises(ValueError, match="finite"):
            split_prefix("hello world", fraction=float("nan"))
        with pytest.raises(ValueError, match=r"\[0, 1\]"):
            split_prefix("hello world", fraction=2.0)
        with pytest.raises(TypeError):
            split_prefix("hello", fraction=True)

    # tdd-review HIGH — _run_diagnose_gate outside-cwd evidence rejected.
    def test_run_diagnose_gate_outside_cwd(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        outside = os.path.realpath(os.path.join(tmp_path, "..", "evil.json"))
        from soup_cli.commands.train import _run_diagnose_gate

        with pytest.raises(ValueError, match="cwd"):
            _run_diagnose_gate(outside, "r1", "base", "adapter")

    # tdd-review MEDIUM — CLI --output outside-cwd rejected.
    def test_cli_output_outside_cwd(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        outside = os.path.realpath(os.path.join(tmp_path, "..", "evil.json"))
        result = runner.invoke(
            app, ["diagnose", "myrun", "--output", outside]
        )
        assert result.exit_code == 1

    # tdd-review MEDIUM — CLI --badge outside-cwd rejected.
    def test_cli_badge_outside_cwd(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        outside = os.path.realpath(os.path.join(tmp_path, "..", "evil.svg"))
        result = runner.invoke(
            app, ["diagnose", "myrun", "--badge", outside]
        )
        assert result.exit_code == 1

    # tdd-review MEDIUM — MINOR overall exits 0, not 2.
    def test_minor_exits_zero(self, tmp_path: Path) -> None:
        os.chdir(tmp_path)
        ev = tmp_path / "ev.json"
        ev.write_text(
            json.dumps(
                {
                    "scores": {
                        "format": {
                            "score": 0.70,
                            "verdict": "MINOR",
                            "evidence": "borderline",
                        }
                    }
                }
            ),
            encoding="utf-8",
        )
        result = runner.invoke(
            app, ["diagnose", "myrun", "--evidence", str(ev)]
        )
        assert result.exit_code == 0, (result.output, repr(result.exception))

    # tdd-review LOW — build_report forwards soup_version + extras.
    def test_build_report_forwards_soup_version(self) -> None:
        report = build_report(
            run_id="r1", base="b", adapter="a",
            scores={}, soup_version="0.56.0",
            extras={"k": "v"},
        )
        assert report.soup_version == "0.56.0"
        assert report.extras["k"] == "v"


# --- end ------------------------------------------------------------------