soup/tests/test_v0610_part_e.py

"""Tests for v0.61.0 Part E — `soup edit diff` (knowledge-injection diff).

Coverage:
- ``DiffReport`` / ``FactChange`` frozen dataclasses.
- ``load_probes`` JSONL loader (cwd-contained, symlink-rejected).
- ``build_diff_report`` schema-only orchestrator.
- ``render_diff_table`` + ``write_diff_report``.
- ``soup edit diff`` CLI smoke.
"""

from __future__ import annotations

import dataclasses
import json
import os
import sys
from pathlib import Path

import pytest
from typer.testing import CliRunner

from soup_cli.cli import app


class TestDataclasses:
    def test_imports(self):
        from soup_cli.utils.edit_diff import (
            DiffReport,
            FactChange,
            build_diff_report,
            load_probes,
            render_diff_table,
            write_diff_report,
        )
        assert callable(build_diff_report)
        assert callable(load_probes)
        assert callable(render_diff_table)
        assert callable(write_diff_report)
        assert dataclasses.is_dataclass(DiffReport)
        assert dataclasses.is_dataclass(FactChange)

    def test_fact_change_frozen(self):
        from soup_cli.utils.edit_diff import FactChange

        c = FactChange(
            prompt="p", before="b", after="a", changed=True,
        )
        with pytest.raises(dataclasses.FrozenInstanceError):
            c.prompt = "x"  # type: ignore

    def test_diff_report_frozen(self):
        from soup_cli.utils.edit_diff import DiffReport, FactChange

        r = DiffReport(
            before_run_id="b",
            after_run_id="a",
            changes=(FactChange(prompt="p", before="b", after="a", changed=False),),
            total_probes=1,
            soup_version="0.61.0",
        )
        with pytest.raises(dataclasses.FrozenInstanceError):
            r.total_probes = 999  # type: ignore

    def test_diff_report_validates_changes_tuple(self):
        from soup_cli.utils.edit_diff import DiffReport

        with pytest.raises(TypeError):
            DiffReport(
                before_run_id="b",
                after_run_id="a",
                changes=[],  # list, not tuple
                total_probes=0,
                soup_version="0.61.0",
            )

    def test_diff_report_to_dict(self):
        from soup_cli.utils.edit_diff import DiffReport, FactChange

        r = DiffReport(
            before_run_id="b",
            after_run_id="a",
            changes=(
                FactChange(prompt="p1", before="x", after="y", changed=True),
            ),
            total_probes=1,
            soup_version="0.61.0",
        )
        d = r.to_dict()
        assert d["before_run_id"] == "b"
        assert d["after_run_id"] == "a"
        assert d["total_probes"] == 1
        assert d["changes"][0]["prompt"] == "p1"
        # JSON round-trip
        assert json.loads(json.dumps(d)) == d

    def test_diff_report_total_probes_bool_rejected(self):
        from soup_cli.utils.edit_diff import DiffReport

        with pytest.raises(TypeError):
            DiffReport(
                before_run_id="b",
                after_run_id="a",
                changes=(),
                total_probes=True,  # type: ignore
                soup_version="0.61.0",
            )


class TestLoadProbes:
    def _write_jsonl(self, path: Path, rows: list[dict]) -> None:
        path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")

    def test_happy_path(self, tmp_path):
        from soup_cli.utils.edit_diff import load_probes

        p = tmp_path / "probes.jsonl"
        self._write_jsonl(p, [{"prompt": "Who is X?"}, {"prompt": "Who is Y?"}])

        # Need to be in cwd
        old = os.getcwd()
        os.chdir(tmp_path)
        try:
            probes = load_probes("probes.jsonl")
        finally:
            os.chdir(old)
        assert probes == ("Who is X?", "Who is Y?")

    def test_missing_file(self, tmp_path):
        from soup_cli.utils.edit_diff import load_probes

        old = os.getcwd()
        os.chdir(tmp_path)
        try:
            with pytest.raises(FileNotFoundError):
                load_probes("missing.jsonl")
        finally:
            os.chdir(old)

    def test_skips_malformed_rows(self, tmp_path):
        from soup_cli.utils.edit_diff import load_probes

        p = tmp_path / "probes.jsonl"
        p.write_text(
            '{"prompt": "good"}\n'
            'this is not json\n'
            '{"prompt": ""}\n'
            '{"prompt": "another good"}\n',
            encoding="utf-8",
        )

        old = os.getcwd()
        os.chdir(tmp_path)
        try:
            probes = load_probes("probes.jsonl")
        finally:
            os.chdir(old)
        assert probes == ("good", "another good")

    def test_outside_cwd_rejected(self, tmp_path):
        from soup_cli.utils.edit_diff import load_probes

        out = tmp_path / "outside.jsonl"
        out.write_text('{"prompt": "p"}', encoding="utf-8")

        # cwd is the original; tmp_path is unrelated
        cwd_dir = tmp_path / "cwd"
        cwd_dir.mkdir()
        old = os.getcwd()
        os.chdir(cwd_dir)
        try:
            with pytest.raises(ValueError, match="cwd"):
                load_probes(str(out))
        finally:
            os.chdir(old)

    def test_null_byte_rejected(self):
        from soup_cli.utils.edit_diff import load_probes

        with pytest.raises(ValueError):
            load_probes("probe\x00.jsonl")

    def test_empty_path_rejected(self):
        from soup_cli.utils.edit_diff import load_probes

        with pytest.raises(ValueError):
            load_probes("")

    @pytest.mark.skipif(sys.platform == "win32", reason="POSIX-only symlink test")
    def test_symlink_rejected(self, tmp_path):
        from soup_cli.utils.edit_diff import load_probes

        real = tmp_path / "real.jsonl"
        real.write_text('{"prompt": "p"}', encoding="utf-8")
        link = tmp_path / "link.jsonl"
        os.symlink(real, link)

        old = os.getcwd()
        os.chdir(tmp_path)
        try:
            with pytest.raises(ValueError, match="symlink"):
                load_probes("link.jsonl")
        finally:
            os.chdir(old)


class TestBuildDiffReport:
    def test_happy_path(self):
        from soup_cli.utils.edit_diff import build_diff_report

        report = build_diff_report(
            before_run_id="before-run",
            after_run_id="after-run",
        )
        assert report.before_run_id == "before-run"
        assert report.after_run_id == "after-run"
        assert report.changes == ()
        assert report.total_probes == 0

    def test_same_run_id_rejected(self):
        from soup_cli.utils.edit_diff import build_diff_report

        with pytest.raises(ValueError, match="differ"):
            build_diff_report(
                before_run_id="x",
                after_run_id="x",
            )

    def test_bool_run_id_rejected(self):
        from soup_cli.utils.edit_diff import build_diff_report

        with pytest.raises(TypeError):
            build_diff_report(
                before_run_id=True,  # type: ignore
                after_run_id="x",
            )

    def test_top_k_bool_rejected(self):
        from soup_cli.utils.edit_diff import build_diff_report

        with pytest.raises(TypeError):
            build_diff_report(
                before_run_id="b",
                after_run_id="a",
                top_k=True,  # type: ignore
            )

    def test_top_k_out_of_range(self):
        from soup_cli.utils.edit_diff import build_diff_report

        with pytest.raises(ValueError):
            build_diff_report(
                before_run_id="b", after_run_id="a", top_k=0,
            )
        with pytest.raises(ValueError):
            build_diff_report(
                before_run_id="b", after_run_id="a", top_k=200,
            )

    def test_with_probe_file(self, tmp_path):
        from soup_cli.utils.edit_diff import build_diff_report

        p = tmp_path / "probes.jsonl"
        p.write_text(
            '{"prompt": "q1"}\n{"prompt": "q2"}\n{"prompt": "q3"}\n',
            encoding="utf-8",
        )
        old = os.getcwd()
        os.chdir(tmp_path)
        try:
            r = build_diff_report(
                before_run_id="b",
                after_run_id="a",
                probe_file="probes.jsonl",
                top_k=2,
            )
        finally:
            os.chdir(old)
        assert r.total_probes == 3
        assert len(r.changes) == 2  # capped by top_k


class TestRenderDiffTable:
    def test_renders(self, capsys):
        from rich.console import Console

        from soup_cli.utils.edit_diff import (
            DiffReport,
            FactChange,
            render_diff_table,
        )

        r = DiffReport(
            before_run_id="b",
            after_run_id="a",
            changes=(
                FactChange(prompt="p1", before="x", after="y", changed=True),
            ),
            total_probes=1,
            soup_version="0.61.0",
        )
        console = Console(force_terminal=False, width=200)
        render_diff_table(r, console)
        # No raise = pass.

    def test_non_report_rejected(self):
        from rich.console import Console

        from soup_cli.utils.edit_diff import render_diff_table

        with pytest.raises(TypeError):
            render_diff_table("not a report", Console())  # type: ignore


class TestWriteDiffReport:
    def test_happy_path(self, tmp_path):
        from soup_cli.utils.edit_diff import (
            DiffReport,
            write_diff_report,
        )

        r = DiffReport(
            before_run_id="b",
            after_run_id="a",
            changes=(),
            total_probes=0,
            soup_version="0.61.0",
        )
        old = os.getcwd()
        os.chdir(tmp_path)
        try:
            out = "report.json"
            write_diff_report(r, out)
            assert Path(out).exists()
            data = json.loads(Path(out).read_text())
            assert data["before_run_id"] == "b"
        finally:
            os.chdir(old)

    def test_outside_cwd_rejected(self, tmp_path):
        from soup_cli.utils.edit_diff import (
            DiffReport,
            write_diff_report,
        )

        r = DiffReport(
            before_run_id="b",
            after_run_id="a",
            changes=(),
            total_probes=0,
            soup_version="0.61.0",
        )
        out = tmp_path / "outside.json"
        cwd_dir = tmp_path / "cwd"
        cwd_dir.mkdir()
        old = os.getcwd()
        os.chdir(cwd_dir)
        try:
            with pytest.raises(ValueError):
                write_diff_report(r, str(out))
        finally:
            os.chdir(old)

    def test_non_report_rejected(self, tmp_path):
        from soup_cli.utils.edit_diff import write_diff_report

        with pytest.raises(TypeError):
            write_diff_report("not a report", "out.json")  # type: ignore


class TestCli:
    def test_help(self):
        runner = CliRunner()
        result = runner.invoke(app, ["edit", "diff", "--help"])
        assert result.exit_code == 0, result.output

    def test_basic_diff(self, tmp_path):
        runner = CliRunner()
        with runner.isolated_filesystem(temp_dir=tmp_path):
            result = runner.invoke(app, [
                "edit", "diff", "before-run", "after-run",
            ])
            assert result.exit_code == 0, result.output

    def test_same_runs_rejected(self, tmp_path):
        runner = CliRunner()
        with runner.isolated_filesystem(temp_dir=tmp_path):
            result = runner.invoke(app, [
                "edit", "diff", "same", "same",
            ])
            assert result.exit_code != 0

    def test_writes_output(self, tmp_path):
        runner = CliRunner()
        with runner.isolated_filesystem(temp_dir=tmp_path) as fs:
            out = Path(fs) / "diff.json"
            result = runner.invoke(app, [
                "edit", "diff", "before-run", "after-run",
                "--output", str(out),
            ])
            assert result.exit_code == 0, result.output
            assert out.exists()
            data = json.loads(out.read_text())
            assert data["total_probes"] == 0