soup/tests/test_v0660_part_b.py

"""v0.66.0 Part B — Live blame runner (DataInf influence-function approximation).

Closes v0.57.1 #171 — the `run_blame` stub in `utils/blame.py` now actually
returns per-row influence scores. The math is a DataInf-style first-order
approximation: per-row gradient L2 norm × cosine similarity to the held-out
probe gradient direction. We compute it pure-numpy on CPU using a tiny
synthetic adapter so the test exercises the runner end-to-end without GPUs.

Public surface under test:

- ``BlameResult`` frozen dataclass — top influencer rows + scores
- ``run_blame(plan, *, probe_fn=None)`` — live runner (no longer raises)
- ``compute_row_influence(row_grads, probe_grad)`` — math kernel
- ``render_blame_json`` / ``render_blame_markdown``
"""

from __future__ import annotations

import json

import numpy as np
import pytest


@pytest.fixture(autouse=True)
def _chdir(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    yield tmp_path


# ---------------------------------------------------------------------------
# Public surface
# ---------------------------------------------------------------------------


def test_module_imports_new_blame_surface():
    from soup_cli.utils import blame

    # v0.66 lifts the stub - same module name, new surface
    for name in (
        "BlameResult",
        "RowInfluence",
        "compute_row_influence",
        "run_blame",
        "render_blame_json",
        "render_blame_markdown",
    ):
        assert hasattr(blame, name), name


def test_run_blame_no_longer_raises_not_implemented_when_probe_supplied():
    """v0.57.0: NotImplementedError. v0.66.0: returns BlameResult."""
    from soup_cli.utils.blame import BlameResult, run_blame

    # Build minimal plan
    plan = _make_tiny_plan()
    # Supply a closure probe that returns synthetic row gradients
    result = run_blame(plan, probe_fn=_synthetic_probe(10, 4))
    assert isinstance(result, BlameResult)


# ---------------------------------------------------------------------------
# Math kernel — per-row influence
# ---------------------------------------------------------------------------


def test_compute_row_influence_zero_when_orthogonal():
    from soup_cli.utils.blame import compute_row_influence

    row_grad = np.array([1.0, 0.0, 0.0], dtype=np.float32)
    probe_grad = np.array([0.0, 1.0, 0.0], dtype=np.float32)
    score = compute_row_influence(row_grad, probe_grad)
    # Orthogonal -> 0
    assert abs(score) < 1e-6


def test_compute_row_influence_positive_when_aligned():
    from soup_cli.utils.blame import compute_row_influence

    row_grad = np.array([1.0, 2.0, 3.0], dtype=np.float32)
    probe_grad = np.array([1.0, 2.0, 3.0], dtype=np.float32)
    score = compute_row_influence(row_grad, probe_grad)
    assert score > 0.0


def test_compute_row_influence_negative_when_opposed():
    from soup_cli.utils.blame import compute_row_influence

    row_grad = np.array([1.0, 2.0, 3.0], dtype=np.float32)
    probe_grad = np.array([-1.0, -2.0, -3.0], dtype=np.float32)
    score = compute_row_influence(row_grad, probe_grad)
    assert score < 0.0


def test_compute_row_influence_rejects_shape_mismatch():
    from soup_cli.utils.blame import compute_row_influence

    a = np.zeros((5,), dtype=np.float32)
    b = np.zeros((10,), dtype=np.float32)
    with pytest.raises(ValueError, match="shape"):
        compute_row_influence(a, b)


def test_compute_row_influence_rejects_non_array():
    from soup_cli.utils.blame import compute_row_influence

    with pytest.raises(TypeError):
        compute_row_influence("bad", np.zeros((5,)))


def test_compute_row_influence_handles_zero_row_grad():
    from soup_cli.utils.blame import compute_row_influence

    row_grad = np.zeros((3,), dtype=np.float32)
    probe_grad = np.array([1.0, 2.0, 3.0], dtype=np.float32)
    # Zero row grad -> influence 0
    score = compute_row_influence(row_grad, probe_grad)
    assert score == 0.0


def test_compute_row_influence_handles_zero_probe_grad():
    from soup_cli.utils.blame import compute_row_influence

    row_grad = np.array([1.0, 2.0, 3.0], dtype=np.float32)
    probe_grad = np.zeros((3,), dtype=np.float32)
    score = compute_row_influence(row_grad, probe_grad)
    assert score == 0.0


# ---------------------------------------------------------------------------
# RowInfluence + BlameResult frozen dataclasses
# ---------------------------------------------------------------------------


def test_row_influence_frozen():
    from soup_cli.utils.blame import RowInfluence

    r = RowInfluence(row_id=0, score=0.5, shard_id=1)
    with pytest.raises((AttributeError, Exception)):
        r.score = 1.0  # type: ignore[misc]


def test_row_influence_rejects_negative_row_id():
    from soup_cli.utils.blame import RowInfluence

    with pytest.raises(ValueError):
        RowInfluence(row_id=-1, score=0.0, shard_id=0)


def test_row_influence_rejects_bool_score():
    from soup_cli.utils.blame import RowInfluence

    with pytest.raises(TypeError):
        RowInfluence(row_id=0, score=True, shard_id=0)


def test_row_influence_rejects_non_finite_score():
    from soup_cli.utils.blame import RowInfluence

    with pytest.raises(ValueError):
        RowInfluence(row_id=0, score=float("inf"), shard_id=0)


def test_blame_result_frozen():
    from soup_cli.utils.blame import BlameResult

    r = BlameResult(
        adapter_dir="x",
        dataset_path="y",
        layer="lm_head",
        top_influencers=tuple(),
        num_rows_scored=0,
        elapsed_seconds=0.0,
    )
    with pytest.raises((AttributeError, Exception)):
        r.num_rows_scored = 5  # type: ignore[misc]


def test_blame_result_rejects_negative_elapsed():
    from soup_cli.utils.blame import BlameResult

    with pytest.raises(ValueError):
        BlameResult(
            adapter_dir="x",
            dataset_path="y",
            layer="lm_head",
            top_influencers=tuple(),
            num_rows_scored=0,
            elapsed_seconds=-1.0,
        )


def test_blame_result_top_influencers_must_be_tuple():
    from soup_cli.utils.blame import BlameResult, RowInfluence

    rows = [RowInfluence(row_id=0, score=0.5, shard_id=0)]
    with pytest.raises(TypeError):
        BlameResult(
            adapter_dir="x",
            dataset_path="y",
            layer="lm_head",
            top_influencers=rows,  # list, not tuple
            num_rows_scored=1,
            elapsed_seconds=0.0,
        )


# ---------------------------------------------------------------------------
# Live runner
# ---------------------------------------------------------------------------


def test_run_blame_returns_top_influencers():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    probe = _synthetic_probe(20, 4)
    result = run_blame(plan, probe_fn=probe)
    assert result.num_rows_scored == 20
    assert len(result.top_influencers) > 0


def test_run_blame_sorts_by_absolute_score():
    """Top influencers are the rows with the largest |influence|."""
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    probe = _synthetic_probe(20, 4)
    result = run_blame(plan, probe_fn=probe)
    scores = [abs(r.score) for r in result.top_influencers]
    # Sorted descending by abs
    assert scores == sorted(scores, reverse=True)


def test_run_blame_caps_top_influencers_at_default():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    probe = _synthetic_probe(200, 4)
    result = run_blame(plan, probe_fn=probe)
    # Default top_k=50 per the planning doc
    assert len(result.top_influencers) <= 50


def test_run_blame_respects_top_k():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    probe = _synthetic_probe(100, 4)
    result = run_blame(plan, probe_fn=probe, top_k=10)
    assert len(result.top_influencers) == 10


def test_run_blame_rejects_non_plan():
    from soup_cli.utils.blame import run_blame

    with pytest.raises(TypeError):
        run_blame("not a plan", probe_fn=_synthetic_probe(1, 4))


def test_run_blame_rejects_non_callable_probe():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    with pytest.raises(TypeError):
        run_blame(plan, probe_fn="not a callable")


def test_run_blame_rejects_bool_top_k():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    with pytest.raises(TypeError):
        run_blame(plan, probe_fn=_synthetic_probe(5, 4), top_k=True)


def test_run_blame_rejects_zero_top_k():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    with pytest.raises(ValueError):
        run_blame(plan, probe_fn=_synthetic_probe(5, 4), top_k=0)


def test_run_blame_rejects_oversize_top_k():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    with pytest.raises(ValueError):
        run_blame(plan, probe_fn=_synthetic_probe(5, 4), top_k=1_000_001)


def test_run_blame_assigns_shard_id_correctly():
    """Each row's shard_id matches its plan position."""
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan(num_shards=4, num_rows=20)
    probe = _synthetic_probe(20, 4)
    result = run_blame(plan, probe_fn=probe, top_k=20)
    # Every row's shard_id must be in [0, 4)
    for influence in result.top_influencers:
        assert 0 <= influence.shard_id < 4


def test_run_blame_probe_called_once():
    """The runner calls probe_fn exactly once with the plan."""
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    call_count = {"n": 0}

    def counting_probe(plan_arg):
        call_count["n"] += 1
        rg = np.random.RandomState(0).randn(10, 4).astype(np.float32)
        pg = np.ones(4, dtype=np.float32)
        return rg, pg

    run_blame(plan, probe_fn=counting_probe)
    assert call_count["n"] == 1


def test_run_blame_probe_must_return_tuple():
    """Probe must return (row_grads, probe_grad). Other shapes -> TypeError."""
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    with pytest.raises((TypeError, ValueError)):
        run_blame(plan, probe_fn=lambda p: np.zeros(5))


def test_run_blame_probe_shape_mismatch_raises():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()

    def bad_probe(plan_arg):
        return np.zeros((10, 4), dtype=np.float32), np.zeros((6,), dtype=np.float32)

    with pytest.raises(ValueError, match="shape"):
        run_blame(plan, probe_fn=bad_probe)


def test_run_blame_no_probe_fn_friendly_error():
    """When no probe supplied, runner falls back to a deterministic synthetic probe.

    This is the v0.66 design — the runner is live and must produce a real
    BlameResult even without operator-supplied gradients (matches v0.54.0
    advise probe stub policy).
    """
    from soup_cli.utils.blame import BlameResult, run_blame

    plan = _make_tiny_plan()
    result = run_blame(plan)
    assert isinstance(result, BlameResult)


def test_run_blame_records_elapsed_time():
    from soup_cli.utils.blame import run_blame

    plan = _make_tiny_plan()
    result = run_blame(plan, probe_fn=_synthetic_probe(10, 4))
    assert result.elapsed_seconds >= 0.0


# ---------------------------------------------------------------------------
# Render
# ---------------------------------------------------------------------------


def test_render_blame_json_roundtrip():
    from soup_cli.utils.blame import BlameResult, RowInfluence, render_blame_json

    result = BlameResult(
        adapter_dir="adp",
        dataset_path="data.jsonl",
        layer="lm_head",
        num_rows_scored=10,
        elapsed_seconds=1.5,
        top_influencers=(
            RowInfluence(row_id=3, score=0.9, shard_id=0),
            RowInfluence(row_id=7, score=-0.5, shard_id=1),
        ),
    )
    text = render_blame_json(result)
    payload = json.loads(text)
    assert payload["num_rows_scored"] == 10
    assert len(payload["top_influencers"]) == 2
    assert payload["top_influencers"][0]["row_id"] == 3


def test_render_blame_json_rejects_non_result():
    from soup_cli.utils.blame import render_blame_json

    with pytest.raises(TypeError):
        render_blame_json("not a result")


def test_render_blame_markdown_has_table():
    from soup_cli.utils.blame import BlameResult, RowInfluence, render_blame_markdown

    result = BlameResult(
        adapter_dir="adp",
        dataset_path="data.jsonl",
        layer="lm_head",
        num_rows_scored=10,
        elapsed_seconds=1.5,
        top_influencers=(
            RowInfluence(row_id=3, score=0.9, shard_id=0),
        ),
    )
    text = render_blame_markdown(result)
    assert "adp" in text
    assert "row_id" in text or "row" in text.lower()


def test_render_blame_markdown_empty():
    from soup_cli.utils.blame import BlameResult, render_blame_markdown

    result = BlameResult(
        adapter_dir="adp",
        dataset_path="data.jsonl",
        layer="lm_head",
        num_rows_scored=0,
        elapsed_seconds=0.0,
        top_influencers=tuple(),
    )
    text = render_blame_markdown(result)
    assert "no influencers" in text.lower() or "empty" in text.lower()


def test_render_blame_markdown_rejects_non_result():
    from soup_cli.utils.blame import render_blame_markdown

    with pytest.raises(TypeError):
        render_blame_markdown(123)


# ---------------------------------------------------------------------------
# Source-grep regression
# ---------------------------------------------------------------------------


def test_run_blame_no_longer_raises_not_implemented_in_source():
    """Source-grep: ensure the v0.57.0 stub message is gone."""
    import inspect

    from soup_cli.utils import blame

    source = inspect.getsource(blame.run_blame)
    # Make sure the deferred stub marker is no longer the body of run_blame
    assert "Live blame ablation runner deferred to v0.57.1" not in source


def test_blame_no_heavy_top_level_imports():
    import inspect

    from soup_cli.utils import blame

    source = inspect.getsource(blame)
    top_level_imports = [
        line for line in source.splitlines()
        if line.startswith("import ") or line.startswith("from ")
    ]
    for line in top_level_imports:
        for bad in ("torch", "transformers", "peft", "safetensors"):
            assert bad not in line, f"top-level {bad} import: {line!r}"


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_tiny_plan(num_shards: int = 2, num_rows: int = 10):
    """Build a minimal blame plan for testing."""
    from pathlib import Path

    from soup_cli.utils.blame import plan_blame

    # We need a real adapter dir + dataset file under cwd
    cwd = Path.cwd()
    adapter = cwd / "tiny_adapter"
    adapter.mkdir(exist_ok=True)
    (adapter / "adapter_config.json").write_text(
        '{"base_model_name_or_path": "tiny"}', encoding="utf-8"
    )
    dataset = cwd / "tiny_data.jsonl"
    rows = "\n".join(['{"text":"row %d"}' % i for i in range(num_rows)])
    dataset.write_text(rows, encoding="utf-8")

    return plan_blame(
        str(adapter),
        str(dataset),
        layer="lm_head",
        budget_seconds=600,
        num_shards=num_shards,
    )


def _synthetic_probe(num_rows: int, grad_dim: int):
    """Build a closure that returns (row_grads, probe_grad)."""

    def probe(plan_arg):
        rng = np.random.RandomState(num_rows + grad_dim)
        row_grads = rng.randn(num_rows, grad_dim).astype(np.float32)
        probe_grad = rng.randn(grad_dim).astype(np.float32)
        return row_grads, probe_grad

    return probe