soup/tests/test_v0660_part_d.py

"""v0.66.0 Part D — Catastrophic interference matrix (TDD).

When you have multiple adapters in the Registry, run all pairwise on a
probe set. Score interference (loss on adapter A's domain when adapter B
is also loaded). Surfaces which adapter pairs you cannot deploy together
via the v0.22 multi-adapter serve.

Public surface:

- ``InterferenceCell`` frozen dataclass — one (A, B) result
- ``InterferenceMatrix`` frozen dataclass — full N×N matrix
- ``compute_interference(loss_a_alone, loss_a_with_b)`` — math kernel
- ``classify_interference(score)`` — OK/MINOR/MAJOR taxonomy
- ``build_interference_matrix(adapters, probe_losses)`` — pure orchestrator
- ``render_matrix_json`` / ``render_matrix_markdown``
"""
from __future__ import annotations

import json

import pytest

# ---------------------------------------------------------------------------
# Public surface
# ---------------------------------------------------------------------------


def test_module_imports():
    from soup_cli.utils import interference

    for name in (
        "InterferenceCell",
        "InterferenceMatrix",
        "compute_interference",
        "classify_interference",
        "build_interference_matrix",
        "render_matrix_json",
        "render_matrix_markdown",
        "INTERFERENCE_VERDICTS",
    ):
        assert hasattr(interference, name), name


def test_verdicts_closed():
    from soup_cli.utils.interference import INTERFERENCE_VERDICTS

    assert INTERFERENCE_VERDICTS == frozenset({"OK", "MINOR", "MAJOR"})


# ---------------------------------------------------------------------------
# compute_interference — math kernel
# ---------------------------------------------------------------------------


def test_compute_interference_zero_when_identical_loss():
    from soup_cli.utils.interference import compute_interference

    # No change: score = 0
    assert compute_interference(2.0, 2.0) == pytest.approx(0.0)


def test_compute_interference_positive_when_loss_increases():
    """When loss_a_with_b > loss_a_alone, interference score is positive."""
    from soup_cli.utils.interference import compute_interference

    score = compute_interference(2.0, 3.0)
    # Relative increase: (3 - 2) / 2 = 0.5
    assert score == pytest.approx(0.5)


def test_compute_interference_negative_when_loss_decreases():
    """When loss decreases (beneficial interaction), score is negative."""
    from soup_cli.utils.interference import compute_interference

    score = compute_interference(2.0, 1.5)
    # (1.5 - 2) / 2 = -0.25
    assert score == pytest.approx(-0.25)


def test_compute_interference_rejects_bool():
    from soup_cli.utils.interference import compute_interference

    with pytest.raises(TypeError):
        compute_interference(True, 1.0)


def test_compute_interference_rejects_non_finite():
    from soup_cli.utils.interference import compute_interference

    with pytest.raises(ValueError):
        compute_interference(float("nan"), 1.0)
    with pytest.raises(ValueError):
        compute_interference(1.0, float("inf"))


def test_compute_interference_rejects_non_positive_baseline():
    """Cannot divide by zero or negative baseline loss."""
    from soup_cli.utils.interference import compute_interference

    with pytest.raises(ValueError):
        compute_interference(0.0, 1.0)
    with pytest.raises(ValueError):
        compute_interference(-1.0, 1.0)


def test_compute_interference_rejects_negative_combined_loss():
    from soup_cli.utils.interference import compute_interference

    with pytest.raises(ValueError):
        compute_interference(1.0, -0.5)


# ---------------------------------------------------------------------------
# classify_interference
# ---------------------------------------------------------------------------


def test_classify_ok_when_small_change():
    from soup_cli.utils.interference import classify_interference

    # < 5% increase
    assert classify_interference(0.02) == "OK"
    assert classify_interference(-0.02) == "OK"


def test_classify_minor_in_5_to_20_pct():
    from soup_cli.utils.interference import classify_interference

    assert classify_interference(0.10) == "MINOR"


def test_classify_major_above_20_pct():
    from soup_cli.utils.interference import classify_interference

    assert classify_interference(0.30) == "MAJOR"


def test_classify_exact_boundaries():
    from soup_cli.utils.interference import classify_interference

    # 0.05 → MINOR (boundary lands in more severe)
    assert classify_interference(0.05) == "MINOR"
    # 0.20 → MAJOR
    assert classify_interference(0.20) == "MAJOR"


def test_classify_uses_absolute_value():
    """Symmetric — large negative (beneficial) is still 'MINOR' worth noting."""
    from soup_cli.utils.interference import classify_interference

    assert classify_interference(-0.10) == "MINOR"


def test_classify_rejects_bool():
    from soup_cli.utils.interference import classify_interference

    with pytest.raises(TypeError):
        classify_interference(True)


def test_classify_rejects_non_finite():
    from soup_cli.utils.interference import classify_interference

    with pytest.raises(ValueError):
        classify_interference(float("nan"))


# ---------------------------------------------------------------------------
# InterferenceCell
# ---------------------------------------------------------------------------


def test_cell_frozen():
    from soup_cli.utils.interference import InterferenceCell

    cell = InterferenceCell(
        adapter_a="a", adapter_b="b", score=0.0, verdict="OK"
    )
    with pytest.raises((AttributeError, Exception)):
        cell.score = 1.0  # type: ignore[misc]


def test_cell_rejects_invalid_verdict():
    from soup_cli.utils.interference import InterferenceCell

    with pytest.raises(ValueError):
        InterferenceCell(adapter_a="a", adapter_b="b", score=0.0, verdict="X")


def test_cell_rejects_empty_adapter_name():
    from soup_cli.utils.interference import InterferenceCell

    with pytest.raises(ValueError):
        InterferenceCell(adapter_a="", adapter_b="b", score=0.0, verdict="OK")


def test_cell_rejects_bool_score():
    from soup_cli.utils.interference import InterferenceCell

    with pytest.raises(TypeError):
        InterferenceCell(adapter_a="a", adapter_b="b", score=True, verdict="OK")


def test_cell_rejects_non_finite_score():
    from soup_cli.utils.interference import InterferenceCell

    with pytest.raises(ValueError):
        InterferenceCell(
            adapter_a="a", adapter_b="b", score=float("inf"), verdict="OK"
        )


def test_cell_rejects_null_byte_adapter_name():
    from soup_cli.utils.interference import InterferenceCell

    with pytest.raises(ValueError):
        InterferenceCell(adapter_a="a\x00b", adapter_b="b", score=0.0, verdict="OK")


# ---------------------------------------------------------------------------
# InterferenceMatrix
# ---------------------------------------------------------------------------


def test_matrix_frozen():
    from soup_cli.utils.interference import InterferenceMatrix

    m = InterferenceMatrix(
        adapters=("a", "b"),
        cells=tuple(),
        worst_pair=None,
        worst_score=0.0,
    )
    with pytest.raises((AttributeError, Exception)):
        m.worst_score = 1.0  # type: ignore[misc]


def test_matrix_rejects_non_tuple_adapters():
    from soup_cli.utils.interference import InterferenceMatrix

    with pytest.raises(TypeError):
        InterferenceMatrix(
            adapters=["a", "b"],  # list, not tuple
            cells=tuple(),
            worst_pair=None,
            worst_score=0.0,
        )


def test_matrix_rejects_duplicate_adapter_names():
    from soup_cli.utils.interference import InterferenceMatrix

    with pytest.raises(ValueError, match="duplicate"):
        InterferenceMatrix(
            adapters=("a", "a"),
            cells=tuple(),
            worst_pair=None,
            worst_score=0.0,
        )


def test_matrix_rejects_too_few_adapters():
    from soup_cli.utils.interference import InterferenceMatrix

    with pytest.raises(ValueError, match="at least 2"):
        InterferenceMatrix(
            adapters=("a",),
            cells=tuple(),
            worst_pair=None,
            worst_score=0.0,
        )


# ---------------------------------------------------------------------------
# build_interference_matrix
# ---------------------------------------------------------------------------


def test_build_matrix_2x2():
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b")
    # Probe losses: (target, with) loss pairs
    #   a on a alone = 2.0
    #   a on b alone = 1.5
    #   b on a alone = 1.0
    #   b on b alone = 2.5
    #   a + b loaded: a target = 2.2, b target = 2.8
    losses = {
        ("a", "a"): 2.0,
        ("b", "b"): 2.5,
        ("a", "b"): 2.2,  # loss on a's domain when b is loaded
        ("b", "a"): 2.8,  # loss on b's domain when a is loaded
    }
    matrix = build_interference_matrix(adapters, losses)
    # Diagonal cells: A vs A — score 0
    diagonal = [c for c in matrix.cells if c.adapter_a == c.adapter_b]
    assert all(c.score == 0.0 for c in diagonal)
    # Off-diagonal: 2 cells (a→b, b→a)
    off_diag = [c for c in matrix.cells if c.adapter_a != c.adapter_b]
    assert len(off_diag) == 2


def test_build_matrix_worst_pair():
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b", "c")
    losses = {
        ("a", "a"): 1.0, ("b", "b"): 1.0, ("c", "c"): 1.0,
        ("a", "b"): 1.1,  # 10% increase
        ("a", "c"): 1.5,  # 50% increase (MAJOR)
        ("b", "a"): 1.05, ("b", "c"): 1.2,
        ("c", "a"): 1.3, ("c", "b"): 1.0,
    }
    matrix = build_interference_matrix(adapters, losses)
    # Worst pair = (a, c) with score 0.5
    assert matrix.worst_pair == ("a", "c")
    assert matrix.worst_score == pytest.approx(0.5)


def test_build_matrix_rejects_unknown_adapter_in_losses():
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b")
    losses = {("a", "a"): 1.0, ("b", "b"): 1.0, ("z", "x"): 0.5}
    with pytest.raises(ValueError, match="unknown"):
        build_interference_matrix(adapters, losses)


def test_build_matrix_rejects_missing_diagonal():
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b")
    # Missing ("b", "b")
    losses = {("a", "a"): 1.0}
    with pytest.raises(ValueError, match="diagonal"):
        build_interference_matrix(adapters, losses)


def test_build_matrix_rejects_bool_loss():
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b")
    losses = {("a", "a"): True, ("b", "b"): 1.0}
    with pytest.raises(TypeError):
        build_interference_matrix(adapters, losses)


def test_build_matrix_rejects_non_finite_loss():
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b")
    losses = {("a", "a"): float("nan"), ("b", "b"): 1.0}
    with pytest.raises(ValueError):
        build_interference_matrix(adapters, losses)


def test_build_matrix_rejects_too_few_adapters():
    from soup_cli.utils.interference import build_interference_matrix

    with pytest.raises(ValueError):
        build_interference_matrix(("a",), {("a", "a"): 1.0})


def test_build_matrix_rejects_too_many_adapters():
    from soup_cli.utils.interference import build_interference_matrix

    # 30 adapters = 900 pairs, but matrix cap is 16 by default
    adapters = tuple(f"a{i}" for i in range(30))
    losses = {(a, a): 1.0 for a in adapters}
    with pytest.raises(ValueError, match="too many"):
        build_interference_matrix(adapters, losses)


def test_build_matrix_handles_empty_pairwise_losses():
    """Missing off-diagonal entries default to baseline (score 0)."""
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b")
    losses = {("a", "a"): 1.0, ("b", "b"): 1.0}  # no off-diagonal
    matrix = build_interference_matrix(adapters, losses)
    # All cells have score 0 (missing pair = no measurement = assume no interference)
    for cell in matrix.cells:
        assert cell.score == 0.0


def test_build_matrix_rejects_non_pair_loss_keys():
    from soup_cli.utils.interference import build_interference_matrix

    adapters = ("a", "b")
    losses = {"not a tuple": 1.0, ("a", "a"): 1.0, ("b", "b"): 1.0}
    with pytest.raises(TypeError, match="tuple"):
        build_interference_matrix(adapters, losses)


def test_build_matrix_rejects_bool_adapter_name():
    from soup_cli.utils.interference import build_interference_matrix

    with pytest.raises(TypeError):
        build_interference_matrix((True, "b"), {})  # type: ignore[arg-type]


def test_build_matrix_rejects_null_byte_adapter_name():
    from soup_cli.utils.interference import build_interference_matrix

    with pytest.raises(ValueError):
        build_interference_matrix(("a\x00", "b"), {})


# ---------------------------------------------------------------------------
# Render
# ---------------------------------------------------------------------------


def test_render_matrix_json_roundtrip():
    from soup_cli.utils.interference import (
        InterferenceCell,
        InterferenceMatrix,
        render_matrix_json,
    )

    m = InterferenceMatrix(
        adapters=("a", "b"),
        cells=(
            InterferenceCell(adapter_a="a", adapter_b="b", score=0.10, verdict="MINOR"),
        ),
        worst_pair=("a", "b"),
        worst_score=0.10,
    )
    text = render_matrix_json(m)
    payload = json.loads(text)
    assert payload["adapters"] == ["a", "b"]
    assert payload["worst_score"] == pytest.approx(0.10)


def test_render_matrix_json_rejects_non_matrix():
    from soup_cli.utils.interference import render_matrix_json

    with pytest.raises(TypeError):
        render_matrix_json("nope")


def test_render_matrix_markdown_has_table():
    from soup_cli.utils.interference import (
        InterferenceCell,
        InterferenceMatrix,
        render_matrix_markdown,
    )

    m = InterferenceMatrix(
        adapters=("a", "b"),
        cells=(
            InterferenceCell(adapter_a="a", adapter_b="b", score=0.30, verdict="MAJOR"),
        ),
        worst_pair=("a", "b"),
        worst_score=0.30,
    )
    text = render_matrix_markdown(m)
    assert "MAJOR" in text
    assert "a" in text


def test_render_matrix_markdown_empty_cells():
    from soup_cli.utils.interference import InterferenceMatrix, render_matrix_markdown

    m = InterferenceMatrix(
        adapters=("a", "b"),
        cells=tuple(),
        worst_pair=None,
        worst_score=0.0,
    )
    text = render_matrix_markdown(m)
    assert "no" in text.lower() or "empty" in text.lower()


def test_render_matrix_markdown_rejects_non_matrix():
    from soup_cli.utils.interference import render_matrix_markdown

    with pytest.raises(TypeError):
        render_matrix_markdown(None)


# ---------------------------------------------------------------------------
# Source-grep
# ---------------------------------------------------------------------------


def test_no_heavy_top_level_imports():
    import inspect

    from soup_cli.utils import interference

    source = inspect.getsource(interference)
    top_level_imports = [
        line for line in source.splitlines()
        if line.startswith("import ") or line.startswith("from ")
    ]
    for line in top_level_imports:
        for bad in ("torch", "transformers", "peft", "safetensors"):
            assert bad not in line, f"top-level {bad} import: {line!r}"