diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e998454..5333890 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -120,7 +120,7 @@ src/soup_cli/ templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (284 files, 13424 tests) +tests/ - Test suite (284 files, 13430 tests) examples/ - Real-world config examples and datasets ``` diff --git a/tests/test_v07111.py b/tests/test_v07111.py index 5ee9354..daf0c3e 100644 --- a/tests/test_v07111.py +++ b/tests/test_v07111.py @@ -521,21 +521,30 @@ class TestMiniLLM: def test_anchor_term_with_file(self, tmp_path, monkeypatch): import torch - from transformers import AutoModelForCausalLM, AutoTokenizer from soup_cli.utils.minillm import MiniLLMConfig, build_minillm_callback + # Loading a tokenizer/model from the Hub flakes on CI runners that get + # HF-rate-limited (the cache-warm step is best-effort). Skip on network + # failure — the anchor math is covered network-free by + # ``test_anchor_term_with_fake_model`` below. + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + + tok = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") + model = AutoModelForCausalLM.from_pretrained( + "hf-internal-testing/tiny-random-gpt2" + ) + except OSError as exc: # pragma: no cover — network-dependent + pytest.skip(f"HF model unavailable (offline / rate-limited): {exc}") + monkeypatch.chdir(tmp_path) anchor = tmp_path / "anchor.jsonl" anchor.write_text( "\n".join(json.dumps({"text": f"sentence number {i}"}) for i in range(4)) ) - tok = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") if tok.pad_token is None: tok.pad_token = tok.eos_token - model = AutoModelForCausalLM.from_pretrained( - "hf-internal-testing/tiny-random-gpt2" - ) cb = build_minillm_callback( MiniLLMConfig(pretrain_anchor_weight=0.1, pretrain_anchor_path="anchor.jsonl"), tokenizer=tok, @@ -544,6 +553,49 @@ class TestMiniLLM: assert term is not None assert torch.isfinite(term) + def test_anchor_term_with_fake_model(self, tmp_path, monkeypatch): + """Network-free coverage of ``_load_anchor`` + ``anchor_term`` — a fake + tokenizer + tiny ``nn.Module`` exercise the same lines as the Hub-backed + test above, so the coverage gate does not depend on HF availability.""" + import torch + import torch.nn as nn + + from soup_cli.utils.minillm import MiniLLMConfig, build_minillm_callback + + class _FakeOut: + def __init__(self, logits): + self.logits = logits + + class _TinyLM(nn.Module): + def __init__(self, vocab=16): + super().__init__() + self.emb = nn.Embedding(vocab, 4) + self.head = nn.Linear(4, vocab) + + def forward(self, input_ids, attention_mask=None): # noqa: ARG002 + return _FakeOut(self.head(self.emb(input_ids))) + + class _FakeTok: + def __call__( + self, texts, return_tensors=None, padding=None, + truncation=None, max_length=None, + ): # noqa: ARG002 + ids = torch.tensor([[1, 2, 3, 4] for _ in texts]) + return {"input_ids": ids, "attention_mask": torch.ones_like(ids)} + + monkeypatch.chdir(tmp_path) + anchor = tmp_path / "anchor.jsonl" + anchor.write_text( + "\n".join(json.dumps({"text": f"sentence {i}"}) for i in range(4)) + ) + cb = build_minillm_callback( + MiniLLMConfig(pretrain_anchor_weight=0.25, pretrain_anchor_path="anchor.jsonl"), + tokenizer=_FakeTok(), + ) + term = cb.anchor_term(_TinyLM()) + assert term is not None + assert torch.isfinite(term) + def test_anchor_term_disabled_returns_none(self): import torch.nn as nn diff --git a/tests/test_v07113.py b/tests/test_v07113.py index 259c41d..ea4e4e4 100644 --- a/tests/test_v07113.py +++ b/tests/test_v07113.py @@ -1396,6 +1396,89 @@ class TestCoverageGaps: assert tool["method"] == "get" +# =========================================================================== +# Coverage cushion — reachable internals exercised network/lib-free so the +# 77% gate does not sit right on the edge (the DSPy/TextGrad/GEPA optimiser +# bodies are genuinely uncoverable without the [compile] extra installed). +# =========================================================================== + + +class TestReachableInternals: + def test_resolve_metric_callable(self): + import types + + import soup_cli.utils.prompt_compile as pc + + mod = types.SimpleNamespace(metric=lambda *a, **k: 1.0) + assert callable(pc._resolve_metric(mod)) + + def test_resolve_metric_absent_returns_none(self): + import types + + import soup_cli.utils.prompt_compile as pc + + assert pc._resolve_metric(types.SimpleNamespace()) is None + + def test_resolve_metric_non_callable_returns_none(self): + import types + + import soup_cli.utils.prompt_compile as pc + + assert pc._resolve_metric(types.SimpleNamespace(metric=42)) is None + + def test_build_provider_fn_delegates_to_make_judge(self, monkeypatch): + import soup_cli.utils.data_forge as df + import soup_cli.utils.prompt_distill as pd + + seen = {} + + def fake_make(provider, *, model, base_url, temperature): + seen.update( + provider=provider, model=model, base_url=base_url, temperature=temperature + ) + return lambda prompt: {"text": f"R:{prompt}"} + + monkeypatch.setattr(df, "make_judge_provider_fn", fake_make) + fn = pd._build_provider_fn( + "ollama", "qwen2.5:0.5b", base_url="http://localhost:11434", temperature=0.2 + ) + assert fn("hi") == {"text": "R:hi"} + assert seen == { + "provider": "ollama", + "model": "qwen2.5:0.5b", + "base_url": "http://localhost:11434", + "temperature": 0.2, + } + + def test_prepare_distill_default_teacher_wires_provider(self, tmp_path, monkeypatch): + """teacher_fn=None routes through _build_provider_fn (default-provider + wiring) — covers the no-injected-seam branch of prepare_distill_dataset.""" + monkeypatch.chdir(tmp_path) + import soup_cli.utils.prompt_distill as pd + + _write_traces(str(tmp_path / "traces.jsonl"), ["q1", "q2"]) + calls = {"n": 0} + + def fake_build(provider, model, *, base_url, temperature): # noqa: ARG001 + def _gen(prompt): + calls["n"] += 1 + return {"text": f"T:{prompt}"} + + return _gen + + monkeypatch.setattr(pd, "_build_provider_fn", fake_build) + plan = pd.build_distill_prompt_plan( + traces_path="traces.jsonl", + teacher="qwen2.5:0.5b", + student="smol", + strategy="sft", + output_path="out.jsonl", + ) + n = pd.prepare_distill_dataset(plan, provider="ollama") + assert n == 2 + assert calls["n"] == 2 + + # =========================================================================== # Patch invariants # ===========================================================================