mirror of https://github.com/razor-ai/soup.git
test(ci): harden flaky MiniLLM anchor test + restore coverage network-free
CI failed on the HF-rate-limited runners: test_anchor_term_with_file did a live from_pretrained that 429'd, so it failed AND its unique MiniLLM-anchor lines went uncovered, tipping the 77% gate to 76.77% on exactly those jobs (macos + 3.11 stayed green where the cache warmed). - Skip test_anchor_term_with_file on OSError (offline / rate-limited) instead of failing. - Add test_anchor_term_with_fake_model: a fake tokenizer + tiny nn.Module exercise the identical _load_anchor + anchor_term lines with no network, so coverage no longer depends on HF availability. - Add TestReachableInternals cushion (prompt_compile._resolve_metric, prompt_distill._build_provider_fn + default-provider wiring) so the gate sits comfortably above 77% (the DSPy/TextGrad/GEPA optimiser bodies are uncoverable without the [compile] extra). Tests 13424 -> 13430.
This commit is contained in:
parent
f528da5328
commit
3ac9e305ce
|
|
@ -120,7 +120,7 @@ src/soup_cli/
|
|||
templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0)
|
||||
ui/ - Web UI (FastAPI + HTML/JS SPA)
|
||||
|
||||
tests/ - Test suite (284 files, 13424 tests)
|
||||
tests/ - Test suite (284 files, 13430 tests)
|
||||
examples/ - Real-world config examples and datasets
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -521,21 +521,30 @@ class TestMiniLLM:
|
|||
|
||||
def test_anchor_term_with_file(self, tmp_path, monkeypatch):
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from soup_cli.utils.minillm import MiniLLMConfig, build_minillm_callback
|
||||
|
||||
# Loading a tokenizer/model from the Hub flakes on CI runners that get
|
||||
# HF-rate-limited (the cache-warm step is best-effort). Skip on network
|
||||
# failure — the anchor math is covered network-free by
|
||||
# ``test_anchor_term_with_fake_model`` below.
|
||||
try:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
tok = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
"hf-internal-testing/tiny-random-gpt2"
|
||||
)
|
||||
except OSError as exc: # pragma: no cover — network-dependent
|
||||
pytest.skip(f"HF model unavailable (offline / rate-limited): {exc}")
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
anchor = tmp_path / "anchor.jsonl"
|
||||
anchor.write_text(
|
||||
"\n".join(json.dumps({"text": f"sentence number {i}"}) for i in range(4))
|
||||
)
|
||||
tok = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")
|
||||
if tok.pad_token is None:
|
||||
tok.pad_token = tok.eos_token
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
"hf-internal-testing/tiny-random-gpt2"
|
||||
)
|
||||
cb = build_minillm_callback(
|
||||
MiniLLMConfig(pretrain_anchor_weight=0.1, pretrain_anchor_path="anchor.jsonl"),
|
||||
tokenizer=tok,
|
||||
|
|
@ -544,6 +553,49 @@ class TestMiniLLM:
|
|||
assert term is not None
|
||||
assert torch.isfinite(term)
|
||||
|
||||
def test_anchor_term_with_fake_model(self, tmp_path, monkeypatch):
|
||||
"""Network-free coverage of ``_load_anchor`` + ``anchor_term`` — a fake
|
||||
tokenizer + tiny ``nn.Module`` exercise the same lines as the Hub-backed
|
||||
test above, so the coverage gate does not depend on HF availability."""
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from soup_cli.utils.minillm import MiniLLMConfig, build_minillm_callback
|
||||
|
||||
class _FakeOut:
|
||||
def __init__(self, logits):
|
||||
self.logits = logits
|
||||
|
||||
class _TinyLM(nn.Module):
|
||||
def __init__(self, vocab=16):
|
||||
super().__init__()
|
||||
self.emb = nn.Embedding(vocab, 4)
|
||||
self.head = nn.Linear(4, vocab)
|
||||
|
||||
def forward(self, input_ids, attention_mask=None): # noqa: ARG002
|
||||
return _FakeOut(self.head(self.emb(input_ids)))
|
||||
|
||||
class _FakeTok:
|
||||
def __call__(
|
||||
self, texts, return_tensors=None, padding=None,
|
||||
truncation=None, max_length=None,
|
||||
): # noqa: ARG002
|
||||
ids = torch.tensor([[1, 2, 3, 4] for _ in texts])
|
||||
return {"input_ids": ids, "attention_mask": torch.ones_like(ids)}
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
anchor = tmp_path / "anchor.jsonl"
|
||||
anchor.write_text(
|
||||
"\n".join(json.dumps({"text": f"sentence {i}"}) for i in range(4))
|
||||
)
|
||||
cb = build_minillm_callback(
|
||||
MiniLLMConfig(pretrain_anchor_weight=0.25, pretrain_anchor_path="anchor.jsonl"),
|
||||
tokenizer=_FakeTok(),
|
||||
)
|
||||
term = cb.anchor_term(_TinyLM())
|
||||
assert term is not None
|
||||
assert torch.isfinite(term)
|
||||
|
||||
def test_anchor_term_disabled_returns_none(self):
|
||||
import torch.nn as nn
|
||||
|
||||
|
|
|
|||
|
|
@ -1396,6 +1396,89 @@ class TestCoverageGaps:
|
|||
assert tool["method"] == "get"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Coverage cushion — reachable internals exercised network/lib-free so the
|
||||
# 77% gate does not sit right on the edge (the DSPy/TextGrad/GEPA optimiser
|
||||
# bodies are genuinely uncoverable without the [compile] extra installed).
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestReachableInternals:
|
||||
def test_resolve_metric_callable(self):
|
||||
import types
|
||||
|
||||
import soup_cli.utils.prompt_compile as pc
|
||||
|
||||
mod = types.SimpleNamespace(metric=lambda *a, **k: 1.0)
|
||||
assert callable(pc._resolve_metric(mod))
|
||||
|
||||
def test_resolve_metric_absent_returns_none(self):
|
||||
import types
|
||||
|
||||
import soup_cli.utils.prompt_compile as pc
|
||||
|
||||
assert pc._resolve_metric(types.SimpleNamespace()) is None
|
||||
|
||||
def test_resolve_metric_non_callable_returns_none(self):
|
||||
import types
|
||||
|
||||
import soup_cli.utils.prompt_compile as pc
|
||||
|
||||
assert pc._resolve_metric(types.SimpleNamespace(metric=42)) is None
|
||||
|
||||
def test_build_provider_fn_delegates_to_make_judge(self, monkeypatch):
|
||||
import soup_cli.utils.data_forge as df
|
||||
import soup_cli.utils.prompt_distill as pd
|
||||
|
||||
seen = {}
|
||||
|
||||
def fake_make(provider, *, model, base_url, temperature):
|
||||
seen.update(
|
||||
provider=provider, model=model, base_url=base_url, temperature=temperature
|
||||
)
|
||||
return lambda prompt: {"text": f"R:{prompt}"}
|
||||
|
||||
monkeypatch.setattr(df, "make_judge_provider_fn", fake_make)
|
||||
fn = pd._build_provider_fn(
|
||||
"ollama", "qwen2.5:0.5b", base_url="http://localhost:11434", temperature=0.2
|
||||
)
|
||||
assert fn("hi") == {"text": "R:hi"}
|
||||
assert seen == {
|
||||
"provider": "ollama",
|
||||
"model": "qwen2.5:0.5b",
|
||||
"base_url": "http://localhost:11434",
|
||||
"temperature": 0.2,
|
||||
}
|
||||
|
||||
def test_prepare_distill_default_teacher_wires_provider(self, tmp_path, monkeypatch):
|
||||
"""teacher_fn=None routes through _build_provider_fn (default-provider
|
||||
wiring) — covers the no-injected-seam branch of prepare_distill_dataset."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
import soup_cli.utils.prompt_distill as pd
|
||||
|
||||
_write_traces(str(tmp_path / "traces.jsonl"), ["q1", "q2"])
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_build(provider, model, *, base_url, temperature): # noqa: ARG001
|
||||
def _gen(prompt):
|
||||
calls["n"] += 1
|
||||
return {"text": f"T:{prompt}"}
|
||||
|
||||
return _gen
|
||||
|
||||
monkeypatch.setattr(pd, "_build_provider_fn", fake_build)
|
||||
plan = pd.build_distill_prompt_plan(
|
||||
traces_path="traces.jsonl",
|
||||
teacher="qwen2.5:0.5b",
|
||||
student="smol",
|
||||
strategy="sft",
|
||||
output_path="out.jsonl",
|
||||
)
|
||||
n = pd.prepare_distill_dataset(plan, provider="ollama")
|
||||
assert n == 2
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Patch invariants
|
||||
# ===========================================================================
|
||||
|
|
|
|||
Loading…
Reference in New Issue