mirror of https://github.com/razor-ai/soup.git
911 lines
34 KiB
Python
911 lines
34 KiB
Python
"""Tests for v0.30.0 Inference Excellence — prefix caching, spec-decoding pairing,
|
|
LoRA hot-swap, structured output, batching dashboard, request tracing, auto-quant.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
import re
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
def _strip_ansi(text: str) -> str:
|
|
"""Remove ANSI escape codes from Rich-formatted output."""
|
|
return re.sub(r'\x1b\[[0-9;]*m', '', text)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Part A: Prefix Caching — vLLM enable_prefix_caching passthrough
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestPrefixCachingCLI:
|
|
def test_serve_prefix_cache_flag_exists(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["serve", "--help"])
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert "--prefix-cache" in _strip_ansi(result.output)
|
|
|
|
def test_serve_prefix_cache_default_false(self):
|
|
from soup_cli.commands.serve import serve
|
|
|
|
sig = inspect.signature(serve)
|
|
param = sig.parameters.get("prefix_cache")
|
|
assert param is not None
|
|
assert param.default.default is False
|
|
|
|
|
|
class TestPrefixCachingEngine:
|
|
def test_create_engine_accepts_prefix_cache(self):
|
|
from soup_cli.utils.vllm import create_vllm_engine
|
|
|
|
sig = inspect.signature(create_vllm_engine)
|
|
assert "enable_prefix_caching" in sig.parameters
|
|
assert sig.parameters["enable_prefix_caching"].default is False
|
|
|
|
def test_create_engine_passes_prefix_cache_to_vllm(self):
|
|
mock_engine = MagicMock()
|
|
mock_args_cls = MagicMock()
|
|
mock_engine_cls = MagicMock()
|
|
mock_engine_cls.from_engine_args.return_value = mock_engine
|
|
|
|
with patch.dict("sys.modules", {
|
|
"vllm": MagicMock(
|
|
AsyncEngineArgs=mock_args_cls,
|
|
AsyncLLMEngine=mock_engine_cls,
|
|
),
|
|
}):
|
|
from importlib import reload
|
|
|
|
import soup_cli.utils.vllm as vllm_mod
|
|
|
|
reload(vllm_mod)
|
|
|
|
vllm_mod.create_vllm_engine(
|
|
model_path="/path/to/model",
|
|
is_adapter=False,
|
|
enable_prefix_caching=True,
|
|
)
|
|
call_kwargs = mock_args_cls.call_args.kwargs
|
|
assert call_kwargs["enable_prefix_caching"] is True
|
|
|
|
def test_create_engine_prefix_cache_default_false(self):
|
|
mock_args_cls = MagicMock()
|
|
mock_engine_cls = MagicMock()
|
|
mock_engine_cls.from_engine_args.return_value = MagicMock()
|
|
|
|
with patch.dict("sys.modules", {
|
|
"vllm": MagicMock(
|
|
AsyncEngineArgs=mock_args_cls,
|
|
AsyncLLMEngine=mock_engine_cls,
|
|
),
|
|
}):
|
|
from importlib import reload
|
|
|
|
import soup_cli.utils.vllm as vllm_mod
|
|
|
|
reload(vllm_mod)
|
|
|
|
vllm_mod.create_vllm_engine(
|
|
model_path="/path/to/model",
|
|
is_adapter=False,
|
|
)
|
|
call_kwargs = mock_args_cls.call_args.kwargs
|
|
assert call_kwargs["enable_prefix_caching"] is False
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Part B: Speculative Decoding Auto-Pairing
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestSpeculativePairing:
|
|
def test_pick_draft_model_for_llama_70b(self):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
result = pick_draft_model("meta-llama/Llama-3.1-70B-Instruct")
|
|
assert result is not None
|
|
assert "1b" in result.lower() or "3b" in result.lower()
|
|
|
|
def test_pick_draft_model_for_llama_8b_returns_none(self):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
# 8B is too small for speculative decoding (draft+target overhead > gain)
|
|
result = pick_draft_model("meta-llama/Llama-3.1-8B-Instruct")
|
|
assert result is None
|
|
|
|
def test_pick_draft_model_for_qwen_72b(self):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
result = pick_draft_model("Qwen/Qwen2.5-72B-Instruct")
|
|
assert result is not None
|
|
assert "qwen" in result.lower()
|
|
|
|
def test_pick_draft_model_unknown_returns_none(self):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
assert pick_draft_model("some/unknown-model") is None
|
|
|
|
def test_pick_draft_model_empty_string_returns_none(self):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
assert pick_draft_model("") is None
|
|
|
|
def test_pick_draft_model_rejects_urls(self):
|
|
"""Guard against URL injection — only HF model IDs."""
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
assert pick_draft_model("https://evil.com/model") is None
|
|
assert pick_draft_model("http://localhost/model") is None
|
|
|
|
def test_pick_draft_model_rejects_null_byte(self):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
assert pick_draft_model("meta-llama/Llama\x00-3.1-70B-Instruct") is None
|
|
|
|
def test_pick_draft_model_case_insensitive_match(self):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
a = pick_draft_model("Meta-Llama/Llama-3.1-70B-Instruct")
|
|
b = pick_draft_model("meta-llama/llama-3.1-70b-instruct")
|
|
assert a == b
|
|
|
|
def test_serve_auto_spec_flag(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["serve", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--auto-spec" in _strip_ansi(result.output)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Part C: Dynamic LoRA Hot-Swap
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestLoRAHotSwap:
|
|
def test_activate_endpoint_exists(self, tmp_path):
|
|
pytest.importorskip("fastapi")
|
|
from fastapi.testclient import TestClient
|
|
|
|
from soup_cli.commands.serve import _create_app
|
|
|
|
# Create valid adapter path under cwd
|
|
adapter_dir = tmp_path / "adapter-a"
|
|
adapter_dir.mkdir()
|
|
(adapter_dir / "adapter_config.json").write_text("{}")
|
|
|
|
import os
|
|
old_cwd = os.getcwd()
|
|
os.chdir(tmp_path)
|
|
try:
|
|
app = _create_app(
|
|
model_obj=MagicMock(),
|
|
tokenizer=MagicMock(),
|
|
device="cpu",
|
|
model_name="test",
|
|
max_tokens_default=512,
|
|
adapter_map={"chat": str(adapter_dir)},
|
|
)
|
|
client = TestClient(app)
|
|
# List route should include activate
|
|
routes = [r.path for r in app.routes]
|
|
assert "/v1/adapters/activate/{name}" in routes or any(
|
|
"/v1/adapters/activate" in r for r in routes
|
|
)
|
|
|
|
# Activate known adapter
|
|
resp = client.post("/v1/adapters/activate/chat")
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["active"] == "chat"
|
|
finally:
|
|
os.chdir(old_cwd)
|
|
|
|
def test_activate_unknown_returns_404(self, tmp_path):
|
|
pytest.importorskip("fastapi")
|
|
from fastapi.testclient import TestClient
|
|
|
|
from soup_cli.commands.serve import _create_app
|
|
|
|
app = _create_app(
|
|
model_obj=MagicMock(),
|
|
tokenizer=MagicMock(),
|
|
device="cpu",
|
|
model_name="test",
|
|
max_tokens_default=512,
|
|
adapter_map={"chat": "./x"},
|
|
)
|
|
client = TestClient(app)
|
|
resp = client.post("/v1/adapters/activate/unknown")
|
|
assert resp.status_code == 404
|
|
|
|
def test_activate_rejects_invalid_names(self, tmp_path):
|
|
"""Name must be alphanumeric + hyphens — prevent path injection."""
|
|
pytest.importorskip("fastapi")
|
|
from fastapi.testclient import TestClient
|
|
|
|
from soup_cli.commands.serve import _create_app
|
|
|
|
app = _create_app(
|
|
model_obj=MagicMock(),
|
|
tokenizer=MagicMock(),
|
|
device="cpu",
|
|
model_name="test",
|
|
max_tokens_default=512,
|
|
adapter_map={"chat": "./x"},
|
|
)
|
|
client = TestClient(app)
|
|
# Path-traversal style names
|
|
resp = client.post("/v1/adapters/activate/..%2Fetc")
|
|
assert resp.status_code in (400, 404, 422)
|
|
|
|
def test_active_adapter_in_health(self, tmp_path):
|
|
pytest.importorskip("fastapi")
|
|
from fastapi.testclient import TestClient
|
|
|
|
from soup_cli.commands.serve import _create_app
|
|
|
|
app = _create_app(
|
|
model_obj=MagicMock(),
|
|
tokenizer=MagicMock(),
|
|
device="cpu",
|
|
model_name="test",
|
|
max_tokens_default=512,
|
|
adapter_map={"chat": "./x"},
|
|
)
|
|
client = TestClient(app)
|
|
resp = client.get("/health")
|
|
assert resp.status_code == 200
|
|
# active may be None initially or the first-loaded adapter
|
|
data = resp.json()
|
|
assert "active_adapter" in data
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Part D: Structured Output (JSON Schema / Regex)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestStructuredOutput:
|
|
def test_structured_output_flag_exists(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["serve", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--structured-output" in _strip_ansi(result.output)
|
|
|
|
def test_validate_structured_output_accepts_known_modes(self):
|
|
from soup_cli.utils.structured_output import validate_mode
|
|
|
|
assert validate_mode("json") == "json"
|
|
assert validate_mode("regex") == "regex"
|
|
assert validate_mode("off") == "off"
|
|
assert validate_mode(None) == "off"
|
|
|
|
def test_validate_structured_output_rejects_unknown(self):
|
|
from soup_cli.utils.structured_output import validate_mode
|
|
|
|
with pytest.raises(ValueError, match="Unknown structured-output mode"):
|
|
validate_mode("xml")
|
|
|
|
def test_validate_json_schema_accepts_dict(self):
|
|
from soup_cli.utils.structured_output import validate_json_schema
|
|
|
|
schema = {"type": "object", "properties": {"name": {"type": "string"}}}
|
|
# Should not raise
|
|
validate_json_schema(schema)
|
|
|
|
def test_validate_json_schema_rejects_non_dict(self):
|
|
from soup_cli.utils.structured_output import validate_json_schema
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_json_schema("not a dict")
|
|
|
|
def test_validate_regex_pattern_rejects_redos(self):
|
|
"""Guard against ReDoS: nested quantifier over catastrophic backtracking."""
|
|
from soup_cli.utils.structured_output import validate_regex_pattern
|
|
|
|
with pytest.raises(ValueError, match="length"):
|
|
validate_regex_pattern("x" * 10_000)
|
|
|
|
def test_validate_regex_pattern_accepts_simple(self):
|
|
from soup_cli.utils.structured_output import validate_regex_pattern
|
|
|
|
validate_regex_pattern(r"\d{3}-\d{4}")
|
|
|
|
def test_validate_regex_pattern_rejects_invalid(self):
|
|
from soup_cli.utils.structured_output import validate_regex_pattern
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_regex_pattern("[unclosed")
|
|
|
|
def test_build_constraint_returns_none_when_off(self):
|
|
from soup_cli.utils.structured_output import build_constraint
|
|
|
|
assert build_constraint("off", None, None) is None
|
|
|
|
def test_build_constraint_json_requires_schema_or_skips(self):
|
|
from soup_cli.utils.structured_output import build_constraint
|
|
|
|
result = build_constraint("json", None, None)
|
|
# When no schema provided, fall back to free-form JSON (not None
|
|
# if we have a library, None otherwise — both acceptable as long
|
|
# as there's no crash).
|
|
assert result is None or isinstance(result, dict)
|
|
|
|
def test_build_constraint_regex_requires_pattern(self):
|
|
from soup_cli.utils.structured_output import build_constraint
|
|
|
|
with pytest.raises(ValueError):
|
|
build_constraint("regex", None, None)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Part E: Continuous Batching Dashboard
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestBatchingDashboard:
|
|
def test_dashboard_flag_exists(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["serve", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--dashboard" in _strip_ansi(result.output)
|
|
|
|
def test_metrics_endpoint_exists(self):
|
|
pytest.importorskip("fastapi")
|
|
from fastapi.testclient import TestClient
|
|
|
|
from soup_cli.commands.serve import _create_app
|
|
|
|
app = _create_app(
|
|
model_obj=MagicMock(),
|
|
tokenizer=MagicMock(),
|
|
device="cpu",
|
|
model_name="test",
|
|
max_tokens_default=512,
|
|
)
|
|
client = TestClient(app)
|
|
resp = client.get("/metrics")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert "requests_total" in data
|
|
assert "tokens_generated_total" in data
|
|
assert "active_requests" in data
|
|
|
|
def test_metrics_counters_start_at_zero(self):
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
m = ServerMetrics()
|
|
snap = m.snapshot()
|
|
assert snap["requests_total"] == 0
|
|
assert snap["tokens_generated_total"] == 0
|
|
assert snap["active_requests"] == 0
|
|
|
|
def test_metrics_tracks_request_lifecycle(self):
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
m = ServerMetrics()
|
|
with m.track_request():
|
|
snap = m.snapshot()
|
|
assert snap["active_requests"] == 1
|
|
assert m.snapshot()["active_requests"] == 0
|
|
assert m.snapshot()["requests_total"] == 1
|
|
|
|
def test_metrics_track_request_handles_exception(self):
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
m = ServerMetrics()
|
|
with pytest.raises(RuntimeError):
|
|
with m.track_request():
|
|
raise RuntimeError("boom")
|
|
assert m.snapshot()["active_requests"] == 0
|
|
assert m.snapshot()["requests_total"] == 1
|
|
|
|
def test_metrics_record_tokens(self):
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
m = ServerMetrics()
|
|
m.record_tokens(42)
|
|
m.record_tokens(8)
|
|
assert m.snapshot()["tokens_generated_total"] == 50
|
|
|
|
def test_metrics_record_tokens_rejects_negative(self):
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
m = ServerMetrics()
|
|
with pytest.raises(ValueError):
|
|
m.record_tokens(-1)
|
|
|
|
def test_metrics_latency_percentiles(self):
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
m = ServerMetrics()
|
|
for ms in [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]:
|
|
m.record_latency(ms)
|
|
snap = m.snapshot()
|
|
assert "latency_p50_ms" in snap
|
|
assert "latency_p95_ms" in snap
|
|
assert 40 <= snap["latency_p50_ms"] <= 60
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Part F: Request Tracing (OpenTelemetry)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestRequestTracing:
|
|
def test_trace_flag_exists(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["serve", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--trace" in _strip_ansi(result.output)
|
|
|
|
def test_is_otel_available_when_missing(self):
|
|
from soup_cli.utils.tracing import is_otel_available
|
|
|
|
with patch.dict("sys.modules", {"opentelemetry": None}):
|
|
assert is_otel_available() is False
|
|
|
|
def test_build_tracer_returns_none_when_disabled(self):
|
|
from soup_cli.utils.tracing import build_tracer
|
|
|
|
assert build_tracer(enabled=False) is None
|
|
|
|
def test_build_tracer_returns_none_when_otel_missing(self):
|
|
from soup_cli.utils.tracing import build_tracer
|
|
|
|
with patch(
|
|
"soup_cli.utils.tracing.is_otel_available", return_value=False
|
|
):
|
|
# Should not raise, just return None + log
|
|
assert build_tracer(enabled=True) is None
|
|
|
|
def test_validate_otlp_endpoint_accepts_localhost_http(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
assert validate_otlp_endpoint("http://localhost:4317") == "http://localhost:4317"
|
|
assert validate_otlp_endpoint("http://127.0.0.1:4317") == "http://127.0.0.1:4317"
|
|
|
|
def test_validate_otlp_endpoint_accepts_https(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
assert (
|
|
validate_otlp_endpoint("https://otlp.example.com:4317")
|
|
== "https://otlp.example.com:4317"
|
|
)
|
|
|
|
def test_validate_otlp_endpoint_rejects_plain_http_remote(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError, match="HTTPS"):
|
|
validate_otlp_endpoint("http://evil.com:4317")
|
|
|
|
def test_validate_otlp_endpoint_rejects_bad_scheme(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_otlp_endpoint("ftp://example.com")
|
|
|
|
def test_validate_otlp_endpoint_rejects_null_byte(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_otlp_endpoint("http://localhost\x00/evil")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Part G: Auto-Quant on Deploy
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestAutoQuant:
|
|
def test_auto_quant_flag_exists(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["serve", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--auto-quant" in _strip_ansi(result.output)
|
|
|
|
def test_candidate_order_default(self):
|
|
from soup_cli.utils.auto_quant import default_candidate_order
|
|
|
|
order = default_candidate_order()
|
|
# First candidate should be fastest-to-try; we canonicalize order.
|
|
assert order[0] in {"gguf", "awq", "gptq", "fp8", "none"}
|
|
# No duplicates
|
|
assert len(order) == len(set(order))
|
|
# At least these quant formats present
|
|
assert "gguf" in order
|
|
assert "awq" in order
|
|
|
|
def test_pick_best_candidate_prefers_fastest_ok(self):
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
candidates = [
|
|
Candidate(name="gguf", score=0.92, latency_ms=200, ok=True),
|
|
Candidate(name="awq", score=0.95, latency_ms=180, ok=True),
|
|
Candidate(name="none", score=0.96, latency_ms=400, ok=True),
|
|
]
|
|
best = pick_best(candidates, min_score=0.90)
|
|
# awq wins: ok, score above threshold, lowest latency
|
|
assert best.name == "awq"
|
|
|
|
def test_pick_best_drops_below_threshold(self):
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
candidates = [
|
|
Candidate(name="gguf", score=0.50, latency_ms=100, ok=True),
|
|
Candidate(name="awq", score=0.60, latency_ms=50, ok=True),
|
|
Candidate(name="none", score=0.95, latency_ms=400, ok=True),
|
|
]
|
|
best = pick_best(candidates, min_score=0.90)
|
|
assert best.name == "none"
|
|
|
|
def test_pick_best_raises_when_no_candidates_pass(self):
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
candidates = [
|
|
Candidate(name="gguf", score=0.50, latency_ms=100, ok=True),
|
|
]
|
|
with pytest.raises(ValueError, match="no candidate"):
|
|
pick_best(candidates, min_score=0.90)
|
|
|
|
def test_pick_best_skips_failed(self):
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
candidates = [
|
|
Candidate(name="gguf", score=0.99, latency_ms=1, ok=False),
|
|
Candidate(name="awq", score=0.91, latency_ms=180, ok=True),
|
|
]
|
|
best = pick_best(candidates, min_score=0.90)
|
|
assert best.name == "awq"
|
|
|
|
def test_candidate_requires_positive_latency(self):
|
|
from soup_cli.utils.auto_quant import Candidate
|
|
|
|
with pytest.raises(ValueError):
|
|
Candidate(name="x", score=0.9, latency_ms=-1, ok=True)
|
|
|
|
def test_candidate_score_bounded(self):
|
|
from soup_cli.utils.auto_quant import Candidate
|
|
|
|
with pytest.raises(ValueError):
|
|
Candidate(name="x", score=1.5, latency_ms=100, ok=True)
|
|
with pytest.raises(ValueError):
|
|
Candidate(name="x", score=-0.1, latency_ms=100, ok=True)
|
|
|
|
def test_min_score_bounds(self):
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
candidates = [
|
|
Candidate(name="awq", score=0.9, latency_ms=100, ok=True),
|
|
]
|
|
with pytest.raises(ValueError):
|
|
pick_best(candidates, min_score=-0.1)
|
|
with pytest.raises(ValueError):
|
|
pick_best(candidates, min_score=1.1)
|
|
|
|
def test_name_validation_on_candidate(self):
|
|
"""Candidate names must be alphanumeric — prevents arbitrary strings
|
|
leaking into log/display."""
|
|
from soup_cli.utils.auto_quant import Candidate
|
|
|
|
with pytest.raises(ValueError):
|
|
Candidate(name="../../etc", score=0.9, latency_ms=100, ok=True)
|
|
|
|
def test_pick_best_tie_break_by_first_encountered(self):
|
|
"""When latency and score are tied, prefer first-encountered
|
|
(stable — consistent with v0.28.0 kernel_picker)."""
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
candidates = [
|
|
Candidate(name="gguf", score=0.92, latency_ms=200, ok=True),
|
|
Candidate(name="awq", score=0.92, latency_ms=200, ok=True),
|
|
]
|
|
best = pick_best(candidates, min_score=0.90)
|
|
assert best.name == "gguf"
|
|
|
|
def test_pick_best_all_failed(self):
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
candidates = [
|
|
Candidate(name="gguf", score=0.99, latency_ms=100, ok=False),
|
|
Candidate(name="awq", score=0.99, latency_ms=50, ok=False),
|
|
]
|
|
with pytest.raises(ValueError, match="no candidate"):
|
|
pick_best(candidates, min_score=0.90)
|
|
|
|
def test_pick_best_empty_list(self):
|
|
from soup_cli.utils.auto_quant import pick_best
|
|
|
|
with pytest.raises(ValueError, match="no candidate"):
|
|
pick_best([], min_score=0.90)
|
|
|
|
def test_pick_best_consumes_generator_once(self):
|
|
"""Regression test: error message counts correctly when input is
|
|
a generator (not a list). v0.30.0 review fix."""
|
|
from soup_cli.utils.auto_quant import Candidate, pick_best
|
|
|
|
def gen():
|
|
yield Candidate(name="gguf", score=0.5, latency_ms=100, ok=True)
|
|
yield Candidate(name="awq", score=0.4, latency_ms=80, ok=True)
|
|
|
|
with pytest.raises(ValueError, match="ran 2 candidates"):
|
|
pick_best(gen(), min_score=0.90)
|
|
|
|
def test_candidate_nan_score_rejected(self):
|
|
from soup_cli.utils.auto_quant import Candidate
|
|
|
|
with pytest.raises(ValueError, match="finite"):
|
|
Candidate(name="x", score=float("nan"), latency_ms=100, ok=True)
|
|
|
|
def test_candidate_nan_latency_rejected(self):
|
|
from soup_cli.utils.auto_quant import Candidate
|
|
|
|
with pytest.raises(ValueError, match="finite"):
|
|
Candidate(name="x", score=0.9, latency_ms=float("nan"), ok=True)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Review-fix follow-up: coverage gaps from tdd-guide review
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestSpeculativePairingFullMatrix:
|
|
"""Cover every architecture family in the pick_draft_model map."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"target",
|
|
[
|
|
"meta-llama/Llama-3.3-70B-Instruct",
|
|
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
|
"meta-llama/Llama-4-Maverick-17B-128E",
|
|
"Qwen/Qwen3-32B",
|
|
"Qwen/Qwen3-14B",
|
|
"mistralai/Mistral-Large-Instruct-2407",
|
|
"mistralai/Mixtral-8x22B-Instruct-v0.1",
|
|
"deepseek-ai/DeepSeek-V3",
|
|
"deepseek-ai/DeepSeek-R1",
|
|
"google/gemma-3-27b-it",
|
|
"google/gemma-2-27b-it",
|
|
],
|
|
)
|
|
def test_known_targets_return_draft(self, target):
|
|
from soup_cli.utils.spec_pairing import pick_draft_model
|
|
|
|
result = pick_draft_model(target)
|
|
assert result is not None, f"no draft mapped for {target}"
|
|
|
|
|
|
class TestLoRADeactivate:
|
|
def test_deactivate_clears_active_state(self):
|
|
pytest.importorskip("fastapi")
|
|
from fastapi.testclient import TestClient
|
|
|
|
from soup_cli.commands.serve import _create_app
|
|
|
|
app = _create_app(
|
|
model_obj=MagicMock(),
|
|
tokenizer=MagicMock(),
|
|
device="cpu",
|
|
model_name="test",
|
|
max_tokens_default=512,
|
|
adapter_map={"chat": "./x"},
|
|
)
|
|
client = TestClient(app)
|
|
activate_resp = client.post("/v1/adapters/activate/chat")
|
|
assert activate_resp.status_code == 200
|
|
assert activate_resp.json()["active"] == "chat"
|
|
|
|
deactivate_resp = client.post("/v1/adapters/deactivate")
|
|
assert deactivate_resp.status_code == 200
|
|
assert deactivate_resp.json()["active"] is None
|
|
|
|
health = client.get("/health").json()
|
|
assert health["active_adapter"] is None
|
|
|
|
def test_activate_rejects_when_no_adapters_loaded(self):
|
|
pytest.importorskip("fastapi")
|
|
from fastapi.testclient import TestClient
|
|
|
|
from soup_cli.commands.serve import _create_app
|
|
|
|
app = _create_app(
|
|
model_obj=MagicMock(),
|
|
tokenizer=MagicMock(),
|
|
device="cpu",
|
|
model_name="test",
|
|
max_tokens_default=512,
|
|
adapter_map=None,
|
|
)
|
|
client = TestClient(app)
|
|
resp = client.post("/v1/adapters/activate/chat")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestMetricsThreadSafety:
|
|
def test_concurrent_track_request(self):
|
|
"""Fire 50 concurrent enters: requests_total must equal 50 exactly
|
|
(catches missing lock / non-atomic increment)."""
|
|
import threading
|
|
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
metrics = ServerMetrics()
|
|
|
|
def worker():
|
|
with metrics.track_request():
|
|
pass
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(50)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
|
|
snap = metrics.snapshot()
|
|
assert snap["requests_total"] == 50
|
|
assert snap["active_requests"] == 0
|
|
|
|
def test_record_tokens_zero_allowed(self):
|
|
from soup_cli.utils.metrics import ServerMetrics
|
|
|
|
metrics = ServerMetrics()
|
|
metrics.record_tokens(0) # must not raise
|
|
assert metrics.snapshot()["tokens_generated_total"] == 0
|
|
|
|
|
|
class TestTracingExtra:
|
|
def test_validate_otlp_rejects_private_ip(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError, match="private"):
|
|
validate_otlp_endpoint("https://192.168.1.10:4317")
|
|
|
|
def test_validate_otlp_rejects_link_local(self):
|
|
"""Cloud-metadata IP must be blocked (SSRF hardening)."""
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError, match="private|link"):
|
|
validate_otlp_endpoint("https://169.254.169.254:4317")
|
|
|
|
def test_validate_otlp_rejects_zero_host(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_otlp_endpoint("https://0.0.0.0:4317")
|
|
|
|
def test_validate_otlp_rejects_missing_host(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError, match="host"):
|
|
validate_otlp_endpoint("http:///path")
|
|
|
|
def test_validate_otlp_rejects_non_string(self):
|
|
from soup_cli.utils.tracing import validate_otlp_endpoint
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_otlp_endpoint(123) # type: ignore[arg-type]
|
|
|
|
|
|
class TestStructuredOutputExtra:
|
|
def test_cli_json_without_schema_errors(self, tmp_path):
|
|
"""--structured-output json must require --json-schema."""
|
|
pytest.importorskip("fastapi") # CLI exits early w/o FastAPI
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
model_dir = tmp_path / "model"
|
|
model_dir.mkdir()
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"serve",
|
|
"--model",
|
|
str(model_dir),
|
|
"--device",
|
|
"cpu",
|
|
"--structured-output",
|
|
"json",
|
|
],
|
|
)
|
|
assert result.exit_code != 0, result.output
|
|
assert "json-schema" in _strip_ansi(result.output)
|
|
|
|
|
|
class TestAutoQuantCLIWarning:
|
|
def test_auto_quant_logs_picker_choice(self, tmp_path):
|
|
"""v0.33.0 #54: --auto-quant runs the live picker and logs the
|
|
chosen candidate (not a deferral warning anymore)."""
|
|
pytest.importorskip("fastapi") # CLI exits early w/o FastAPI
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
model_dir = tmp_path / "model"
|
|
model_dir.mkdir()
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"serve",
|
|
"--model",
|
|
str(model_dir),
|
|
"--device",
|
|
"cpu",
|
|
"--auto-quant",
|
|
],
|
|
)
|
|
# Command will fail later (no real model); just check the picker
|
|
# ran (either picked a candidate or surfaced a controlled error).
|
|
output = _strip_ansi(result.output).lower()
|
|
assert "auto-quant" in output
|
|
|
|
|
|
class TestJsonSchemaContainment:
|
|
def test_json_schema_outside_cwd_rejected(self, tmp_path, monkeypatch):
|
|
"""JSON schema path must stay under cwd."""
|
|
pytest.importorskip("fastapi") # CLI exits early w/o FastAPI
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
# Put the schema OUTSIDE cwd
|
|
outside_dir = tmp_path / "outside"
|
|
outside_dir.mkdir()
|
|
schema_file = outside_dir / "schema.json"
|
|
schema_file.write_text('{"type": "object"}')
|
|
|
|
cwd = tmp_path / "work"
|
|
cwd.mkdir()
|
|
model_dir = cwd / "model"
|
|
model_dir.mkdir()
|
|
monkeypatch.chdir(cwd)
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"serve",
|
|
"--model",
|
|
str(model_dir),
|
|
"--device",
|
|
"cpu",
|
|
"--structured-output",
|
|
"json",
|
|
"--json-schema",
|
|
str(schema_file),
|
|
],
|
|
)
|
|
# Command may fail for other reasons too, but must reject the path
|
|
assert "under" in result.output.lower() or result.exit_code != 0
|