mirror of https://github.com/razor-ai/soup.git
735 lines
26 KiB
Python
735 lines
26 KiB
Python
"""v0.68.0 review-fix follow-ups.
|
|
|
|
Closes every TDD-review finding plus the additional manual code/security
|
|
review surface gaps:
|
|
|
|
- HIGH H1: `validate_student_id` rejection-matrix parity with teacher
|
|
- HIGH H2: Part C CLI `unknown-optimizer` exit 2 test
|
|
- HIGH H3: `harvest_dpo_pairs` edge cases (one-up-no-down / dedup / multi-prompt)
|
|
- HIGH H4: POSIX symlink skip predicate uses `sys.platform` not `hasattr(os, "symlink")`
|
|
- MED M1: `CompileResult` Inf rejection + bool-score rejection
|
|
- MED M2: `validate_eval_suite_path` symlink test
|
|
- MED M3: Part C `validate_tool_optimizer` empty + oversize tests
|
|
- MED M4: Part C `validate_spec_path` null-byte + symlink tests
|
|
- MED M5: `TestBuildToolCompilePlan` factory test (Part C)
|
|
- MED M6: Part D `validate_direction` non-string TypeError + oversize tests
|
|
- MED M7: `TestBuildAppleAdapterPlan` factory test (Part D)
|
|
- MED M8: Part E backend / train_method validators — full rejection matrix
|
|
- MED M9: `record_thumb` null-byte + oversize on `response`
|
|
- MED M10: `SUPPORTED_LOCAL_RL_TRAIN_METHODS` immutability test
|
|
- LOW L1: Part E `TestInitDb` column-level schema assertions
|
|
- LOW L2: Part D `output_dir` null-byte / oversize / outside-cwd tests
|
|
- LOW L3: `validate_db_path` is a public symbol
|
|
- LOW L4: `DpoPair` frozen + invariants
|
|
- Sec/code: `validate_db_path` exported in `__all__`
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cross-platform symlink test gate (HIGH H4)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _symlinks_available() -> bool:
|
|
"""``os.symlink`` exists on Windows but needs elevation. Use platform."""
|
|
return sys.platform != "win32"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part B — validate_student_id rejection-matrix parity (HIGH H1)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateStudentIdParity:
|
|
def test_oversize_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_distill import validate_student_id
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_student_id("a" * 513)
|
|
|
|
def test_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_distill import validate_student_id
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_student_id("a\x00b")
|
|
|
|
def test_bool_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_distill import validate_student_id
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_student_id(True) # type: ignore[arg-type]
|
|
|
|
def test_empty_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_distill import validate_student_id
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_student_id("")
|
|
|
|
def test_non_string_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_distill import validate_student_id
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_student_id(42) # type: ignore[arg-type]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part C — CLI unknown-optimizer exits 2 (HIGH H2)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPartCUnknownOptimizerCli:
|
|
def test_exits_2(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text("{}", encoding="utf-8")
|
|
eval_suite = tmp_path / "eval.jsonl"
|
|
eval_suite.write_text("[]", encoding="utf-8")
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"compile-tools",
|
|
str(spec),
|
|
"--eval",
|
|
str(eval_suite),
|
|
"--optimizer",
|
|
"evil",
|
|
"--plan-only",
|
|
],
|
|
)
|
|
assert result.exit_code == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — harvest_dpo_pairs edge cases (HIGH H3)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestHarvestEdgeCases:
|
|
def test_one_up_no_down_returns_empty(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import (
|
|
harvest_dpo_pairs,
|
|
init_local_rl_db,
|
|
record_thumb,
|
|
)
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
record_thumb(
|
|
db_path="rl.db", prompt="q", response="r", thumb="up"
|
|
)
|
|
assert harvest_dpo_pairs("rl.db") == ()
|
|
|
|
def test_one_down_no_up_returns_empty(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import (
|
|
harvest_dpo_pairs,
|
|
init_local_rl_db,
|
|
record_thumb,
|
|
)
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
record_thumb(
|
|
db_path="rl.db", prompt="q", response="r", thumb="down"
|
|
)
|
|
assert harvest_dpo_pairs("rl.db") == ()
|
|
|
|
def test_multiple_prompts_independent(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import (
|
|
harvest_dpo_pairs,
|
|
init_local_rl_db,
|
|
record_thumb,
|
|
)
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
for prompt, chosen, rejected in (("q1", "c1", "r1"), ("q2", "c2", "r2")):
|
|
record_thumb(db_path="rl.db", prompt=prompt, response=chosen, thumb="up")
|
|
record_thumb(
|
|
db_path="rl.db", prompt=prompt, response=rejected, thumb="down"
|
|
)
|
|
pairs = harvest_dpo_pairs("rl.db")
|
|
assert len(pairs) == 2
|
|
prompts = sorted(p.prompt for p in pairs)
|
|
assert prompts == ["q1", "q2"]
|
|
|
|
def test_dedup_keeps_latest_per_prompt(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""Duplicate thumbs for the same prompt collapse to one DPO pair."""
|
|
from soup_cli.utils.local_rl import (
|
|
harvest_dpo_pairs,
|
|
init_local_rl_db,
|
|
record_thumb,
|
|
)
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
# Two ups + two downs for the same prompt should still yield exactly one pair.
|
|
record_thumb(db_path="rl.db", prompt="q", response="up1", thumb="up")
|
|
record_thumb(db_path="rl.db", prompt="q", response="up2", thumb="up")
|
|
record_thumb(db_path="rl.db", prompt="q", response="down1", thumb="down")
|
|
record_thumb(db_path="rl.db", prompt="q", response="down2", thumb="down")
|
|
pairs = harvest_dpo_pairs("rl.db")
|
|
assert len(pairs) == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part A — CompileResult Inf + bool-score (MED M1)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCompileResultBoundaries:
|
|
def test_positive_inf_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
with pytest.raises(ValueError):
|
|
CompileResult(
|
|
program_text="# x",
|
|
score=float("inf"),
|
|
iterations=1,
|
|
converged=True,
|
|
)
|
|
|
|
def test_negative_inf_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
with pytest.raises(ValueError):
|
|
CompileResult(
|
|
program_text="# x",
|
|
score=float("-inf"),
|
|
iterations=1,
|
|
converged=True,
|
|
)
|
|
|
|
def test_bool_score_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
with pytest.raises(TypeError):
|
|
CompileResult(
|
|
program_text="# x",
|
|
score=True, # type: ignore[arg-type]
|
|
iterations=1,
|
|
converged=True,
|
|
)
|
|
|
|
def test_zero_iterations_accepted(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
result = CompileResult(
|
|
program_text="# x", score=0.5, iterations=0, converged=False
|
|
)
|
|
assert result.iterations == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part A — validate_eval_suite_path symlink test (MED M2)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.skipif(not _symlinks_available(), reason="POSIX symlink only")
|
|
class TestEvalSuitePathSymlink:
|
|
def test_symlink_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_eval_suite_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "suite.json"
|
|
real.write_text("[]", encoding="utf-8")
|
|
link = tmp_path / "link.json"
|
|
try:
|
|
os.symlink(real, link)
|
|
except (OSError, NotImplementedError):
|
|
pytest.skip("symlinks not creatable on this filesystem")
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
validate_eval_suite_path(str(link))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part C — validate_tool_optimizer empty + oversize (MED M3)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateToolOptimizerExtras:
|
|
def test_empty_rejected(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_tool_optimizer
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_tool_optimizer("")
|
|
|
|
def test_oversize_rejected(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_tool_optimizer
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_tool_optimizer("a" * 33)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part C — validate_spec_path null-byte + symlink (MED M4)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateSpecPathExtras:
|
|
def test_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_spec_path
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_spec_path("spec\x00.json")
|
|
|
|
@pytest.mark.skipif(not _symlinks_available(), reason="POSIX symlink only")
|
|
def test_symlink_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.compile_tools import validate_spec_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.json"
|
|
real.write_text("{}", encoding="utf-8")
|
|
link = tmp_path / "link.json"
|
|
try:
|
|
os.symlink(real, link)
|
|
except (OSError, NotImplementedError):
|
|
pytest.skip("symlinks not creatable on this filesystem")
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
validate_spec_path(str(link))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part C — build_tool_compile_plan factory (MED M5)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBuildToolCompilePlan:
|
|
def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.utils.compile_tools import build_tool_compile_plan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text("{}", encoding="utf-8")
|
|
eval_suite = tmp_path / "eval.jsonl"
|
|
eval_suite.write_text("[]", encoding="utf-8")
|
|
plan = build_tool_compile_plan(
|
|
spec_path=str(spec),
|
|
eval_suite_path=str(eval_suite),
|
|
optimizer="GEPA", # case-insensitive
|
|
output_path="tools.json",
|
|
)
|
|
assert plan.optimizer == "gepa"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part D — validate_direction non-string + oversize (MED M6)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateDirectionExtras:
|
|
def test_non_string_rejected(self) -> None:
|
|
from soup_cli.utils.apple_adapter import validate_direction
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_direction(42) # type: ignore[arg-type]
|
|
|
|
def test_oversize_rejected(self) -> None:
|
|
from soup_cli.utils.apple_adapter import validate_direction
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_direction("a" * 33)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part D — build_apple_adapter_plan factory (MED M7)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBuildAppleAdapterPlan:
|
|
def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.utils.apple_adapter import build_apple_adapter_plan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
adapter = tmp_path / "adapter"
|
|
adapter.mkdir()
|
|
(adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
|
|
plan = build_apple_adapter_plan(
|
|
source_dir=str(adapter),
|
|
output_dir="out",
|
|
direction="HF-TO-MLX", # case-insensitive
|
|
sign=False,
|
|
)
|
|
assert plan.direction == "hf-to-mlx"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part D — output_dir validation (LOW L2)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAppleAdapterOutputDirValidation:
|
|
def test_empty_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.apple_adapter import AppleAdapterPlan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
adapter = tmp_path / "a"
|
|
adapter.mkdir()
|
|
(adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
|
|
with pytest.raises(ValueError):
|
|
AppleAdapterPlan(
|
|
source_dir=str(adapter),
|
|
output_dir="",
|
|
direction="hf-to-mlx",
|
|
sign=False,
|
|
)
|
|
|
|
def test_null_byte_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.apple_adapter import AppleAdapterPlan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
adapter = tmp_path / "a"
|
|
adapter.mkdir()
|
|
(adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
|
|
with pytest.raises(ValueError):
|
|
AppleAdapterPlan(
|
|
source_dir=str(adapter),
|
|
output_dir="out\x00",
|
|
direction="hf-to-mlx",
|
|
sign=False,
|
|
)
|
|
|
|
def test_bool_output_dir_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.apple_adapter import AppleAdapterPlan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
adapter = tmp_path / "a"
|
|
adapter.mkdir()
|
|
(adapter / "adapter_config.json").write_text("{}", encoding="utf-8")
|
|
with pytest.raises(TypeError):
|
|
AppleAdapterPlan(
|
|
source_dir=str(adapter),
|
|
output_dir=True, # type: ignore[arg-type]
|
|
direction="hf-to-mlx",
|
|
sign=False,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — backend / train_method full rejection matrix (MED M8)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPartEValidatorParity:
|
|
def test_backend_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_backend
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_local_rl_backend("ollama\x00")
|
|
|
|
def test_backend_empty_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_backend
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_local_rl_backend("")
|
|
|
|
def test_backend_oversize_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_backend
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_local_rl_backend("a" * 33)
|
|
|
|
def test_backend_non_string_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_backend
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_local_rl_backend(42) # type: ignore[arg-type]
|
|
|
|
def test_train_method_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_train_method
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_local_rl_train_method("dpo\x00")
|
|
|
|
def test_train_method_empty_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_train_method
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_local_rl_train_method("")
|
|
|
|
def test_train_method_oversize_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_train_method
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_local_rl_train_method("a" * 33)
|
|
|
|
def test_train_method_non_string_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_train_method
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_local_rl_train_method(42) # type: ignore[arg-type]
|
|
|
|
def test_train_method_bool_rejected(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_local_rl_train_method
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_local_rl_train_method(True) # type: ignore[arg-type]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — record_thumb response validation (MED M9)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRecordThumbResponseValidation:
|
|
def test_response_null_byte_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import init_local_rl_db, record_thumb
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
with pytest.raises(ValueError):
|
|
record_thumb(
|
|
db_path="rl.db", prompt="p", response="r\x00", thumb="up"
|
|
)
|
|
|
|
def test_response_oversize_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import (
|
|
MAX_RESPONSE_LEN,
|
|
init_local_rl_db,
|
|
record_thumb,
|
|
)
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
with pytest.raises(ValueError):
|
|
record_thumb(
|
|
db_path="rl.db",
|
|
prompt="p",
|
|
response="a" * (MAX_RESPONSE_LEN + 1),
|
|
thumb="up",
|
|
)
|
|
|
|
def test_response_empty_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import init_local_rl_db, record_thumb
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
with pytest.raises(ValueError):
|
|
record_thumb(
|
|
db_path="rl.db", prompt="p", response="", thumb="up"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — train_method allowlist immutability (MED M10)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTrainMethodAllowlistImmutability:
|
|
def test_immutable(self) -> None:
|
|
from soup_cli.utils.local_rl import SUPPORTED_LOCAL_RL_TRAIN_METHODS
|
|
|
|
with pytest.raises(AttributeError):
|
|
SUPPORTED_LOCAL_RL_TRAIN_METHODS.add("ppo") # type: ignore[attr-defined]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — schema column-level assertions (LOW L1)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestInitDbColumnLevel:
|
|
def test_thumbs_columns(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import init_local_rl_db
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
with sqlite3.connect("rl.db") as conn:
|
|
rows = conn.execute("PRAGMA table_info(thumbs)").fetchall()
|
|
names = {r[1] for r in rows}
|
|
# Schema regression guard: every named column must exist.
|
|
for col in ("id", "ts", "prompt", "response", "thumb"):
|
|
assert col in names
|
|
|
|
def test_interactions_columns(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import init_local_rl_db
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
init_local_rl_db("rl.db")
|
|
with sqlite3.connect("rl.db") as conn:
|
|
rows = conn.execute("PRAGMA table_info(interactions)").fetchall()
|
|
names = {r[1] for r in rows}
|
|
for col in ("id", "ts", "prompt", "response"):
|
|
assert col in names
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — validate_db_path is public (LOW L3)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateDbPathPublic:
|
|
def test_importable(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_db_path # noqa: F401
|
|
|
|
def test_in_all(self) -> None:
|
|
from soup_cli.utils import local_rl
|
|
|
|
assert "validate_db_path" in local_rl.__all__
|
|
|
|
def test_rejects_null_byte(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_db_path
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_db_path("db\x00.db")
|
|
|
|
def test_rejects_bool(self) -> None:
|
|
from soup_cli.utils.local_rl import validate_db_path
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_db_path(True) # type: ignore[arg-type]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — DpoPair frozen invariant (LOW L4)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestDpoPairFrozen:
|
|
def test_frozen(self) -> None:
|
|
from soup_cli.utils.local_rl import DpoPair
|
|
|
|
pair = DpoPair(prompt="p", chosen="c", rejected="r")
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
pair.prompt = "evil" # type: ignore[misc]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Part E — harvest_dpo_pairs missing-file rejection (manual review)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestHarvestMissingFile:
|
|
def test_missing_db_raises(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.local_rl import harvest_dpo_pairs
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(FileNotFoundError):
|
|
harvest_dpo_pairs("no_such.db")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Source-grep regression guards (LOW)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestSourceGuards:
|
|
def test_no_shell_true_in_v068_modules(self) -> None:
|
|
"""No subprocess invocation in v0.68.0 utility modules."""
|
|
for stem in (
|
|
"prompt_compile",
|
|
"prompt_distill",
|
|
"compile_tools",
|
|
"apple_adapter",
|
|
"local_rl",
|
|
):
|
|
path = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli"
|
|
/ "utils"
|
|
/ f"{stem}.py"
|
|
)
|
|
text = path.read_text(encoding="utf-8")
|
|
assert "shell=True" not in text, f"{stem}.py contains shell=True"
|
|
|
|
def test_no_resolve_relative_to(self) -> None:
|
|
"""Project policy: realpath + commonpath, not Path.resolve()+relative_to."""
|
|
for stem in (
|
|
"prompt_compile",
|
|
"prompt_distill",
|
|
"compile_tools",
|
|
"apple_adapter",
|
|
"local_rl",
|
|
):
|
|
path = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli"
|
|
/ "utils"
|
|
/ f"{stem}.py"
|
|
)
|
|
text = path.read_text(encoding="utf-8")
|
|
assert ".resolve()" not in text or ".resolve().parent" in text, (
|
|
f"{stem}.py should not use Path.resolve() for containment"
|
|
)
|
|
|
|
def test_local_rl_cli_imports_public_helper(self) -> None:
|
|
"""CLI must import the public `validate_db_path`, not the private one."""
|
|
path = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli"
|
|
/ "commands"
|
|
/ "local_rl.py"
|
|
)
|
|
text = path.read_text(encoding="utf-8")
|
|
assert "_validate_db_path" not in text, (
|
|
"CLI must use public `validate_db_path`, not the private alias"
|
|
)
|
|
assert "validate_db_path" in text
|
|
|
|
def test_all_command_modules_use_typer_exit(self) -> None:
|
|
"""All v0.68 CLI command modules raise typer.Exit on rejection."""
|
|
for stem in (
|
|
"compile_cmd",
|
|
"distill_prompt",
|
|
"compile_tools",
|
|
"apple_adapter",
|
|
"local_rl",
|
|
):
|
|
path = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli"
|
|
/ "commands"
|
|
/ f"{stem}.py"
|
|
)
|
|
text = path.read_text(encoding="utf-8")
|
|
assert "typer.Exit" in text, f"{stem}.py missing typer.Exit"
|
|
|
|
def test_version_bumped(self) -> None:
|
|
from soup_cli import __version__
|
|
|
|
# Floor-check: must be at or above the v0.68.0 release.
|
|
major, minor, patch = (int(x) for x in __version__.split("."))
|
|
assert (major, minor) >= (0, 68)
|