mirror of https://github.com/razor-ai/soup.git
469 lines
16 KiB
Python
469 lines
16 KiB
Python
"""v0.64.0 review fix follow-ups.
|
|
|
|
Covers wave-1 review-fix gaps:
|
|
- HIGH H2/H3/H4 — symlink rejection on the new READ paths
|
|
(plan/apply YAML loader, tunability load_report, env_lock read_lock,
|
|
terraform_plan read_state).
|
|
- HIGH H5 — compute_dataset_sha cwd-containment + symlink rejection.
|
|
- HIGH H6 — symlink rejection on dataset_path through `soup plan`.
|
|
- MEDIUM M1 — compute_config_sha rejects non-JSON-serialisable values.
|
|
- MEDIUM M3 — read_state rejects non-bool `applied`.
|
|
- MEDIUM M4 — _activation_bytes overflow-clamps at sanity cap.
|
|
- MEDIUM M6 — flag_downstream_risk tight Llama-family allowlist.
|
|
- MEDIUM M7 — Windows CUDA path parse (vN.M pattern).
|
|
- MEDIUM M8 — load_report containment-before-existence ordering.
|
|
- LOW L3 — source-grep regression: atomic_write_text usage in all 3 writers.
|
|
- LOW L4 — license-advisor MAU upper cap.
|
|
- LOW L5 — `Sequence` imported from `collections.abc` not `typing`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HIGH H2 — plan `_load_yaml_config` rejects symlink on YAML path
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlinks")
|
|
def test_plan_yaml_symlink_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.commands.plan import _load_yaml_config
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.yaml"
|
|
real.write_text("base: x\n")
|
|
link = tmp_path / "link.yaml"
|
|
os.symlink(real, link)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
_load_yaml_config(str(link))
|
|
|
|
|
|
def test_plan_yaml_outside_cwd_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.commands.plan import _load_yaml_config
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
outside = tmp_path.parent / "evil.yaml"
|
|
outside.write_text("base: x\n")
|
|
with pytest.raises(ValueError, match="cwd"):
|
|
_load_yaml_config(str(outside))
|
|
|
|
|
|
def test_plan_yaml_null_byte_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.commands.plan import _load_yaml_config
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="null"):
|
|
_load_yaml_config("config\x00.yaml")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HIGH H3 — tunability load_report rejects symlink + null-byte
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlinks")
|
|
def test_tunability_load_report_symlink_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.tunability import load_report
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.json"
|
|
real.write_text("{}")
|
|
link = tmp_path / "link.json"
|
|
os.symlink(real, link)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
load_report(str(link))
|
|
|
|
|
|
def test_tunability_load_report_null_byte_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.tunability import load_report
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="null"):
|
|
load_report("report\x00.json")
|
|
|
|
|
|
def test_tunability_load_report_containment_before_existence(tmp_path, monkeypatch):
|
|
"""Outside-cwd path must raise ValueError(cwd), NOT FileNotFoundError.
|
|
|
|
Otherwise an attacker can distinguish "file exists out of cwd" from
|
|
"file missing" via the exception type — MEDIUM M8 review fix.
|
|
"""
|
|
from soup_cli.utils.tunability import load_report
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
outside = tmp_path.parent / "definitely-missing.json"
|
|
with pytest.raises(ValueError, match="cwd"):
|
|
load_report(str(outside))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HIGH H4 — env read_lock rejects symlink + null-byte + ordering
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlinks")
|
|
def test_env_read_lock_symlink_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.env_lock import read_lock
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.lock"
|
|
real.write_text("{}")
|
|
link = tmp_path / "link.lock"
|
|
os.symlink(real, link)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
read_lock(str(link))
|
|
|
|
|
|
def test_env_read_lock_null_byte_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.env_lock import read_lock
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="null"):
|
|
read_lock("lock\x00.lock")
|
|
|
|
|
|
def test_env_read_lock_outside_cwd_before_existence(tmp_path, monkeypatch):
|
|
from soup_cli.utils.env_lock import read_lock
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
outside = tmp_path.parent / "missing.lock"
|
|
with pytest.raises(ValueError, match="cwd"):
|
|
read_lock(str(outside))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HIGH H4 (terraform read_state) — symlink rejection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlinks")
|
|
def test_terraform_read_state_symlink_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.terraform_plan import read_state
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.tfstate"
|
|
real.write_text("{}")
|
|
link = tmp_path / "link.tfstate"
|
|
os.symlink(real, link)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
read_state(str(link))
|
|
|
|
|
|
def test_terraform_read_state_null_byte_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.terraform_plan import read_state
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="null"):
|
|
read_state("state\x00.tfstate")
|
|
|
|
|
|
def test_terraform_read_state_outside_cwd_before_existence(tmp_path, monkeypatch):
|
|
from soup_cli.utils.terraform_plan import read_state
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
outside = tmp_path.parent / "missing.tfstate"
|
|
with pytest.raises(ValueError, match="cwd"):
|
|
read_state(str(outside))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HIGH H5 — compute_dataset_sha cwd containment + symlink rejection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_compute_dataset_sha_empty_returns_zero():
|
|
from soup_cli.utils.terraform_plan import compute_dataset_sha
|
|
|
|
assert compute_dataset_sha("") == "0" * 64
|
|
|
|
|
|
def test_compute_dataset_sha_outside_cwd_returns_zero(tmp_path, monkeypatch):
|
|
"""Out-of-cwd path silently returns zero-hash (no file read).
|
|
|
|
Defends against `data.train: /etc/shadow` smuggling file contents
|
|
into the SHA — H5 review fix.
|
|
"""
|
|
from soup_cli.utils.terraform_plan import compute_dataset_sha
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
outside = tmp_path.parent / "evil.jsonl"
|
|
if outside.exists():
|
|
# Don't depend on the parent dir being writable.
|
|
return
|
|
sha = compute_dataset_sha(str(outside))
|
|
assert sha == "0" * 64
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlinks")
|
|
def test_compute_dataset_sha_symlink_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.terraform_plan import compute_dataset_sha
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.jsonl"
|
|
real.write_text("{}\n")
|
|
link = tmp_path / "link.jsonl"
|
|
os.symlink(real, link)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
compute_dataset_sha(str(link))
|
|
|
|
|
|
def test_compute_dataset_sha_null_byte_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.utils.terraform_plan import compute_dataset_sha
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="null"):
|
|
compute_dataset_sha("data\x00.jsonl")
|
|
|
|
|
|
def test_compute_dataset_sha_missing_returns_zero(tmp_path, monkeypatch):
|
|
from soup_cli.utils.terraform_plan import compute_dataset_sha
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
sha = compute_dataset_sha("missing.jsonl")
|
|
assert sha == "0" * 64
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MEDIUM M1 — compute_config_sha rejects non-JSON-serialisable values
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_compute_config_sha_strict_no_default_str():
|
|
"""A non-serialisable value must raise, not silently `str()` it."""
|
|
from soup_cli.utils.terraform_plan import compute_config_sha
|
|
|
|
# `set` is not JSON-serialisable
|
|
with pytest.raises(TypeError):
|
|
compute_config_sha({"a": {1, 2, 3}})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MEDIUM M3 — read_state rejects non-bool `applied`
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_read_state_rejects_non_bool_applied(tmp_path, monkeypatch):
|
|
import json
|
|
|
|
from soup_cli.utils.terraform_plan import read_state
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
bad = tmp_path / "bad.tfstate"
|
|
payload = {
|
|
"plan": {
|
|
"base": "m",
|
|
"task": "sft",
|
|
"config_sha": "a" * 64,
|
|
"dataset_sha": "b" * 64,
|
|
"estimated_cost_usd": 0.5,
|
|
"estimated_minutes": 10.0,
|
|
"peak_vram_gb": 8.0,
|
|
"spot_price_usd_per_hour": 0.30,
|
|
},
|
|
"applied": "yes", # str, not bool — must raise
|
|
"applied_at": None,
|
|
"run_id": None,
|
|
}
|
|
bad.write_text(json.dumps(payload))
|
|
with pytest.raises(ValueError, match="applied"):
|
|
read_state(str(bad))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MEDIUM M4 — _activation_bytes overflow clamp
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_activation_bytes_clamped_at_extreme():
|
|
"""Even at schema max seq_len * batch_size, result stays finite."""
|
|
import math
|
|
|
|
from soup_cli.utils.hardware_fit import HardwareFitInput, estimate_peak_vram_gb
|
|
|
|
inp = HardwareFitInput(
|
|
params_b=1000.0,
|
|
seq_len=1_048_576,
|
|
batch_size=1024,
|
|
optimizer="adamw_torch",
|
|
quant="none",
|
|
peft="full",
|
|
gradient_checkpointing=False,
|
|
)
|
|
bd = estimate_peak_vram_gb(inp)
|
|
assert math.isfinite(bd.total_gb)
|
|
assert math.isfinite(bd.activations_gb)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MEDIUM M6 — Llama community license allowlist (no .startswith match)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_flag_downstream_risk_unknown_llama_variant_does_not_trip_mau_gate():
|
|
"""A hypothetical `llama-permissive-2030` should NOT trigger MAU block.
|
|
|
|
Tight allowlist defends against future Meta licence id surprises.
|
|
"""
|
|
from soup_cli.utils.license_advisor import flag_downstream_risk
|
|
|
|
# `llama-permissive-2030` is not in `_LLAMA_COMMUNITY_LICENSES` AND
|
|
# is not in the known matrix, so it should fall through to the
|
|
# "unknown license -> warn" branch, not "block by MAU".
|
|
r = flag_downstream_risk(
|
|
license_id="llama-permissive-2030",
|
|
target="b2c",
|
|
monthly_active_users=800_000_000,
|
|
)
|
|
# severity might be warn (unknown) but NOT block-by-MAU.
|
|
assert "monthly active users" not in r.reason.lower() or r.severity != "block"
|
|
|
|
|
|
def test_flag_downstream_risk_llama_3_under_cap_is_warn_not_block():
|
|
from soup_cli.utils.license_advisor import flag_downstream_risk
|
|
|
|
r = flag_downstream_risk(
|
|
license_id="llama-3",
|
|
target="b2c",
|
|
monthly_active_users=100,
|
|
)
|
|
# Restricted-use on B2C with low MAU = warn, never block.
|
|
assert r.severity != "block"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MEDIUM M7 — Windows CUDA path parse (vN.M)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_detect_cuda_version_windows_path(monkeypatch):
|
|
"""`v12.1` token at end of a Windows-style path is extracted."""
|
|
from soup_cli.utils import env_lock
|
|
|
|
monkeypatch.setenv("CUDA_HOME", r"C:\Program Files\NVIDIA\CUDA\v12.1")
|
|
# Ensure no other env var pre-empts
|
|
monkeypatch.delenv("CUDA_VERSION", raising=False)
|
|
# Sidestep torch — drop it from sys.modules
|
|
monkeypatch.setattr(env_lock.sys, "modules", {**env_lock.sys.modules, "torch": None})
|
|
version = env_lock._detect_cuda_version()
|
|
assert version is not None
|
|
assert "12.1" in version
|
|
|
|
|
|
def test_detect_cuda_version_posix_path(monkeypatch):
|
|
from soup_cli.utils import env_lock
|
|
|
|
monkeypatch.setenv("CUDA_HOME", "/usr/local/cuda-12.1")
|
|
monkeypatch.delenv("CUDA_VERSION", raising=False)
|
|
monkeypatch.setattr(env_lock.sys, "modules", {**env_lock.sys.modules, "torch": None})
|
|
version = env_lock._detect_cuda_version()
|
|
assert version is not None
|
|
assert "12.1" in version
|
|
|
|
|
|
def test_detect_cuda_version_none_when_no_env(monkeypatch):
|
|
from soup_cli.utils import env_lock
|
|
|
|
monkeypatch.delenv("CUDA_HOME", raising=False)
|
|
monkeypatch.delenv("CUDA_VERSION", raising=False)
|
|
monkeypatch.setattr(env_lock.sys, "modules", {**env_lock.sys.modules, "torch": None})
|
|
# Should silently return None — no env, no torch.
|
|
assert env_lock._detect_cuda_version() is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LOW L3 — source-grep regression: atomic_write_text in all 3 writers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_tunability_uses_atomic_write_text():
|
|
src = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "utils" / "tunability.py"
|
|
text = src.read_text(encoding="utf-8")
|
|
assert "atomic_write_text" in text, "tunability.write_report must use shared helper"
|
|
|
|
|
|
def test_terraform_plan_uses_atomic_write_text():
|
|
src = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli" / "utils" / "terraform_plan.py"
|
|
)
|
|
text = src.read_text(encoding="utf-8")
|
|
assert "atomic_write_text" in text, "terraform_plan.write_state must use shared helper"
|
|
|
|
|
|
def test_env_lock_uses_atomic_write_text():
|
|
src = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "utils" / "env_lock.py"
|
|
text = src.read_text(encoding="utf-8")
|
|
assert "atomic_write_text" in text, "env_lock.write_lock must use shared helper"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LOW L4 — MAU upper cap
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_mau_oversize_rejected():
|
|
from soup_cli.utils.license_advisor import flag_downstream_risk
|
|
|
|
with pytest.raises(ValueError, match="monthly_active_users"):
|
|
flag_downstream_risk(
|
|
license_id="apache-2.0",
|
|
target="b2c",
|
|
monthly_active_users=10**18,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# LOW L5 — Sequence imported from collections.abc, not typing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_tunability_sequence_imported_from_collections_abc():
|
|
"""`typing.Sequence` doesn't work with isinstance on every Py 3.9 build.
|
|
|
|
Use `collections.abc.Sequence` for runtime isinstance checks.
|
|
"""
|
|
src = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "utils" / "tunability.py"
|
|
text = src.read_text(encoding="utf-8")
|
|
# Either explicit `from collections.abc import Sequence`, or NO
|
|
# mention of `typing.Sequence` in an isinstance context.
|
|
assert "from collections.abc import Sequence" in text, (
|
|
"tunability.py must import Sequence from collections.abc"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Drift-on-modify regression — `soup plan` -> mutate -> `soup apply`
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_apply_refuses_drift_end_to_end(tmp_path, monkeypatch):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
monkeypatch.chdir(tmp_path)
|
|
cfg = tmp_path / "soup.yaml"
|
|
cfg.write_text(
|
|
"base: meta-llama/Llama-3.2-1B\n"
|
|
"task: sft\n"
|
|
"data:\n"
|
|
" train: ./data.jsonl\n"
|
|
"training:\n"
|
|
" epochs: 1\n"
|
|
" lr: 0.00005\n"
|
|
" batch_size: 4\n"
|
|
)
|
|
(tmp_path / "data.jsonl").write_text("{}\n")
|
|
r = runner.invoke(app, ["plan", "--config", str(cfg)])
|
|
assert r.exit_code == 0, r.output
|
|
cfg.write_text(cfg.read_text().replace("epochs: 1", "epochs: 5"))
|
|
r2 = runner.invoke(app, ["apply", "--config", str(cfg), "--dry-run"])
|
|
assert r2.exit_code == 3, r2.output # exit 3 = drift refused
|
|
assert "drift" in r2.output.lower()
|