mirror of https://github.com/razor-ai/soup.git
233 lines
9.5 KiB
Python
233 lines
9.5 KiB
Python
"""Part F — Standalone hardening (v0.33.0).
|
|
|
|
Tests for:
|
|
- #21 RLVR code_exec_reward: OS-level isolation strategy detection +
|
|
Linux unshare attempt + macOS sandbox-exec wrapper detection.
|
|
- #22 checkpoint_intelligence.prune_checkpoints: TOCTOU-safe symlink
|
|
handling via os.lstat + S_ISLNK and onerror abort on rmtree walk.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import stat
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# #22 — prune_checkpoints TOCTOU hardening
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPruneCheckpointsTOCTOU:
|
|
def test_prune_skips_top_level_symlink_via_lstat(self, tmp_path):
|
|
"""Top-level symlink masquerading as a checkpoint dir must be skipped
|
|
without following the link target."""
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
# Real checkpoint to keep
|
|
(tmp_path / "checkpoint-100").mkdir()
|
|
|
|
# Decoy target outside the prune root
|
|
outside = tmp_path.parent / "outside_target_dir"
|
|
outside.mkdir(exist_ok=True)
|
|
(outside / "sentinel.txt").write_text("must-not-delete", encoding="utf-8")
|
|
|
|
link = tmp_path / "checkpoint-200"
|
|
try:
|
|
os.symlink(str(outside), str(link), target_is_directory=True)
|
|
except (OSError, NotImplementedError):
|
|
pytest.skip("symlink creation not permitted (Windows non-admin)")
|
|
|
|
tracker = CheckpointTracker(metric="composite", keep_top=1)
|
|
tracker.record(step=100, score=0.9)
|
|
tracker.record(step=200, score=0.5)
|
|
|
|
removed = tracker.prune_checkpoints(tmp_path)
|
|
|
|
# The symlink must NOT be followed — sentinel survives
|
|
assert (outside / "sentinel.txt").exists()
|
|
# Symlink itself was skipped (not in removed list)
|
|
assert 200 not in removed
|
|
|
|
def test_prune_aborts_on_symlink_inside_checkpoint(self, tmp_path):
|
|
"""If rmtree encounters a symlink mid-walk inside a doomed checkpoint,
|
|
it must abort instead of following it (defence-in-depth)."""
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
# Two checkpoints; we'll keep the top one
|
|
ckpt_keep = tmp_path / "checkpoint-100"
|
|
ckpt_keep.mkdir()
|
|
ckpt_doomed = tmp_path / "checkpoint-200"
|
|
ckpt_doomed.mkdir()
|
|
|
|
# Plant a symlink INSIDE the doomed checkpoint pointing outside
|
|
outside = tmp_path.parent / "siblings_must_survive"
|
|
outside.mkdir(exist_ok=True)
|
|
(outside / "secret.txt").write_text("keep-me", encoding="utf-8")
|
|
|
|
nested_link = ckpt_doomed / "linked"
|
|
try:
|
|
os.symlink(str(outside), str(nested_link), target_is_directory=True)
|
|
except (OSError, NotImplementedError):
|
|
pytest.skip("symlink creation not permitted (Windows non-admin)")
|
|
|
|
tracker = CheckpointTracker(metric="composite", keep_top=1)
|
|
tracker.record(step=100, score=0.9)
|
|
tracker.record(step=200, score=0.5)
|
|
|
|
# Should not follow the nested symlink
|
|
tracker.prune_checkpoints(tmp_path)
|
|
|
|
assert (outside / "secret.txt").exists(), "rmtree followed a symlink"
|
|
|
|
def test_prune_uses_lstat_for_symlink_check(self, tmp_path, monkeypatch):
|
|
"""Verify prune uses os.lstat-based check, not Path.is_symlink, so a
|
|
broken symlink (target removed mid-walk) is still rejected."""
|
|
from soup_cli.eval import checkpoint_intelligence as ci
|
|
|
|
# Create a broken symlink as 'checkpoint-300'
|
|
broken = tmp_path / "checkpoint-300"
|
|
try:
|
|
os.symlink(str(tmp_path / "_nonexistent_"), str(broken))
|
|
except (OSError, NotImplementedError):
|
|
pytest.skip("symlink creation not permitted")
|
|
|
|
tracker = ci.CheckpointTracker(metric="composite", keep_top=1)
|
|
tracker.record(step=100, score=0.9)
|
|
tracker.record(step=300, score=0.5)
|
|
|
|
removed = tracker.prune_checkpoints(tmp_path)
|
|
assert 300 not in removed
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# #21 — RLVR code_exec_reward OS-level isolation strategy
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCodeExecIsolationStrategy:
|
|
def test_get_isolation_strategy_returns_known_value(self):
|
|
from soup_cli.trainer.rewards import _get_isolation_strategy
|
|
|
|
strategy = _get_isolation_strategy()
|
|
assert strategy in {"namespaces", "sandbox-exec", "best-effort"}
|
|
|
|
def test_isolation_strategy_linux_with_unshare(self, monkeypatch):
|
|
from soup_cli.trainer import rewards
|
|
|
|
# Use _compute_isolation_strategy (uncached) like the other tests so
|
|
# the sys.platform patch actually takes effect — _get_isolation_strategy
|
|
# may return a cached value populated on a different platform during
|
|
# earlier tests / on the macOS / Windows CI runner.
|
|
monkeypatch.setattr(sys, "platform", "linux")
|
|
# Inject a fake os.unshare so the namespaces branch is reachable
|
|
# regardless of the host kernel.
|
|
monkeypatch.setattr(os, "unshare", lambda *_a, **_k: None,
|
|
raising=False)
|
|
if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"):
|
|
rewards._ISOLATION_STRATEGY_CACHE = None
|
|
strategy = rewards._compute_isolation_strategy()
|
|
# On a Linux-shaped host with os.unshare present, the strategy must
|
|
# be "namespaces". The "best-effort" branch is covered by the
|
|
# separate "linux_unshare_unavailable" test.
|
|
assert strategy == "namespaces"
|
|
|
|
def test_isolation_strategy_macos_with_sandbox_exec(self, monkeypatch):
|
|
import shutil as shutil_mod
|
|
|
|
from soup_cli.trainer import rewards
|
|
|
|
# Force fresh evaluation
|
|
monkeypatch.setattr(sys, "platform", "darwin")
|
|
monkeypatch.setattr(shutil_mod, "which", lambda name: (
|
|
"/usr/bin/sandbox-exec" if name == "sandbox-exec" else None
|
|
))
|
|
# Bypass any module-level cache
|
|
if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"):
|
|
rewards._ISOLATION_STRATEGY_CACHE = None
|
|
strategy = rewards._compute_isolation_strategy()
|
|
assert strategy == "sandbox-exec"
|
|
|
|
def test_isolation_strategy_macos_without_sandbox_exec(self, monkeypatch):
|
|
import shutil as shutil_mod
|
|
|
|
from soup_cli.trainer import rewards
|
|
|
|
monkeypatch.setattr(sys, "platform", "darwin")
|
|
monkeypatch.setattr(shutil_mod, "which", lambda _name: None)
|
|
if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"):
|
|
rewards._ISOLATION_STRATEGY_CACHE = None
|
|
strategy = rewards._compute_isolation_strategy()
|
|
assert strategy == "best-effort"
|
|
|
|
def test_isolation_strategy_windows(self, monkeypatch):
|
|
from soup_cli.trainer import rewards
|
|
|
|
monkeypatch.setattr(sys, "platform", "win32")
|
|
if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"):
|
|
rewards._ISOLATION_STRATEGY_CACHE = None
|
|
strategy = rewards._compute_isolation_strategy()
|
|
assert strategy == "best-effort"
|
|
|
|
def test_isolation_strategy_linux_unshare_unavailable(self, monkeypatch):
|
|
from soup_cli.trainer import rewards
|
|
|
|
monkeypatch.setattr(sys, "platform", "linux")
|
|
# Pretend os.unshare doesn't exist
|
|
if hasattr(os, "unshare"):
|
|
monkeypatch.delattr(os, "unshare", raising=False)
|
|
if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"):
|
|
rewards._ISOLATION_STRATEGY_CACHE = None
|
|
strategy = rewards._compute_isolation_strategy()
|
|
assert strategy == "best-effort"
|
|
|
|
def test_macos_sandbox_profile_blocks_network_and_writes(self):
|
|
"""The macOS sandbox profile must deny network and writes outside /tmp."""
|
|
from soup_cli.trainer.rewards import MACOS_SANDBOX_PROFILE
|
|
|
|
# Profile must default-deny then explicitly allow narrow process needs
|
|
assert "(deny default)" in MACOS_SANDBOX_PROFILE
|
|
assert "network" in MACOS_SANDBOX_PROFILE
|
|
# Must be a single-line or properly formatted scheme expression
|
|
assert "(version 1)" in MACOS_SANDBOX_PROFILE
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Smoke-test that existing code_exec_reward path still works on this host
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCodeExecRewardSmoke:
|
|
def test_correct_code_still_scores_one(self):
|
|
from soup_cli.trainer.rewards import code_exec_reward
|
|
|
|
completions = [[{"role": "assistant", "content": "```python\nprint(2+2)\n```"}]]
|
|
scores = code_exec_reward(completions, expected=["4"])
|
|
assert scores == [1.0]
|
|
|
|
def test_wrong_code_still_scores_zero(self):
|
|
from soup_cli.trainer.rewards import code_exec_reward
|
|
|
|
completions = [[{"role": "assistant", "content": "```python\nprint(3)\n```"}]]
|
|
scores = code_exec_reward(completions, expected=["4"])
|
|
assert scores == [0.0]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper: confirm S_ISLNK lstat-style check works
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_lstat_islnk_detects_symlink(tmp_path):
|
|
target = tmp_path / "real"
|
|
target.mkdir()
|
|
link = tmp_path / "link"
|
|
try:
|
|
os.symlink(str(target), str(link))
|
|
except (OSError, NotImplementedError):
|
|
pytest.skip("symlinks not supported")
|
|
assert stat.S_ISLNK(os.lstat(str(link)).st_mode)
|
|
assert not stat.S_ISLNK(os.lstat(str(target)).st_mode)
|