"""Part F — Standalone hardening (v0.33.0). Tests for: - #21 RLVR code_exec_reward: OS-level isolation strategy detection + Linux unshare attempt + macOS sandbox-exec wrapper detection. - #22 checkpoint_intelligence.prune_checkpoints: TOCTOU-safe symlink handling via os.lstat + S_ISLNK and onerror abort on rmtree walk. """ from __future__ import annotations import os import stat import sys import pytest # --------------------------------------------------------------------------- # #22 — prune_checkpoints TOCTOU hardening # --------------------------------------------------------------------------- class TestPruneCheckpointsTOCTOU: def test_prune_skips_top_level_symlink_via_lstat(self, tmp_path): """Top-level symlink masquerading as a checkpoint dir must be skipped without following the link target.""" from soup_cli.eval.checkpoint_intelligence import CheckpointTracker # Real checkpoint to keep (tmp_path / "checkpoint-100").mkdir() # Decoy target outside the prune root outside = tmp_path.parent / "outside_target_dir" outside.mkdir(exist_ok=True) (outside / "sentinel.txt").write_text("must-not-delete", encoding="utf-8") link = tmp_path / "checkpoint-200" try: os.symlink(str(outside), str(link), target_is_directory=True) except (OSError, NotImplementedError): pytest.skip("symlink creation not permitted (Windows non-admin)") tracker = CheckpointTracker(metric="composite", keep_top=1) tracker.record(step=100, score=0.9) tracker.record(step=200, score=0.5) removed = tracker.prune_checkpoints(tmp_path) # The symlink must NOT be followed — sentinel survives assert (outside / "sentinel.txt").exists() # Symlink itself was skipped (not in removed list) assert 200 not in removed def test_prune_aborts_on_symlink_inside_checkpoint(self, tmp_path): """If rmtree encounters a symlink mid-walk inside a doomed checkpoint, it must abort instead of following it (defence-in-depth).""" from soup_cli.eval.checkpoint_intelligence import CheckpointTracker # Two checkpoints; we'll keep the top one ckpt_keep = tmp_path / "checkpoint-100" ckpt_keep.mkdir() ckpt_doomed = tmp_path / "checkpoint-200" ckpt_doomed.mkdir() # Plant a symlink INSIDE the doomed checkpoint pointing outside outside = tmp_path.parent / "siblings_must_survive" outside.mkdir(exist_ok=True) (outside / "secret.txt").write_text("keep-me", encoding="utf-8") nested_link = ckpt_doomed / "linked" try: os.symlink(str(outside), str(nested_link), target_is_directory=True) except (OSError, NotImplementedError): pytest.skip("symlink creation not permitted (Windows non-admin)") tracker = CheckpointTracker(metric="composite", keep_top=1) tracker.record(step=100, score=0.9) tracker.record(step=200, score=0.5) # Should not follow the nested symlink tracker.prune_checkpoints(tmp_path) assert (outside / "secret.txt").exists(), "rmtree followed a symlink" def test_prune_uses_lstat_for_symlink_check(self, tmp_path, monkeypatch): """Verify prune uses os.lstat-based check, not Path.is_symlink, so a broken symlink (target removed mid-walk) is still rejected.""" from soup_cli.eval import checkpoint_intelligence as ci # Create a broken symlink as 'checkpoint-300' broken = tmp_path / "checkpoint-300" try: os.symlink(str(tmp_path / "_nonexistent_"), str(broken)) except (OSError, NotImplementedError): pytest.skip("symlink creation not permitted") tracker = ci.CheckpointTracker(metric="composite", keep_top=1) tracker.record(step=100, score=0.9) tracker.record(step=300, score=0.5) removed = tracker.prune_checkpoints(tmp_path) assert 300 not in removed # --------------------------------------------------------------------------- # #21 — RLVR code_exec_reward OS-level isolation strategy # --------------------------------------------------------------------------- class TestCodeExecIsolationStrategy: def test_get_isolation_strategy_returns_known_value(self): from soup_cli.trainer.rewards import _get_isolation_strategy strategy = _get_isolation_strategy() assert strategy in {"namespaces", "sandbox-exec", "best-effort"} def test_isolation_strategy_linux_with_unshare(self, monkeypatch): from soup_cli.trainer import rewards # Use _compute_isolation_strategy (uncached) like the other tests so # the sys.platform patch actually takes effect — _get_isolation_strategy # may return a cached value populated on a different platform during # earlier tests / on the macOS / Windows CI runner. monkeypatch.setattr(sys, "platform", "linux") # Inject a fake os.unshare so the namespaces branch is reachable # regardless of the host kernel. monkeypatch.setattr(os, "unshare", lambda *_a, **_k: None, raising=False) if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"): rewards._ISOLATION_STRATEGY_CACHE = None strategy = rewards._compute_isolation_strategy() # On a Linux-shaped host with os.unshare present, the strategy must # be "namespaces". The "best-effort" branch is covered by the # separate "linux_unshare_unavailable" test. assert strategy == "namespaces" def test_isolation_strategy_macos_with_sandbox_exec(self, monkeypatch): import shutil as shutil_mod from soup_cli.trainer import rewards # Force fresh evaluation monkeypatch.setattr(sys, "platform", "darwin") monkeypatch.setattr(shutil_mod, "which", lambda name: ( "/usr/bin/sandbox-exec" if name == "sandbox-exec" else None )) # Bypass any module-level cache if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"): rewards._ISOLATION_STRATEGY_CACHE = None strategy = rewards._compute_isolation_strategy() assert strategy == "sandbox-exec" def test_isolation_strategy_macos_without_sandbox_exec(self, monkeypatch): import shutil as shutil_mod from soup_cli.trainer import rewards monkeypatch.setattr(sys, "platform", "darwin") monkeypatch.setattr(shutil_mod, "which", lambda _name: None) if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"): rewards._ISOLATION_STRATEGY_CACHE = None strategy = rewards._compute_isolation_strategy() assert strategy == "best-effort" def test_isolation_strategy_windows(self, monkeypatch): from soup_cli.trainer import rewards monkeypatch.setattr(sys, "platform", "win32") if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"): rewards._ISOLATION_STRATEGY_CACHE = None strategy = rewards._compute_isolation_strategy() assert strategy == "best-effort" def test_isolation_strategy_linux_unshare_unavailable(self, monkeypatch): from soup_cli.trainer import rewards monkeypatch.setattr(sys, "platform", "linux") # Pretend os.unshare doesn't exist if hasattr(os, "unshare"): monkeypatch.delattr(os, "unshare", raising=False) if hasattr(rewards, "_ISOLATION_STRATEGY_CACHE"): rewards._ISOLATION_STRATEGY_CACHE = None strategy = rewards._compute_isolation_strategy() assert strategy == "best-effort" def test_macos_sandbox_profile_blocks_network_and_writes(self): """The macOS sandbox profile must deny network and writes outside /tmp.""" from soup_cli.trainer.rewards import MACOS_SANDBOX_PROFILE # Profile must default-deny then explicitly allow narrow process needs assert "(deny default)" in MACOS_SANDBOX_PROFILE assert "network" in MACOS_SANDBOX_PROFILE # Must be a single-line or properly formatted scheme expression assert "(version 1)" in MACOS_SANDBOX_PROFILE # --------------------------------------------------------------------------- # Smoke-test that existing code_exec_reward path still works on this host # --------------------------------------------------------------------------- class TestCodeExecRewardSmoke: def test_correct_code_still_scores_one(self): from soup_cli.trainer.rewards import code_exec_reward completions = [[{"role": "assistant", "content": "```python\nprint(2+2)\n```"}]] scores = code_exec_reward(completions, expected=["4"]) assert scores == [1.0] def test_wrong_code_still_scores_zero(self): from soup_cli.trainer.rewards import code_exec_reward completions = [[{"role": "assistant", "content": "```python\nprint(3)\n```"}]] scores = code_exec_reward(completions, expected=["4"]) assert scores == [0.0] # --------------------------------------------------------------------------- # Helper: confirm S_ISLNK lstat-style check works # --------------------------------------------------------------------------- def test_lstat_islnk_detects_symlink(tmp_path): target = tmp_path / "real" target.mkdir() link = tmp_path / "link" try: os.symlink(str(target), str(link)) except (OSError, NotImplementedError): pytest.skip("symlinks not supported") assert stat.S_ISLNK(os.lstat(str(link)).st_mode) assert not stat.S_ISLNK(os.lstat(str(target)).st_mode)