mirror of https://github.com/razor-ai/soup.git
279 lines
9.9 KiB
Python
279 lines
9.9 KiB
Python
"""Tests for ``--trust-remote-code`` opt-in (v0.36.0 Part B).
|
|
|
|
Replaces the previous unconditional ``trust_remote_code=True`` smell across
|
|
sft.py / chat.py / serve.py with an explicit, auditable opt-in flag plus a
|
|
trusted-org allowlist that suppresses noise on first-party models.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from io import StringIO
|
|
|
|
import pytest
|
|
from rich.console import Console
|
|
|
|
# Rich help renderer can split a flag like --trust-remote-code with ANSI
|
|
# colour escapes between `-`, `-trust`, `-remote-code` when the terminal
|
|
# is narrow (macOS CI runners hit this; Windows local does not). Strip
|
|
# ANSI so substring assertions are robust. Mirrors the helper in
|
|
# tests/test_log_level.py.
|
|
_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*[mK]")
|
|
|
|
|
|
def _strip_ansi(text: str) -> str:
|
|
return _ANSI_ESCAPE.sub("", text)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Allowlist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestKnownSafePrefixes:
|
|
def test_meta_llama_is_safe(self):
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert is_known_safe("meta-llama/Llama-3.2-1B")
|
|
|
|
def test_qwen_is_safe(self):
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert is_known_safe("Qwen/Qwen2.5-7B")
|
|
|
|
def test_mistral_is_safe(self):
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert is_known_safe("mistralai/Mistral-7B-Instruct-v0.3")
|
|
|
|
def test_random_org_not_safe(self):
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert not is_known_safe("randomuser/SomeModel")
|
|
|
|
def test_local_path_not_safe(self):
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert not is_known_safe("./local-checkpoint")
|
|
|
|
def test_partial_prefix_does_not_match(self):
|
|
"""`meta-llama-evil/...` must NOT match the `meta-llama/` prefix."""
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert not is_known_safe("meta-llama-evil/SomeModel")
|
|
|
|
def test_empty_string_not_safe(self):
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert not is_known_safe("")
|
|
|
|
def test_non_string_not_safe(self):
|
|
from soup_cli.utils.trust_remote import is_known_safe
|
|
|
|
assert not is_known_safe(None)
|
|
assert not is_known_safe(123)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# resolve_trust_remote_code — main entry
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestResolve:
|
|
def test_default_off_for_safe_prefix_passes_silently(self):
|
|
"""Trusted org + flag off → returns False, no warning."""
|
|
from soup_cli.utils.trust_remote import resolve_trust_remote_code
|
|
|
|
buf = StringIO()
|
|
console = Console(file=buf, force_terminal=False)
|
|
out = resolve_trust_remote_code(
|
|
"meta-llama/Llama-3.2-1B",
|
|
requested=False,
|
|
console=console,
|
|
requires_remote_code=False,
|
|
)
|
|
assert out is False
|
|
assert buf.getvalue() == ""
|
|
|
|
def test_flag_enabled_warns_once(self):
|
|
"""User opted in → return True + warning panel."""
|
|
from soup_cli.utils.trust_remote import resolve_trust_remote_code
|
|
|
|
buf = StringIO()
|
|
console = Console(file=buf, force_terminal=False)
|
|
out = resolve_trust_remote_code(
|
|
"shady-org/SomeModel",
|
|
requested=True,
|
|
console=console,
|
|
requires_remote_code=True,
|
|
)
|
|
assert out is True
|
|
output = buf.getvalue()
|
|
assert "trust_remote_code" in output.lower() or "remote code" in output.lower()
|
|
assert "shady-org/SomeModel" in output
|
|
|
|
def test_flag_enabled_safe_prefix_suppresses_warning(self):
|
|
"""Trusted org doesn't ship custom code — suppress warning even when flag set."""
|
|
from soup_cli.utils.trust_remote import resolve_trust_remote_code
|
|
|
|
buf = StringIO()
|
|
console = Console(file=buf, force_terminal=False)
|
|
out = resolve_trust_remote_code(
|
|
"meta-llama/Llama-3.2-1B",
|
|
requested=True,
|
|
console=console,
|
|
requires_remote_code=False,
|
|
)
|
|
assert out is True
|
|
# No noisy panel when the model is from a trusted prefix.
|
|
assert "WARNING" not in buf.getvalue().upper()
|
|
|
|
def test_default_off_for_unknown_with_remote_code_raises(self):
|
|
"""Model needs custom code + flag off → fail fast with actionable error."""
|
|
from soup_cli.utils.trust_remote import resolve_trust_remote_code
|
|
|
|
buf = StringIO()
|
|
console = Console(file=buf, force_terminal=False)
|
|
with pytest.raises(ValueError) as exc_info:
|
|
resolve_trust_remote_code(
|
|
"shady-org/CustomModel",
|
|
requested=False,
|
|
console=console,
|
|
requires_remote_code=True,
|
|
)
|
|
msg = str(exc_info.value)
|
|
assert "shady-org/CustomModel" in msg
|
|
assert "--trust-remote-code" in msg
|
|
|
|
def test_default_off_for_unknown_without_remote_code_passes(self):
|
|
"""Standard model + flag off → returns False, no error."""
|
|
from soup_cli.utils.trust_remote import resolve_trust_remote_code
|
|
|
|
buf = StringIO()
|
|
console = Console(file=buf, force_terminal=False)
|
|
out = resolve_trust_remote_code(
|
|
"shady-org/StandardLlama",
|
|
requested=False,
|
|
console=console,
|
|
requires_remote_code=False,
|
|
)
|
|
assert out is False
|
|
|
|
def test_console_optional(self):
|
|
"""resolve_trust_remote_code must work when console is None."""
|
|
from soup_cli.utils.trust_remote import resolve_trust_remote_code
|
|
|
|
out = resolve_trust_remote_code(
|
|
"meta-llama/Llama-3.2-1B",
|
|
requested=True,
|
|
console=None,
|
|
requires_remote_code=False,
|
|
)
|
|
assert out is True
|
|
|
|
def test_invalid_model_name_rejected(self):
|
|
from soup_cli.utils.trust_remote import resolve_trust_remote_code
|
|
|
|
with pytest.raises(ValueError, match="model_name"):
|
|
resolve_trust_remote_code(
|
|
"",
|
|
requested=False,
|
|
console=None,
|
|
requires_remote_code=False,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# model_requires_trust_remote_code (probe HF config for auto_map)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRequiresProbe:
|
|
def test_local_path_no_auto_map_returns_false(self, tmp_path, monkeypatch):
|
|
"""Local path with config.json lacking auto_map → False."""
|
|
from soup_cli.utils.trust_remote import model_requires_trust_remote_code
|
|
|
|
config = tmp_path / "config.json"
|
|
config.write_text('{"model_type": "llama"}', encoding="utf-8")
|
|
assert model_requires_trust_remote_code(str(tmp_path)) is False
|
|
|
|
def test_local_path_with_auto_map_returns_true(self, tmp_path):
|
|
from soup_cli.utils.trust_remote import model_requires_trust_remote_code
|
|
|
|
config = tmp_path / "config.json"
|
|
config.write_text(
|
|
'{"model_type": "custom", "auto_map": '
|
|
'{"AutoModelForCausalLM": "modeling.Custom"}}',
|
|
encoding="utf-8",
|
|
)
|
|
assert model_requires_trust_remote_code(str(tmp_path)) is True
|
|
|
|
def test_missing_config_returns_none(self, tmp_path):
|
|
"""Missing config.json → None (unknown — caller decides)."""
|
|
from soup_cli.utils.trust_remote import model_requires_trust_remote_code
|
|
|
|
out = model_requires_trust_remote_code(str(tmp_path))
|
|
assert out is None
|
|
|
|
def test_malformed_config_returns_none(self, tmp_path):
|
|
from soup_cli.utils.trust_remote import model_requires_trust_remote_code
|
|
|
|
config = tmp_path / "config.json"
|
|
config.write_text("{this is not json", encoding="utf-8")
|
|
out = model_requires_trust_remote_code(str(tmp_path))
|
|
assert out is None
|
|
|
|
def test_non_dict_root_returns_none(self, tmp_path):
|
|
"""Config with non-dict root (e.g. JSON array) → None."""
|
|
from soup_cli.utils.trust_remote import model_requires_trust_remote_code
|
|
|
|
config = tmp_path / "config.json"
|
|
config.write_text("[1, 2, 3]", encoding="utf-8")
|
|
out = model_requires_trust_remote_code(str(tmp_path))
|
|
assert out is None
|
|
|
|
def test_non_directory_path_returns_none(self):
|
|
"""Bare HF repo id (not a local dir) → None (unknown)."""
|
|
from soup_cli.utils.trust_remote import model_requires_trust_remote_code
|
|
|
|
out = model_requires_trust_remote_code("meta-llama/Llama-3.2-1B")
|
|
assert out is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI flag plumbing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCLIPlumbing:
|
|
"""Smoke check that --trust-remote-code is a registered Typer option."""
|
|
|
|
def test_train_help_lists_flag(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["train", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--trust-remote-code" in _strip_ansi(result.output), result.output
|
|
|
|
def test_chat_help_lists_flag(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["chat", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--trust-remote-code" in _strip_ansi(result.output), result.output
|
|
|
|
def test_serve_help_lists_flag(self):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["serve", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--trust-remote-code" in _strip_ansi(result.output), result.output
|