mirror of https://github.com/razor-ai/soup.git
432 lines
16 KiB
Python
432 lines
16 KiB
Python
"""Tests for tool-calling / agentic fine-tuning pipeline (Part B of v0.25.0)."""
|
|
|
|
import json
|
|
from io import StringIO
|
|
|
|
import pytest
|
|
from rich.console import Console
|
|
from typer.testing import CliRunner
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Format detection + normalization
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallingFormat:
|
|
"""Format detection + normalization for tool-calling data."""
|
|
|
|
@staticmethod
|
|
def _row():
|
|
return {
|
|
"messages": [
|
|
{"role": "user", "content": "What's the weather in Tokyo?"},
|
|
],
|
|
"tools": [{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"description": "Get current weather for a city",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"city": {"type": "string"},
|
|
},
|
|
"required": ["city"],
|
|
},
|
|
},
|
|
}],
|
|
"tool_calls": [{
|
|
"function": {
|
|
"name": "get_weather",
|
|
"arguments": "{\"city\": \"Tokyo\"}",
|
|
},
|
|
}],
|
|
}
|
|
|
|
def test_detect_format(self):
|
|
from soup_cli.data.formats import detect_format
|
|
|
|
row = self._row()
|
|
assert detect_format([row]) == "tool-calling"
|
|
|
|
def test_normalize_row(self):
|
|
from soup_cli.data.formats import format_to_messages
|
|
|
|
result = format_to_messages(self._row(), "tool-calling")
|
|
assert result is not None
|
|
messages = result["messages"]
|
|
# First message is system with embedded tool schema
|
|
assert messages[0]["role"] == "system"
|
|
assert "get_weather" in messages[0]["content"]
|
|
# User message is preserved
|
|
assert any(m["role"] == "user" and "Tokyo" in m["content"] for m in messages)
|
|
# Assistant message with tool_calls appended
|
|
assistant_msgs = [m for m in messages if m["role"] == "assistant"]
|
|
assert len(assistant_msgs) == 1
|
|
assert "tool_calls" in assistant_msgs[0]
|
|
assert assistant_msgs[0]["tool_calls"][0]["function"]["name"] == "get_weather"
|
|
|
|
def test_detect_prefers_tool_calling_over_chatml(self):
|
|
"""tool-calling check must come before chatml, since chatml sig is a subset."""
|
|
from soup_cli.data.formats import detect_format
|
|
|
|
row = self._row()
|
|
assert detect_format([row]) == "tool-calling"
|
|
|
|
def test_normalize_invalid_tools_returns_none(self):
|
|
from soup_cli.data.formats import format_to_messages
|
|
|
|
bad = {
|
|
"messages": [{"role": "user", "content": "x"}],
|
|
"tools": "not-a-list",
|
|
"tool_calls": [],
|
|
}
|
|
assert format_to_messages(bad, "tool-calling") is None
|
|
|
|
def test_normalize_invalid_tool_calls_args_returns_none(self):
|
|
"""Non-JSON parseable tool_call arguments are rejected."""
|
|
from soup_cli.data.formats import format_to_messages
|
|
|
|
bad = {
|
|
"messages": [{"role": "user", "content": "x"}],
|
|
"tools": [
|
|
{
|
|
"type": "function",
|
|
"function": {"name": "f", "parameters": {"type": "object"}},
|
|
}
|
|
],
|
|
"tool_calls": [
|
|
{"function": {"name": "f", "arguments": "not-json-content{"}}
|
|
],
|
|
}
|
|
assert format_to_messages(bad, "tool-calling") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Synth data template
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallingTemplate:
|
|
def test_build_prompt_contains_domains_and_count(self):
|
|
from soup_cli.data.templates.tool_calling import build_prompt
|
|
|
|
prompt = build_prompt(count=5, fmt="tool-calling", format_spec="{...}", domain="weather")
|
|
assert "5" in prompt
|
|
assert "weather" in prompt.lower()
|
|
assert "function" in prompt.lower() or "tool" in prompt.lower()
|
|
|
|
def test_build_prompt_default_domain(self):
|
|
from soup_cli.data.templates.tool_calling import build_prompt
|
|
|
|
prompt = build_prompt(count=3, fmt="tool-calling", format_spec="{}")
|
|
assert "3" in prompt
|
|
|
|
def test_available_domains(self):
|
|
from soup_cli.data.templates.tool_calling import TEMPLATE_SPEC
|
|
|
|
for key in ("weather", "search", "database", "filesystem"):
|
|
assert key in TEMPLATE_SPEC["domains"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DataConfig literal accepts tool-calling
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallingConfig:
|
|
def test_dataconfig_accepts_tool_calling(self):
|
|
from soup_cli.config.schema import DataConfig
|
|
|
|
cfg = DataConfig(train="data.jsonl", format="tool-calling")
|
|
assert cfg.format == "tool-calling"
|
|
|
|
def test_soupconfig_tool_calling(self):
|
|
from soup_cli.config.loader import load_config_from_string
|
|
|
|
yaml_str = """
|
|
base: meta-llama/Llama-3.1-8B-Instruct
|
|
task: sft
|
|
data:
|
|
train: ./data/train.jsonl
|
|
format: tool-calling
|
|
training:
|
|
epochs: 1
|
|
lr: 2e-4
|
|
output: ./output
|
|
"""
|
|
cfg = load_config_from_string(yaml_str)
|
|
assert cfg.data.format == "tool-calling"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Init template
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallingInitTemplate:
|
|
def test_tool_calling_template_in_templates(self):
|
|
from soup_cli.config.schema import TEMPLATES
|
|
|
|
assert "tool-calling" in TEMPLATES
|
|
|
|
def test_tool_calling_template_loads(self):
|
|
from soup_cli.config.loader import load_config_from_string
|
|
from soup_cli.config.schema import TEMPLATES
|
|
|
|
cfg = load_config_from_string(TEMPLATES["tool-calling"])
|
|
assert cfg.data.format == "tool-calling"
|
|
assert cfg.task == "sft"
|
|
|
|
def test_init_with_tool_calling_template(self, tmp_path, monkeypatch):
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
result = runner.invoke(app, [
|
|
"init", "--template", "tool-calling",
|
|
])
|
|
assert result.exit_code == 0
|
|
soup_yaml = tmp_path / "soup.yaml"
|
|
assert soup_yaml.exists()
|
|
content = soup_yaml.read_text(encoding="utf-8")
|
|
assert "tool-calling" in content
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Eval scoring functions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallScoring:
|
|
"""Scoring functions for tool-call evaluation."""
|
|
|
|
def test_tool_call_match_exact(self):
|
|
from soup_cli.eval.custom import tool_call_match
|
|
|
|
output = json.dumps({
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"Tokyo\"}"},
|
|
})
|
|
expected = json.dumps({
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"Tokyo\"}"},
|
|
})
|
|
assert tool_call_match(output, expected) is True
|
|
|
|
def test_tool_call_match_wrong_name(self):
|
|
from soup_cli.eval.custom import tool_call_match
|
|
|
|
output = json.dumps({
|
|
"function": {"name": "get_time", "arguments": "{\"city\": \"Tokyo\"}"},
|
|
})
|
|
expected = json.dumps({
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"Tokyo\"}"},
|
|
})
|
|
assert tool_call_match(output, expected) is False
|
|
|
|
def test_tool_call_match_wrong_args(self):
|
|
from soup_cli.eval.custom import tool_call_match
|
|
|
|
output = json.dumps({
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"Kyoto\"}"},
|
|
})
|
|
expected = json.dumps({
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"Tokyo\"}"},
|
|
})
|
|
assert tool_call_match(output, expected) is False
|
|
|
|
def test_tool_call_name_match_only_name(self):
|
|
from soup_cli.eval.custom import tool_call_name_match
|
|
|
|
output = json.dumps({
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"Osaka\"}"},
|
|
})
|
|
expected = json.dumps({
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"Tokyo\"}"},
|
|
})
|
|
assert tool_call_name_match(output, expected) is True
|
|
|
|
def test_tool_call_args_subset_partial(self):
|
|
from soup_cli.eval.custom import tool_call_args_subset
|
|
|
|
# Expected args is a subset of output args — partial credit
|
|
output = json.dumps({
|
|
"function": {
|
|
"name": "search",
|
|
"arguments": "{\"query\": \"cats\", \"limit\": 10}",
|
|
},
|
|
})
|
|
expected = json.dumps({
|
|
"function": {
|
|
"name": "search",
|
|
"arguments": "{\"query\": \"cats\"}",
|
|
},
|
|
})
|
|
score = tool_call_args_subset(output, expected)
|
|
assert score > 0.9 # name matches + query matches
|
|
|
|
def test_tool_call_args_subset_no_match(self):
|
|
from soup_cli.eval.custom import tool_call_args_subset
|
|
|
|
output = json.dumps({
|
|
"function": {"name": "search", "arguments": "{\"query\": \"dogs\"}"},
|
|
})
|
|
expected = json.dumps({
|
|
"function": {"name": "search", "arguments": "{\"query\": \"cats\"}"},
|
|
})
|
|
score = tool_call_args_subset(output, expected)
|
|
assert score < 1.0
|
|
|
|
def test_tool_call_match_invalid_json_returns_false(self):
|
|
from soup_cli.eval.custom import tool_call_match
|
|
|
|
assert tool_call_match("not-json", "{}") is False
|
|
|
|
def test_tool_call_scoring_registered(self):
|
|
"""tool_call_* scorings are registered for use as eval 'scoring' strings."""
|
|
from soup_cli.eval.custom import VALID_SCORING
|
|
|
|
assert "tool_call_match" in VALID_SCORING
|
|
assert "tool_call_name_match" in VALID_SCORING
|
|
assert "tool_call_args_subset" in VALID_SCORING
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Recipes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallingRecipes:
|
|
def test_qwen3_8b_tools_recipe(self):
|
|
from soup_cli.config.loader import load_config_from_string
|
|
from soup_cli.recipes.catalog import get_recipe
|
|
|
|
recipe = get_recipe("qwen3-8b-tools")
|
|
assert recipe is not None
|
|
cfg = load_config_from_string(recipe.yaml_str)
|
|
assert cfg.data.format == "tool-calling"
|
|
assert cfg.task == "sft"
|
|
|
|
def test_llama4_scout_tools_recipe(self):
|
|
from soup_cli.config.loader import load_config_from_string
|
|
from soup_cli.recipes.catalog import get_recipe
|
|
|
|
recipe = get_recipe("llama4-scout-tools")
|
|
assert recipe is not None
|
|
cfg = load_config_from_string(recipe.yaml_str)
|
|
assert cfg.data.format == "tool-calling"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Round-trip and security
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallingRoundTrip:
|
|
def test_validate_and_stats_tool_calling(self, tmp_path):
|
|
from soup_cli.data.loader import load_raw_data
|
|
from soup_cli.data.validator import validate_and_stats
|
|
|
|
row = {
|
|
"messages": [{"role": "user", "content": "Query weather"}],
|
|
"tools": [{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"city": {"type": "string"}},
|
|
},
|
|
},
|
|
}],
|
|
"tool_calls": [{
|
|
"function": {"name": "get_weather", "arguments": "{\"city\": \"NYC\"}"},
|
|
}],
|
|
}
|
|
|
|
data_file = tmp_path / "tools.jsonl"
|
|
data_file.write_text(json.dumps(row) + "\n", encoding="utf-8")
|
|
|
|
data = load_raw_data(data_file)
|
|
stats = validate_and_stats(data, "tool-calling")
|
|
assert stats["total"] == 1
|
|
# Render via StringIO console (project convention)
|
|
console = Console(file=StringIO())
|
|
console.print(f"Stats: {stats}")
|
|
assert "total" in console.file.getvalue()
|
|
|
|
def test_empty_tool_calls_with_partial_schema(self):
|
|
"""Partial tool schema with empty tool_calls still produces normalized messages."""
|
|
from soup_cli.data.formats import format_to_messages
|
|
|
|
row = {
|
|
"messages": [{"role": "user", "content": "x"}],
|
|
"tools": [{"type": "function"}],
|
|
"tool_calls": [],
|
|
}
|
|
result = format_to_messages(row, "tool-calling")
|
|
assert result is not None
|
|
assert "messages" in result
|
|
# Only the synthesized system + the user message
|
|
assert any(m["role"] == "system" for m in result["messages"])
|
|
|
|
def test_tool_call_non_dict_rejected(self):
|
|
"""A non-dict tool_calls entry is rejected."""
|
|
from soup_cli.data.formats import format_to_messages
|
|
|
|
row = {
|
|
"messages": [{"role": "user", "content": "x"}],
|
|
"tools": [{"type": "function", "function": {"name": "f"}}],
|
|
"tool_calls": ["not-a-dict"],
|
|
}
|
|
assert format_to_messages(row, "tool-calling") is None
|
|
|
|
def test_tool_call_referencing_unknown_function_still_normalizes(self):
|
|
"""Cross-reference: tool_calls may name a function not in tools.
|
|
|
|
v0.25.0 does not enforce strict tools↔tool_calls cross-reference — the
|
|
format normalizer passes the call through so the model learns what the
|
|
user actually asked for. This test documents that contract.
|
|
"""
|
|
from soup_cli.data.formats import format_to_messages
|
|
|
|
row = {
|
|
"messages": [{"role": "user", "content": "x"}],
|
|
"tools": [{"type": "function", "function": {"name": "get_weather"}}],
|
|
"tool_calls": [{"function": {"name": "search_web", "arguments": "{}"}}],
|
|
}
|
|
result = format_to_messages(row, "tool-calling")
|
|
assert result is not None
|
|
# The assistant turn carries the original (unknown) function name
|
|
assistant_msgs = [m for m in result["messages"] if m["role"] == "assistant"]
|
|
assert len(assistant_msgs) == 1
|
|
assert assistant_msgs[0]["tool_calls"][0]["function"]["name"] == "search_web"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Data validator accepts tool-calling format
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolCallingValidator:
|
|
def test_validator_accepts_tool_calling_format(self, tmp_path):
|
|
import json as json_mod
|
|
|
|
from soup_cli.data.loader import load_raw_data
|
|
from soup_cli.data.validator import validate_and_stats
|
|
|
|
rows = [
|
|
{
|
|
"messages": [{"role": "user", "content": f"q{i}"}],
|
|
"tools": [{"type": "function", "function": {
|
|
"name": "f",
|
|
"parameters": {"type": "object"},
|
|
}}],
|
|
"tool_calls": [{"function": {"name": "f", "arguments": "{}"}}],
|
|
}
|
|
for i in range(3)
|
|
]
|
|
path = tmp_path / "toolcalls.jsonl"
|
|
path.write_text("\n".join(json_mod.dumps(r) for r in rows) + "\n", encoding="utf-8")
|
|
|
|
data = load_raw_data(path)
|
|
stats = validate_and_stats(data, "tool-calling")
|
|
assert stats["total"] == 3
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|