mirror of https://github.com/razor-ai/soup.git
460 lines
14 KiB
Python
460 lines
14 KiB
Python
"""v0.63.0 Part A — soup ingest universal trace importer tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import json
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Module surface
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_module_imports():
|
|
from soup_cli.utils import ingest_sources # noqa: F401
|
|
|
|
assert hasattr(ingest_sources, "SUPPORTED_INGEST_SOURCES")
|
|
assert hasattr(ingest_sources, "TraceRecord")
|
|
assert hasattr(ingest_sources, "validate_source_name")
|
|
assert hasattr(ingest_sources, "parse_langfuse")
|
|
assert hasattr(ingest_sources, "parse_langsmith")
|
|
assert hasattr(ingest_sources, "parse_helicone")
|
|
assert hasattr(ingest_sources, "parse_openpipe")
|
|
assert hasattr(ingest_sources, "parse_otel")
|
|
assert hasattr(ingest_sources, "parse_openai_stored")
|
|
assert hasattr(ingest_sources, "ingest_traces")
|
|
assert hasattr(ingest_sources, "resolve_auth_env")
|
|
|
|
|
|
def test_supported_sources_exact():
|
|
from soup_cli.utils.ingest_sources import SUPPORTED_INGEST_SOURCES
|
|
|
|
assert SUPPORTED_INGEST_SOURCES == frozenset(
|
|
{"langfuse", "langsmith", "helicone", "openpipe", "otel", "openai-stored"}
|
|
)
|
|
|
|
|
|
def test_trace_record_frozen():
|
|
from soup_cli.utils.ingest_sources import TraceRecord
|
|
|
|
rec = TraceRecord(
|
|
trace_id="abc",
|
|
prompt="hello",
|
|
output="world",
|
|
source="langfuse",
|
|
signal="none",
|
|
metadata={},
|
|
)
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
rec.prompt = "tampered" # type: ignore[misc]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_source_name
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_validate_source_name_happy():
|
|
from soup_cli.utils.ingest_sources import validate_source_name
|
|
|
|
for name in ["langfuse", "langsmith", "helicone", "openpipe", "otel", "openai-stored"]:
|
|
assert validate_source_name(name) == name
|
|
|
|
|
|
def test_validate_source_name_case_insensitive():
|
|
from soup_cli.utils.ingest_sources import validate_source_name
|
|
|
|
assert validate_source_name("LANGFUSE") == "langfuse"
|
|
assert validate_source_name("OpenPipe") == "openpipe"
|
|
|
|
|
|
def test_validate_source_name_unknown():
|
|
from soup_cli.utils.ingest_sources import validate_source_name
|
|
|
|
with pytest.raises(ValueError, match="unknown"):
|
|
validate_source_name("evilcorp")
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [None, 123, True, "", "x" * 33, "lang\x00fuse"])
|
|
def test_validate_source_name_rejects(bad):
|
|
from soup_cli.utils.ingest_sources import validate_source_name
|
|
|
|
with pytest.raises((TypeError, ValueError)):
|
|
validate_source_name(bad)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-source parsers (offline, no network)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_parse_langfuse_basic():
|
|
from soup_cli.utils.ingest_sources import parse_langfuse
|
|
|
|
events = [
|
|
{
|
|
"id": "trace-1",
|
|
"input": {"messages": [{"role": "user", "content": "hello"}]},
|
|
"output": "hi there",
|
|
},
|
|
{
|
|
"id": "trace-2",
|
|
"input": "raw prompt",
|
|
"output": {"content": "raw response"},
|
|
},
|
|
]
|
|
rows = list(parse_langfuse(events))
|
|
assert len(rows) == 2
|
|
assert rows[0].trace_id == "trace-1"
|
|
assert rows[0].prompt == "hello"
|
|
assert rows[0].output == "hi there"
|
|
assert rows[0].source == "langfuse"
|
|
assert rows[1].prompt == "raw prompt"
|
|
assert rows[1].output == "raw response"
|
|
|
|
|
|
def test_parse_langfuse_skips_non_dict():
|
|
from soup_cli.utils.ingest_sources import parse_langfuse
|
|
|
|
rows = list(parse_langfuse(["string-row", 42, None, {"input": "x", "output": "y"}]))
|
|
assert len(rows) == 1
|
|
|
|
|
|
def test_parse_langfuse_missing_fields():
|
|
from soup_cli.utils.ingest_sources import parse_langfuse
|
|
|
|
rows = list(parse_langfuse([{"id": "x"}, {"input": "only-input"}, {"output": "only-output"}]))
|
|
assert rows == []
|
|
|
|
|
|
def test_parse_langsmith_basic():
|
|
from soup_cli.utils.ingest_sources import parse_langsmith
|
|
|
|
events = [
|
|
{
|
|
"id": "run-1",
|
|
"inputs": {"messages": [{"role": "user", "content": "Q"}]},
|
|
"outputs": {"generations": [[{"text": "A"}]]},
|
|
"feedback_stats": {"thumbs": {"avg": 1.0}},
|
|
},
|
|
]
|
|
rows = list(parse_langsmith(events))
|
|
assert len(rows) == 1
|
|
assert rows[0].prompt == "Q"
|
|
assert rows[0].output == "A"
|
|
assert rows[0].source == "langsmith"
|
|
assert rows[0].signal == "thumbs_up"
|
|
|
|
|
|
def test_parse_helicone_basic():
|
|
from soup_cli.utils.ingest_sources import parse_helicone
|
|
|
|
events = [
|
|
{
|
|
"request_id": "req-1",
|
|
"request": {"body": {"messages": [{"role": "user", "content": "hi"}]}},
|
|
"response": {"body": {"choices": [{"message": {"content": "bye"}}]}},
|
|
}
|
|
]
|
|
rows = list(parse_helicone(events))
|
|
assert len(rows) == 1
|
|
assert rows[0].trace_id == "req-1"
|
|
assert rows[0].prompt == "hi"
|
|
assert rows[0].output == "bye"
|
|
assert rows[0].source == "helicone"
|
|
|
|
|
|
def test_parse_openpipe_basic():
|
|
from soup_cli.utils.ingest_sources import parse_openpipe
|
|
|
|
events = [
|
|
{
|
|
"id": "op-1",
|
|
"messages": [{"role": "user", "content": "hello"}],
|
|
"response": "world",
|
|
}
|
|
]
|
|
rows = list(parse_openpipe(events))
|
|
assert len(rows) == 1
|
|
assert rows[0].source == "openpipe"
|
|
|
|
|
|
def test_parse_otel_basic():
|
|
from soup_cli.utils.ingest_sources import parse_otel
|
|
|
|
spans = [
|
|
{
|
|
"traceId": "abc123",
|
|
"attributes": {
|
|
"llm.prompt": "what is 2+2",
|
|
"llm.completion": "4",
|
|
"llm.model": "gpt-4o",
|
|
},
|
|
}
|
|
]
|
|
rows = list(parse_otel(spans))
|
|
assert len(rows) == 1
|
|
assert rows[0].prompt == "what is 2+2"
|
|
assert rows[0].output == "4"
|
|
assert rows[0].source == "otel"
|
|
assert rows[0].metadata.get("model") == "gpt-4o"
|
|
|
|
|
|
def test_parse_otel_skips_non_llm_spans():
|
|
from soup_cli.utils.ingest_sources import parse_otel
|
|
|
|
spans = [
|
|
{"attributes": {"http.method": "GET"}},
|
|
{"attributes": {"llm.prompt": "Q", "llm.completion": "A"}},
|
|
]
|
|
rows = list(parse_otel(spans))
|
|
assert len(rows) == 1
|
|
|
|
|
|
def test_parse_openai_stored_basic():
|
|
from soup_cli.utils.ingest_sources import parse_openai_stored
|
|
|
|
events = [
|
|
{
|
|
"id": "chatcmpl-1",
|
|
"input": [{"role": "user", "content": "hello"}],
|
|
"output": [{"role": "assistant", "content": "hi"}],
|
|
}
|
|
]
|
|
rows = list(parse_openai_stored(events))
|
|
assert len(rows) == 1
|
|
assert rows[0].prompt == "hello"
|
|
assert rows[0].output == "hi"
|
|
assert rows[0].source == "openai-stored"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ingest_traces dispatch + file containment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_ingest_traces_dispatches_to_parser(tmp_path, monkeypatch):
|
|
"""Smoke: ingest_traces routes a JSONL log through the matching parser."""
|
|
from soup_cli.utils.ingest_sources import ingest_traces
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
log_file = tmp_path / "langfuse.jsonl"
|
|
payload = [
|
|
{"id": "t1", "input": "hi", "output": "there"},
|
|
{"id": "t2", "input": "x", "output": "y"},
|
|
]
|
|
log_file.write_text("\n".join(json.dumps(r) for r in payload), encoding="utf-8")
|
|
|
|
rows = list(ingest_traces(source="langfuse", path=str(log_file)))
|
|
assert len(rows) == 2
|
|
assert all(r.source == "langfuse" for r in rows)
|
|
|
|
|
|
def test_ingest_traces_rejects_outside_cwd(tmp_path, monkeypatch):
|
|
from soup_cli.utils.ingest_sources import ingest_traces
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
elsewhere = tmp_path.parent / "stray.jsonl"
|
|
elsewhere.write_text('{"input":"x","output":"y"}\n', encoding="utf-8")
|
|
try:
|
|
with pytest.raises(ValueError, match="outside"):
|
|
list(ingest_traces(source="langfuse", path=str(elsewhere)))
|
|
finally:
|
|
if elsewhere.exists():
|
|
elsewhere.unlink()
|
|
|
|
|
|
def test_ingest_traces_rejects_unknown_source(tmp_path, monkeypatch):
|
|
from soup_cli.utils.ingest_sources import ingest_traces
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
log_file = tmp_path / "x.jsonl"
|
|
log_file.write_text('{"input":"x","output":"y"}\n', encoding="utf-8")
|
|
with pytest.raises(ValueError, match="unknown"):
|
|
list(ingest_traces(source="bogus", path=str(log_file)))
|
|
|
|
|
|
def test_ingest_traces_missing_file(tmp_path, monkeypatch):
|
|
from soup_cli.utils.ingest_sources import ingest_traces
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(FileNotFoundError):
|
|
list(ingest_traces(source="langfuse", path=str(tmp_path / "missing.jsonl")))
|
|
|
|
|
|
def test_ingest_traces_rejects_null_byte_path(tmp_path, monkeypatch):
|
|
from soup_cli.utils.ingest_sources import ingest_traces
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError):
|
|
list(ingest_traces(source="langfuse", path="bad\x00path.jsonl"))
|
|
|
|
|
|
def test_ingest_traces_caps_lines(tmp_path, monkeypatch):
|
|
"""Should not hang on extremely large files; respect _MAX_INGEST_LINES."""
|
|
from soup_cli.utils import ingest_sources
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
log_file = tmp_path / "huge.jsonl"
|
|
# Write enough valid rows to trip cap (use very small cap via monkeypatch).
|
|
monkeypatch.setattr(ingest_sources, "_MAX_INGEST_LINES", 5)
|
|
rows = [{"input": "x", "output": "y"} for _ in range(20)]
|
|
log_file.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
|
|
|
|
result = list(ingest_sources.ingest_traces(source="langfuse", path=str(log_file)))
|
|
assert len(result) <= 5
|
|
|
|
|
|
def test_ingest_traces_skips_malformed_lines(tmp_path, monkeypatch):
|
|
from soup_cli.utils.ingest_sources import ingest_traces
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
log_file = tmp_path / "mixed.jsonl"
|
|
log_file.write_text(
|
|
'{"input":"x","output":"y"}\n'
|
|
"not-json\n"
|
|
'{"input":"a","output":"b"}\n',
|
|
encoding="utf-8",
|
|
)
|
|
rows = list(ingest_traces(source="langfuse", path=str(log_file)))
|
|
assert len(rows) == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Auth env resolution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_resolve_auth_env_each_source(monkeypatch):
|
|
from soup_cli.utils.ingest_sources import resolve_auth_env
|
|
|
|
monkeypatch.setenv("LANGFUSE_KEY", "lf-key")
|
|
monkeypatch.setenv("LANGSMITH_API_KEY", "ls-key")
|
|
monkeypatch.setenv("HELICONE_API_KEY", "h-key")
|
|
monkeypatch.setenv("OPENPIPE_API_KEY", "op-key")
|
|
monkeypatch.setenv("OPENAI_API_KEY", "oai-key")
|
|
monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=Bearer x")
|
|
|
|
assert resolve_auth_env("langfuse") == "lf-key"
|
|
assert resolve_auth_env("langsmith") == "ls-key"
|
|
assert resolve_auth_env("helicone") == "h-key"
|
|
assert resolve_auth_env("openpipe") == "op-key"
|
|
assert resolve_auth_env("openai-stored") == "oai-key"
|
|
assert resolve_auth_env("otel") is not None
|
|
|
|
|
|
def test_resolve_auth_env_missing(monkeypatch):
|
|
from soup_cli.utils.ingest_sources import resolve_auth_env
|
|
|
|
for env in [
|
|
"LANGFUSE_KEY",
|
|
"LANGSMITH_API_KEY",
|
|
"HELICONE_API_KEY",
|
|
"OPENPIPE_API_KEY",
|
|
"OPENAI_API_KEY",
|
|
"OTEL_EXPORTER_OTLP_HEADERS",
|
|
]:
|
|
monkeypatch.delenv(env, raising=False)
|
|
assert resolve_auth_env("langfuse") is None
|
|
|
|
|
|
def test_resolve_auth_env_unknown_raises():
|
|
from soup_cli.utils.ingest_sources import resolve_auth_env
|
|
|
|
with pytest.raises(ValueError):
|
|
resolve_auth_env("evilcorp")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI smoke
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_cli_ingest_help():
|
|
from soup_cli.cli import app
|
|
|
|
result = runner.invoke(app, ["ingest", "--help"])
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
# Sources + path must surface
|
|
assert "langfuse" in result.output.lower() or "source" in result.output.lower()
|
|
|
|
|
|
def test_cli_ingest_happy(tmp_path, monkeypatch):
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
log = tmp_path / "lf.jsonl"
|
|
log.write_text(
|
|
'{"id":"t1","input":"a","output":"b"}\n{"id":"t2","input":"c","output":"d"}\n',
|
|
encoding="utf-8",
|
|
)
|
|
out = tmp_path / "traces.jsonl"
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["ingest", "--source", "langfuse", "--logs", str(log), "--output", str(out)],
|
|
)
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert out.exists()
|
|
content = out.read_text(encoding="utf-8").strip().splitlines()
|
|
assert len(content) == 2
|
|
for line in content:
|
|
row = json.loads(line)
|
|
assert row["source"] == "langfuse"
|
|
assert "prompt" in row and "output" in row
|
|
|
|
|
|
def test_cli_ingest_unknown_source(tmp_path, monkeypatch):
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
log = tmp_path / "x.jsonl"
|
|
log.write_text("{}\n", encoding="utf-8")
|
|
result = runner.invoke(
|
|
app,
|
|
["ingest", "--source", "bogus", "--logs", str(log)],
|
|
)
|
|
assert result.exit_code != 0
|
|
assert "unknown" in result.output.lower() or "source" in result.output.lower()
|
|
|
|
|
|
def test_cli_ingest_outside_cwd_rejected(tmp_path, monkeypatch):
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
outside = tmp_path.parent / "stray_log.jsonl"
|
|
outside.write_text('{"input":"x","output":"y"}\n', encoding="utf-8")
|
|
try:
|
|
result = runner.invoke(
|
|
app,
|
|
["ingest", "--source", "langfuse", "--logs", str(outside)],
|
|
)
|
|
assert result.exit_code != 0
|
|
assert "outside" in result.output.lower()
|
|
finally:
|
|
if outside.exists():
|
|
outside.unlink()
|
|
|
|
|
|
def test_cli_ingest_pii_panel_prints(tmp_path, monkeypatch):
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
log = tmp_path / "lf.jsonl"
|
|
log.write_text('{"input":"x","output":"y"}\n', encoding="utf-8")
|
|
out = tmp_path / "out.jsonl"
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["ingest", "--source", "langfuse", "--logs", str(log), "--output", str(out)],
|
|
)
|
|
assert result.exit_code == 0
|
|
# PII reminder shown
|
|
assert "pii" in result.output.lower() or "sensitive" in result.output.lower()
|