219 lines
8.3 KiB
Python
219 lines
8.3 KiB
Python
"""Tests for full-fidelity trace events, dedup, and the CloudEvents exporter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from src.llm.backend import CompletionResult
|
|
from src.llm.capture import CapturedLLMCall, build_captured_call
|
|
from src.telemetry import trace_session
|
|
from src.telemetry.events.trace import LLMCallTracedEvent, TraceContentEvent
|
|
|
|
|
|
class TestLLMCallTracedEvent:
|
|
def test_metadata(self):
|
|
assert LLMCallTracedEvent.event_type() == "llm.call.traced"
|
|
assert LLMCallTracedEvent.category() == "trace"
|
|
# Ground-truth — never sampled (the system of record).
|
|
assert LLMCallTracedEvent.volume_class() == "ground_truth"
|
|
|
|
def test_resource_id_format(self):
|
|
event = LLMCallTracedEvent(
|
|
span_id="s1",
|
|
iteration=2,
|
|
attempt=1,
|
|
step_seq=3,
|
|
transport="anthropic",
|
|
model="m",
|
|
)
|
|
# {span_id}:{iteration}:{attempt}:{step_seq} — tool_call_seq dropped.
|
|
assert event.get_resource_id() == "s1:2:1:3"
|
|
|
|
def test_keeps_default_evt_id(self):
|
|
event = LLMCallTracedEvent(
|
|
span_id="s1",
|
|
iteration=1,
|
|
attempt=1,
|
|
step_seq=1,
|
|
transport="anthropic",
|
|
model="m",
|
|
)
|
|
event_id = event.generate_id()
|
|
assert event_id.startswith("evt_")
|
|
assert len(event_id) == 26
|
|
|
|
|
|
class TestTraceContentEvent:
|
|
def test_metadata(self):
|
|
assert TraceContentEvent.event_type() == "trace.content"
|
|
assert TraceContentEvent.category() == "trace"
|
|
assert TraceContentEvent.volume_class() == "ground_truth"
|
|
|
|
def test_resource_id_is_content_hash(self):
|
|
event = TraceContentEvent(content_hash="sha256:abc", role="user", content="hi")
|
|
assert event.get_resource_id() == "sha256:abc"
|
|
|
|
def test_generate_id_is_content_addressed_and_timestamp_free(self):
|
|
# Two instances with the same hash but DIFFERENT timestamps must collide
|
|
# on id, so cross-process/retry re-sends dedupe at the transport layer.
|
|
import datetime
|
|
|
|
a = TraceContentEvent(
|
|
content_hash="sha256:deadbeefdeadbeefdeadbeef",
|
|
role="user",
|
|
content="hi",
|
|
timestamp=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC),
|
|
)
|
|
b = TraceContentEvent(
|
|
content_hash="sha256:deadbeefdeadbeefdeadbeef",
|
|
role="user",
|
|
content="hi",
|
|
timestamp=datetime.datetime(2026, 6, 22, tzinfo=datetime.UTC),
|
|
)
|
|
assert a.generate_id() == b.generate_id()
|
|
assert a.generate_id().startswith("content_")
|
|
# Different content → different id.
|
|
c = TraceContentEvent(content_hash="sha256:other", role="user", content="hi")
|
|
assert c.generate_id() != a.generate_id()
|
|
|
|
|
|
class TestTraceSessionDedup:
|
|
def setup_method(self):
|
|
trace_session.reset()
|
|
|
|
def teardown_method(self):
|
|
trace_session.reset()
|
|
|
|
def test_first_emit_true_repeat_false(self):
|
|
assert trace_session.mark_emitted("run-1", "h1") is True
|
|
assert trace_session.mark_emitted("run-1", "h1") is False # already shipped
|
|
assert trace_session.mark_emitted("run-1", "h2") is True # new hash
|
|
|
|
def test_runs_are_independent(self):
|
|
assert trace_session.mark_emitted("run-1", "h1") is True
|
|
assert trace_session.mark_emitted("run-2", "h1") is True # different run
|
|
|
|
def test_lru_evicts_least_recently_used_run(self, monkeypatch: pytest.MonkeyPatch):
|
|
# Shrink the window so eviction is testable without _MAX_RUNS runs.
|
|
monkeypatch.setattr(trace_session, "_MAX_RUNS", 2)
|
|
trace_session.mark_emitted("run-1", "h1")
|
|
trace_session.mark_emitted("run-2", "h1")
|
|
# Touch run-1 so run-2 becomes the least-recently-used run.
|
|
trace_session.mark_emitted("run-1", "h2")
|
|
# A third run evicts the LRU run (run-2), keeping run-1.
|
|
trace_session.mark_emitted("run-3", "h1")
|
|
# run-1 is still tracked → its already-shipped hash stays deduped.
|
|
assert trace_session.mark_emitted("run-1", "h1") is False
|
|
# run-2 was evicted → its hash ships again as if a fresh run.
|
|
assert trace_session.mark_emitted("run-2", "h1") is True
|
|
|
|
|
|
class _FakeTraceEmitter:
|
|
"""Stand-in for the trace emitter that records emitted events."""
|
|
|
|
def __init__(self) -> None:
|
|
self.events: list[object] = []
|
|
|
|
def emit(self, event: object) -> None:
|
|
self.events.append(event)
|
|
|
|
|
|
@pytest.fixture
|
|
def trace_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[_FakeTraceEmitter]:
|
|
"""Enable payload tracing and route emit_trace at a fake emitter."""
|
|
from src.config import settings
|
|
from src.telemetry import emitter as emitter_mod
|
|
|
|
monkeypatch.setattr(settings.TELEMETRY, "TRACE_PAYLOADS_ENABLED", True)
|
|
fake = _FakeTraceEmitter()
|
|
monkeypatch.setattr(emitter_mod, "_trace_emitter", fake)
|
|
trace_session.reset()
|
|
yield fake
|
|
trace_session.reset()
|
|
|
|
|
|
def _captured(
|
|
messages: list[dict[str, Any]], *, content: str = "answer", run: str = "r1"
|
|
) -> CapturedLLMCall:
|
|
from src.llm.types import LLMTelemetryContext
|
|
|
|
return build_captured_call(
|
|
telemetry=LLMTelemetryContext(
|
|
workspace_name="ws",
|
|
call_purpose="dialectic.answer",
|
|
parent_category="dialectic",
|
|
run_id=run,
|
|
trace_id=run,
|
|
span_id=run,
|
|
iteration=1,
|
|
step_seq=1,
|
|
),
|
|
transport="anthropic",
|
|
provider_label=None,
|
|
model="claude-x",
|
|
messages=messages,
|
|
tools=None,
|
|
tool_choice=None,
|
|
result=CompletionResult(content=content, finish_reason="stop"),
|
|
attempt=1,
|
|
was_fallback=False,
|
|
was_stream=False,
|
|
finish_reason="stop",
|
|
)
|
|
|
|
|
|
class TestTraceExporter:
|
|
def test_refs_match_emitted_content(self, trace_on: _FakeTraceEmitter):
|
|
from src.telemetry.trace_exporter import TraceExporter
|
|
|
|
call = _captured([{"role": "user", "content": "q"}])
|
|
TraceExporter().export(call)
|
|
|
|
traced = [e for e in trace_on.events if isinstance(e, LLMCallTracedEvent)]
|
|
contents = [e for e in trace_on.events if isinstance(e, TraceContentEvent)]
|
|
assert len(traced) == 1
|
|
# input message + output content → two content events.
|
|
emitted_hashes = {c.content_hash for c in contents}
|
|
# Every input ref points at an emitted trace.content.
|
|
for ref in traced[0].input_message_refs:
|
|
assert ref in emitted_hashes
|
|
assert traced[0].output_content_ref in emitted_hashes
|
|
|
|
def test_dedup_across_iterations(self, trace_on: _FakeTraceEmitter):
|
|
from src.telemetry.trace_exporter import TraceExporter
|
|
|
|
exporter = TraceExporter()
|
|
shared = {"role": "user", "content": "system context"}
|
|
# Iteration 1: messages [shared]; iteration 2: [shared, follow-up].
|
|
exporter.export(_captured([shared]))
|
|
before = sum(isinstance(e, TraceContentEvent) for e in trace_on.events)
|
|
exporter.export(_captured([shared, {"role": "user", "content": "more"}]))
|
|
after = sum(isinstance(e, TraceContentEvent) for e in trace_on.events)
|
|
# `shared` already shipped this run → only the new message (+ output if
|
|
# not already seen) emit again; `shared` is NOT re-emitted.
|
|
shared_hash = None
|
|
for e in trace_on.events:
|
|
if isinstance(e, TraceContentEvent) and e.content == "system context":
|
|
shared_hash = e.content_hash
|
|
emitted_shared = [
|
|
e
|
|
for e in trace_on.events
|
|
if isinstance(e, TraceContentEvent) and e.content_hash == shared_hash
|
|
]
|
|
assert len(emitted_shared) == 1 # shipped once across both iterations
|
|
assert after > before # the new message did ship
|
|
|
|
def test_purpose_allowlist_filters(
|
|
self, trace_on: _FakeTraceEmitter, monkeypatch: pytest.MonkeyPatch
|
|
):
|
|
from src.config import settings
|
|
from src.telemetry.trace_exporter import TraceExporter
|
|
|
|
monkeypatch.setattr(settings.TELEMETRY, "TRACE_PURPOSES", ["summary.short"])
|
|
TraceExporter().export(_captured([{"role": "user", "content": "q"}]))
|
|
# call_purpose is dialectic.answer, not in the allowlist → nothing emits.
|
|
assert trace_on.events == []
|