From 451a8640964c2517c87d2d208d7a0a0a2ed79b62 Mon Sep 17 00:00:00 2001 From: Eugene Eisenstein Date: Thu, 3 Sep 2026 16:43:39 -0400 Subject: [PATCH] refactor(dialectic): drop message content from evidence Evidence carried the first 200 characters of every message the agent read. Remove it: messages now report identity and provenance only. Message content is caller-supplied and unbounded -- up to MAX_MESSAGE_SIZE, 25k characters -- and a single answer can touch a few hundred messages across its search, grep and date-range tools. Carrying even a slice of each invites callers to read messages in bulk out of evidence rather than fetching the ones they want, which is the opposite of what this is for: evidence is an audit and analytics surface, not a read API and not something to sit in a hot path. Conclusions keep their text, which is the asymmetry worth stating. A conclusion is written by the deriver, is short, and is the thing being audited; a message is raw input that already has endpoints of its own. Co-Authored-By: Claude Opus 5 (1M context) --- .../features/advanced/evidence.mdx | 7 +++-- src/schemas/api.py | 27 ++++++++++++------- src/utils/evidence.py | 13 ++++++--- tests/utils/test_evidence.py | 24 +++++++---------- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/docs/v3/documentation/features/advanced/evidence.mdx b/docs/v3/documentation/features/advanced/evidence.mdx index 5a7ca621..1fa1db8b 100644 --- a/docs/v3/documentation/features/advanced/evidence.mdx +++ b/docs/v3/documentation/features/advanced/evidence.mdx @@ -62,6 +62,8 @@ That has a consequence worth being clear about: **evidence over-reports**. A con The alternative — asking the model which sources it used — reads better but fails quietly. Weaker models and lower reasoning levels produce incomplete citations, invented IDs, or none at all, and you cannot tell a sparse citation list from a sparse answer. Collation is deterministic, costs no model tokens, and behaves identically at every reasoning level. Treat evidence as *what was available to the answer*, and audit within it. +Evidence is built for auditing and analytics — working out why an answer looks the way it does, or measuring what recall actually reaches the agent. It is not a read API, and it is not meant to sit in a hot path. + Two further limits: - `tool_calls` records **successful** invocations. A tool call that errored is retried or worked around by the agent and does not appear, so this is not a complete execution trace. @@ -97,7 +99,6 @@ Two further limits: "id": "m1N2o3P4q5R6s7T8u9V0w", "session_id": "session-xyz", "peer_id": "user-123", - "content_preview": "I just picked up some amazing dark roast from the place down the road...", "created_at": "2026-03-20T10:12:00Z" } ], @@ -112,7 +113,9 @@ Two further limits: **Conclusions** carry the ID, level, and text of each conclusion read. `source_ids` names the conclusions a derived one was reasoned from, so you can walk a chain back toward the explicit statements at its base; explicit conclusions have none, since they derive from messages rather than from other conclusions. `session_id` is null for a conclusion that was reasoned across sessions and so belongs to none. Timestamps are when a conclusion was derived, taken from its source messages where that is recorded. -**Messages** carry an ID and the first 200 characters of the content, not the whole message. Fetch the message itself if you need the rest. +**Messages** carry identity and provenance only — no content. Fetch a message by its `id` when you need the text. + +That asymmetry with conclusions is deliberate. A conclusion's text is written by the deriver, is short, and *is* the thing you are auditing, so it comes along. A message's content is whatever a caller sent, up to the 25,000-character ingest limit, and one answer can touch a few hundred messages — carrying it would let a single response drag megabytes behind it, and would turn evidence into a way to read messages in bulk. Evidence is for auditing and analytics, not a substitute for the message endpoints. `reasoning_trace_id` is a placeholder for stored reasoning traces and is currently always null. diff --git a/src/schemas/api.py b/src/schemas/api.py index de949071..2ca10681 100644 --- a/src/schemas/api.py +++ b/src/schemas/api.py @@ -838,10 +838,6 @@ class WorkspaceChatOptions(BaseModel): ) -# How much of a message's content an evidence reference carries. -EVIDENCE_MESSAGE_PREVIEW_CHARS = 200 - - class EvidenceObservation(BaseModel): """A conclusion the dialectic agent read while answering.""" @@ -869,14 +865,18 @@ class EvidenceObservation(BaseModel): class EvidenceMessageRef(BaseModel): - """A message the dialectic agent read while answering.""" + """A message the dialectic agent read while answering. + + Identity and provenance only -- no content. Message content is + caller-supplied and unbounded, so carrying it would let one answer drag + megabytes behind it, and would invite callers to read messages out of + evidence in bulk rather than asking for the ones they want. Fetch the + message by `id` when the text is needed. + """ id: str = Field(description="Message ID") session_id: str = Field(description="Session the message belongs to") peer_id: str = Field(description="Peer who sent the message") - content_preview: str = Field( - description="The beginning of the message content, truncated" - ) created_at: datetime.datetime = Field(description="When the message was sent") @@ -895,6 +895,12 @@ class Evidence(BaseModel): Collated from the agent's own reads rather than reported by the model, so it is deterministic but over-reports: it lists what the agent accessed, which is not necessarily what the answer relied on. + + Meant for auditing and analytics -- inspecting why an answer looks the way + it does, or measuring what recall actually reaches the agent. It is not a + read API: conclusions carry their text because that text is the thing being + audited and the deriver keeps it short, while messages carry identity alone + (see `EvidenceMessageRef`). """ conclusions: list[EvidenceObservation] = Field( @@ -903,7 +909,10 @@ class Evidence(BaseModel): ) messages: list[EvidenceMessageRef] = Field( default_factory=list, - description="Messages the agent read via its search and grep tools", + description=( + "Messages the agent read via its search and grep tools, by ID and" + " provenance only. Fetch a message to read its content." + ), ) tool_calls: list[EvidenceToolCall] = Field( default_factory=list, diff --git a/src/utils/evidence.py b/src/utils/evidence.py index e10df95f..cf1ea3f5 100644 --- a/src/utils/evidence.py +++ b/src/utils/evidence.py @@ -9,6 +9,15 @@ conclusion it does not end up leaning on. Because nothing is re-queried, evidence inherits the scoping of the reads that produced it: whatever the workspace, observer/observed pair and session allowlist permitted the agent to see is exactly what can appear here. + +What it is for shapes what it carries. Evidence is an audit and analytics +surface -- for asking why an answer looks the way it does, or measuring what +recall reaches the agent -- not a bulk read API. So conclusions carry their +text, which is short, model-written, and the thing being audited, while +messages carry identity alone: message content is caller-supplied and +unbounded, and including it would both inflate every response and invite +callers to read messages out of evidence instead of asking for the ones they +want. """ from collections.abc import Iterable, Sequence @@ -18,7 +27,6 @@ from typing import Any, cast from src import models from src.schemas.api import ( - EVIDENCE_MESSAGE_PREVIEW_CHARS, Evidence, EvidenceMessageRef, EvidenceObservation, @@ -148,9 +156,6 @@ class EvidenceAccumulator: id=message.public_id, session_id=message.session_name, peer_id=message.peer_name, - content_preview=message.content[ - :EVIDENCE_MESSAGE_PREVIEW_CHARS - ], created_at=message.created_at, ) for message in self.messages.values() diff --git a/tests/utils/test_evidence.py b/tests/utils/test_evidence.py index 6eb7c080..6cdbd9e3 100644 --- a/tests/utils/test_evidence.py +++ b/tests/utils/test_evidence.py @@ -11,7 +11,6 @@ from typing import Any import pytest from src import models -from src.schemas.api import EVIDENCE_MESSAGE_PREVIEW_CHARS from src.utils.evidence import EvidenceAccumulator NOW = datetime(2026, 1, 1, tzinfo=UTC) @@ -211,23 +210,20 @@ class TestMessageCollection: assert [m.id for m in accumulator.build().messages] == ["msg-1"] @pytest.mark.parametrize( - ("content", "expected_length"), - [ - ("short", len("short")), - ("x" * EVIDENCE_MESSAGE_PREVIEW_CHARS, EVIDENCE_MESSAGE_PREVIEW_CHARS), - ("x" * 5000, EVIDENCE_MESSAGE_PREVIEW_CHARS), - ], - ids=["short", "exactly-at-the-cap", "long"], + "content", ["short", "x" * 5000], ids=["short", "pasted-document"] ) - def test_caps_the_preview_without_truncating_short_messages( - self, content: str, expected_length: int - ): + def test_carries_no_message_content_at_any_length(self, content: str): + """Evidence names messages; it does not reproduce them. + + Message content is caller-supplied and unbounded, so a single answer + could otherwise drag megabytes behind it. Callers fetch by ID. + """ accumulator = EvidenceAccumulator() accumulator.add_messages([make_message("msg-1", content=content)]) - preview = accumulator.build().messages[0].content_preview - assert preview == content[:expected_length] - assert len(preview) == expected_length + (message,) = accumulator.build().messages + assert not hasattr(message, "content_preview") + assert content not in accumulator.build().model_dump_json() def test_orders_messages_chronologically(self): accumulator = EvidenceAccumulator()