feat: add source_indices to deriver output for evidence entailment

Each explicit observation now carries source_indices — 0-based indices
into the <messages> block pointing to the messages that directly support
the conclusion. Messages in the prompt are prefixed with [N] indices.

This enables the action-based judge's evidence entailment criterion to
slice the conversation to just the supporting messages instead of using
the full conversation as evidence context (degraded mode).

No text duplication — the LLM outputs indices (cheap, low-error), and
the judge looks up the real message text from the trace input.
This commit is contained in:
Ubuntu 2026-07-22 15:01:55 +00:00 committed by Matt Landers
parent 855cd04eab
commit 11b3603ab1
5 changed files with 27 additions and 3 deletions

View File

@ -181,6 +181,7 @@ class RepresentationManager:
message_ids=message_ids,
premises=obs_premises,
message_created_at=format_datetime_utc(message_created_at),
source_indices=getattr(obs, "source_indices", []),
)
documents_to_create.append(

View File

@ -103,10 +103,10 @@ async def process_representation_tasks_batch(
"id",
)
# Format messages with timestamps
# Format messages with timestamps and 0-based indices for source citation
formatted_messages = "\n".join(
format_new_turn_with_timestamp(msg.content, msg.created_at, msg.peer_name)
for msg in messages
f"[{i}] {format_new_turn_with_timestamp(msg.content, msg.created_at, msg.peer_name)}"
for i, msg in enumerate(messages)
)
# Track token usage - count only tokens from messages being processed

View File

@ -81,6 +81,7 @@ RULES:
- Observations should make sense on their own. Each observation will be used in the future to better understand {peer_id}.
- Extract ALL observations from {peer_id} messages, using others as context.
- Contextualize each observation sufficiently (e.g. "Ann is nervous about the job interview at the pharmacy" not just "Ann is nervous")
- Each message in the <messages> block is prefixed with a 0-based index like [0], [1], [2]. For each observation, set source_indices to the indices of the messages that directly support it. Include the message containing any context needed to interpret the conclusion (e.g., the question being answered by "the first one"). Only include messages that directly support the observation not the entire conversation.
EXAMPLES (using `{peer_id}` as the target peer id):
- EXPLICIT: "I just had my 25th birthday last Saturday" "{peer_id} is 25 years old", "{peer_id}'s birthday is June 21st"

View File

@ -56,6 +56,11 @@ class DocumentMetadata(BaseModel):
default=None,
description="Confidence level (high, medium, low) -- only applicable for inductive documents",
)
source_indices: list[int] = Field(
default_factory=list,
description="0-based indices into the deriver batch's message list "
"indicating which messages directly support this observation",
)
class DocumentCreate(DocumentBase):

View File

@ -83,10 +83,25 @@ class ObservationMetadata(BaseModel):
created_at: datetime
message_ids: list[int]
session_name: str | None = None
source_indices: list[int] = Field(
default_factory=list,
description="0-based indices into the deriver batch's message list "
"indicating which messages directly support this observation",
)
class ExplicitObservationBase(BaseModel):
content: str = Field(description="The explicit observation")
source_indices: list[int] = Field(
default_factory=list,
description=(
"0-based indices of the messages in the <messages> block that "
"directly support this observation. Include the message "
"containing any context needed to interpret the conclusion "
'(e.g., the question being answered by "the first one"). '
"Only include messages that directly support the observation."
),
)
class DeductiveObservationBase(BaseModel):
@ -622,6 +637,7 @@ class Representation(BaseModel):
doc.internal_metadata.get("message_ids", [])
),
session_name=doc.session_name,
source_indices=doc.internal_metadata.get("source_indices", []),
)
for doc in documents
if doc.level == "explicit"
@ -696,6 +712,7 @@ class Representation(BaseModel):
explicit=[
ExplicitObservation(
content=e.content,
source_indices=e.source_indices,
created_at=created_at,
message_ids=message_ids,
session_name=session_name,