From 11b3603ab172cd53057e57749e53358a4b91ba90 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 22 Jul 2026 15:01:55 +0000 Subject: [PATCH] feat: add source_indices to deriver output for evidence entailment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each explicit observation now carries source_indices — 0-based indices into the block pointing to the messages that directly support the conclusion. Messages in the prompt are prefixed with [N] indices. This enables the action-based judge's evidence entailment criterion to slice the conversation to just the supporting messages instead of using the full conversation as evidence context (degraded mode). No text duplication — the LLM outputs indices (cheap, low-error), and the judge looks up the real message text from the trace input. --- src/crud/representation.py | 1 + src/deriver/deriver.py | 6 +++--- src/deriver/prompts.py | 1 + src/schemas/internal.py | 5 +++++ src/utils/representation.py | 17 +++++++++++++++++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/crud/representation.py b/src/crud/representation.py index d4b86ffb..ffe18085 100644 --- a/src/crud/representation.py +++ b/src/crud/representation.py @@ -181,6 +181,7 @@ class RepresentationManager: message_ids=message_ids, premises=obs_premises, message_created_at=format_datetime_utc(message_created_at), + source_indices=getattr(obs, "source_indices", []), ) documents_to_create.append( diff --git a/src/deriver/deriver.py b/src/deriver/deriver.py index 2ad7d14f..05e89d1e 100644 --- a/src/deriver/deriver.py +++ b/src/deriver/deriver.py @@ -103,10 +103,10 @@ async def process_representation_tasks_batch( "id", ) - # Format messages with timestamps + # Format messages with timestamps and 0-based indices for source citation formatted_messages = "\n".join( - format_new_turn_with_timestamp(msg.content, msg.created_at, msg.peer_name) - for msg in messages + f"[{i}] {format_new_turn_with_timestamp(msg.content, msg.created_at, msg.peer_name)}" + for i, msg in enumerate(messages) ) # Track token usage - count only tokens from messages being processed diff --git a/src/deriver/prompts.py b/src/deriver/prompts.py index bf8d61fd..aae6356f 100644 --- a/src/deriver/prompts.py +++ b/src/deriver/prompts.py @@ -81,6 +81,7 @@ RULES: - Observations should make sense on their own. Each observation will be used in the future to better understand {peer_id}. - Extract ALL observations from {peer_id} messages, using others as context. - Contextualize each observation sufficiently (e.g. "Ann is nervous about the job interview at the pharmacy" not just "Ann is nervous") +- Each message in the block is prefixed with a 0-based index like [0], [1], [2]. For each observation, set source_indices to the indices of the messages that directly support it. Include the message containing any context needed to interpret the conclusion (e.g., the question being answered by "the first one"). Only include messages that directly support the observation — not the entire conversation. EXAMPLES (using `{peer_id}` as the target peer id): - EXPLICIT: "I just had my 25th birthday last Saturday" → "{peer_id} is 25 years old", "{peer_id}'s birthday is June 21st" diff --git a/src/schemas/internal.py b/src/schemas/internal.py index e014431f..16890b24 100644 --- a/src/schemas/internal.py +++ b/src/schemas/internal.py @@ -56,6 +56,11 @@ class DocumentMetadata(BaseModel): default=None, description="Confidence level (high, medium, low) -- only applicable for inductive documents", ) + source_indices: list[int] = Field( + default_factory=list, + description="0-based indices into the deriver batch's message list " + "indicating which messages directly support this observation", + ) class DocumentCreate(DocumentBase): diff --git a/src/utils/representation.py b/src/utils/representation.py index 01e4b70a..97ad27c2 100644 --- a/src/utils/representation.py +++ b/src/utils/representation.py @@ -83,10 +83,25 @@ class ObservationMetadata(BaseModel): created_at: datetime message_ids: list[int] session_name: str | None = None + source_indices: list[int] = Field( + default_factory=list, + description="0-based indices into the deriver batch's message list " + "indicating which messages directly support this observation", + ) class ExplicitObservationBase(BaseModel): content: str = Field(description="The explicit observation") + source_indices: list[int] = Field( + default_factory=list, + description=( + "0-based indices of the messages in the block that " + "directly support this observation. Include the message " + "containing any context needed to interpret the conclusion " + '(e.g., the question being answered by "the first one"). ' + "Only include messages that directly support the observation." + ), + ) class DeductiveObservationBase(BaseModel): @@ -622,6 +637,7 @@ class Representation(BaseModel): doc.internal_metadata.get("message_ids", []) ), session_name=doc.session_name, + source_indices=doc.internal_metadata.get("source_indices", []), ) for doc in documents if doc.level == "explicit" @@ -696,6 +712,7 @@ class Representation(BaseModel): explicit=[ ExplicitObservation( content=e.content, + source_indices=e.source_indices, created_at=created_at, message_ids=message_ids, session_name=session_name,