feat: add source_indices to deriver output for evidence entailment
Each explicit observation now carries source_indices — 0-based indices into the <messages> block pointing to the messages that directly support the conclusion. Messages in the prompt are prefixed with [N] indices. This enables the action-based judge's evidence entailment criterion to slice the conversation to just the supporting messages instead of using the full conversation as evidence context (degraded mode). No text duplication — the LLM outputs indices (cheap, low-error), and the judge looks up the real message text from the trace input.
This commit is contained in:
parent
855cd04eab
commit
11b3603ab1
|
|
@ -181,6 +181,7 @@ class RepresentationManager:
|
|||
message_ids=message_ids,
|
||||
premises=obs_premises,
|
||||
message_created_at=format_datetime_utc(message_created_at),
|
||||
source_indices=getattr(obs, "source_indices", []),
|
||||
)
|
||||
|
||||
documents_to_create.append(
|
||||
|
|
|
|||
|
|
@ -103,10 +103,10 @@ async def process_representation_tasks_batch(
|
|||
"id",
|
||||
)
|
||||
|
||||
# Format messages with timestamps
|
||||
# Format messages with timestamps and 0-based indices for source citation
|
||||
formatted_messages = "\n".join(
|
||||
format_new_turn_with_timestamp(msg.content, msg.created_at, msg.peer_name)
|
||||
for msg in messages
|
||||
f"[{i}] {format_new_turn_with_timestamp(msg.content, msg.created_at, msg.peer_name)}"
|
||||
for i, msg in enumerate(messages)
|
||||
)
|
||||
|
||||
# Track token usage - count only tokens from messages being processed
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ RULES:
|
|||
- Observations should make sense on their own. Each observation will be used in the future to better understand {peer_id}.
|
||||
- Extract ALL observations from {peer_id} messages, using others as context.
|
||||
- Contextualize each observation sufficiently (e.g. "Ann is nervous about the job interview at the pharmacy" not just "Ann is nervous")
|
||||
- Each message in the <messages> block is prefixed with a 0-based index like [0], [1], [2]. For each observation, set source_indices to the indices of the messages that directly support it. Include the message containing any context needed to interpret the conclusion (e.g., the question being answered by "the first one"). Only include messages that directly support the observation — not the entire conversation.
|
||||
|
||||
EXAMPLES (using `{peer_id}` as the target peer id):
|
||||
- EXPLICIT: "I just had my 25th birthday last Saturday" → "{peer_id} is 25 years old", "{peer_id}'s birthday is June 21st"
|
||||
|
|
|
|||
|
|
@ -56,6 +56,11 @@ class DocumentMetadata(BaseModel):
|
|||
default=None,
|
||||
description="Confidence level (high, medium, low) -- only applicable for inductive documents",
|
||||
)
|
||||
source_indices: list[int] = Field(
|
||||
default_factory=list,
|
||||
description="0-based indices into the deriver batch's message list "
|
||||
"indicating which messages directly support this observation",
|
||||
)
|
||||
|
||||
|
||||
class DocumentCreate(DocumentBase):
|
||||
|
|
|
|||
|
|
@ -83,10 +83,25 @@ class ObservationMetadata(BaseModel):
|
|||
created_at: datetime
|
||||
message_ids: list[int]
|
||||
session_name: str | None = None
|
||||
source_indices: list[int] = Field(
|
||||
default_factory=list,
|
||||
description="0-based indices into the deriver batch's message list "
|
||||
"indicating which messages directly support this observation",
|
||||
)
|
||||
|
||||
|
||||
class ExplicitObservationBase(BaseModel):
|
||||
content: str = Field(description="The explicit observation")
|
||||
source_indices: list[int] = Field(
|
||||
default_factory=list,
|
||||
description=(
|
||||
"0-based indices of the messages in the <messages> block that "
|
||||
"directly support this observation. Include the message "
|
||||
"containing any context needed to interpret the conclusion "
|
||||
'(e.g., the question being answered by "the first one"). '
|
||||
"Only include messages that directly support the observation."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class DeductiveObservationBase(BaseModel):
|
||||
|
|
@ -622,6 +637,7 @@ class Representation(BaseModel):
|
|||
doc.internal_metadata.get("message_ids", [])
|
||||
),
|
||||
session_name=doc.session_name,
|
||||
source_indices=doc.internal_metadata.get("source_indices", []),
|
||||
)
|
||||
for doc in documents
|
||||
if doc.level == "explicit"
|
||||
|
|
@ -696,6 +712,7 @@ class Representation(BaseModel):
|
|||
explicit=[
|
||||
ExplicitObservation(
|
||||
content=e.content,
|
||||
source_indices=e.source_indices,
|
||||
created_at=created_at,
|
||||
message_ids=message_ids,
|
||||
session_name=session_name,
|
||||
|
|
|
|||
Loading…
Reference in New Issue