fix: [critical] update prompt to

fix wording around deriving from multiple turns and remove bars

feat: add DEDUPLICATE config flag, when true, uses cosine+token similarly in tandem to dedup

.env.template

@@ -83,6 +83,7 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
 # DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5
 # DERIVER_PROVIDER=google
 # DERIVER_MODEL=gemini-2.0-flash-lite
+# DERIVER_DEDUPLICATE=true
 # DERIVER_MAX_OUTPUT_TOKENS=2500
 # only applied when using Anthropic as provider
 # DERIVER_THINKING_BUDGET_TOKENS=1024

config.toml.example

@@ -64,6 +64,7 @@ POLLING_SLEEP_INTERVAL_SECONDS = 1.0
 STALE_SESSION_TIMEOUT_MINUTES = 5
 PROVIDER = "google"
 MODEL = "gemini-2.0-flash-lite"
+DEDUPLICATE = true
 MAX_OUTPUT_TOKENS = 2500
 THINKING_BUDGET_TOKENS = 1024 # only applied when using Anthropic
 

src/config.py

@@ -199,6 +199,9 @@ class DeriverSettings(HonchoSettings):
     PROVIDER: SupportedProviders = "google"
     MODEL: str = "gemini-2.5-flash-lite"
 
+    # Whether to deduplicate documents when creating them
+    DEDUPLICATE: bool = True
+
     MAX_OUTPUT_TOKENS: Annotated[int, Field(default=10_000, gt=0, le=100_000)] = 10_000
     # Thinking budget tokens are only applied when using Anthropic as provider
     THINKING_BUDGET_TOKENS: Annotated[int, Field(default=1024, gt=0, le=5000)] = 1024

src/crud/document.py

@@ -103,9 +103,10 @@ async def create_documents(
     *,
     observer: str,
     observed: str,
+    deduplicate: bool = False,
 ) -> int:
     """
-    Create multiple documents with NO duplicate detection.
+    Create multiple documents with optional duplicate detection.
 
     Args:
         db: Database session
@@ -120,6 +121,16 @@ async def create_documents(
     honcho_documents: list[models.Document] = []
     for doc in documents:
         try:
+            # for each document, if deduplicate is True, perform a process
+            # that checks against existing documents and either rejects this document
+            # as a duplicate OR deletes an existing document that is a duplicate.
+            if deduplicate:
+                is_duplicate = await is_rejected_duplicate(
+                    db, doc, workspace_name, observer=observer, observed=observed
+                )
+                if is_duplicate:
+                    continue
+
             metadata_dict = doc.metadata.model_dump(exclude_none=True)
             honcho_documents.append(
                 models.Document(
@@ -147,3 +158,68 @@ async def create_documents(
         ) from e
 
     return len(honcho_documents)
+
+
+async def is_rejected_duplicate(
+    db: AsyncSession,
+    doc: schemas.DocumentCreate,
+    workspace_name: str,
+    *,
+    observer: str,
+    observed: str,
+) -> bool:
+    """
+    Check if a document is a duplicate of an existing document.
+
+    Uses: 1) Cosine similarity (>=0.95), 2) Token diff for retention.
+
+    Returns True if both:
+    - the document is deemed a duplicate of an existing document
+    - the existing document is deemed a superior duplicate
+
+    If the document is not a duplicate, returns False.
+
+    If the document is a duplicate AND the new document is superior,
+    deletes the existing document and returns False.
+    """
+    # Step 1: Find potential duplicates using cosine similarity
+    similar_docs = await query_documents(
+        db=db,
+        workspace_name=workspace_name,
+        query=doc.content,
+        observer=observer,
+        observed=observed,
+        max_distance=0.05,
+        top_k=1,
+        embedding=doc.embedding,
+    )
+
+    if not similar_docs:
+        return False
+
+    existing_doc = similar_docs[0]
+
+    # Step 2: Determine which has more information using token set difference
+    tokens_new = set(embedding_client.encoding.encode(doc.content))
+    tokens_existing = set(embedding_client.encoding.encode(existing_doc.content))
+
+    unique_new = len(tokens_new - tokens_existing)
+    unique_existing = len(tokens_existing - tokens_new)
+
+    score_new = len(tokens_new) + (unique_new * 10)
+    score_existing = len(tokens_existing) + (unique_existing * 10)
+
+    # If new document has more or equal information, keep it and delete existing
+    if score_new >= score_existing:
+        logger.warning(
+            f"[DUPLICATE DETECTION] Deleting existing in favor of new. new='{doc.content}', existing='{existing_doc.content}'."
+        )
+        await db.delete(existing_doc)
+        await db.flush()  # Flush to make deletion visible in this transaction
+        return False  # Don't reject the new document
+
+    # Existing document has more information, reject the new one
+    logger.warning(
+        f"[DUPLICATE DETECTION] Rejecting new in favor of existing. new='{doc.content}', existing='{existing_doc.content}'."
+    )
+    return True

src/crud/representation.py

@@ -160,13 +160,14 @@ class RepresentationManager:
                 )
             )
 
-        # Use bulk creation with NO duplicate detection
+        # Use bulk creation with optional duplicate detection
         new_documents = await crud.create_documents(
             db,
             documents_to_create,
             self.workspace_name,
             observer=self.observer,
             observed=self.observed,
+            deduplicate=settings.DERIVER.DEDUPLICATE,
         )
 
         try:

src/deriver/prompts.py

@@ -65,11 +65,9 @@ Current understanding of {peer_id}:
 You are an agent who critically analyzes messages from {peer_id} through rigorous logical reasoning to produce only conclusions about them that are CERTAIN.
 
 TARGET USER TO ANALYZE
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 You are analyzing: {peer_id}
 
 The conversation may include messages from multiple participants, but you MUST focus ONLY on deriving conclusions about {peer_id}. Only use other participants' messages as context for understanding {peer_id}.
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
 IMPORTANT NAMING RULES
 • When you write a conclusion about {peer_id}, always start the sentence with their name (e.g. "Anthony is 25 years old").
@@ -87,7 +85,7 @@ Here are strict definitions for the reasoning modes you are to employ:
         - Current date and time (which is: {message_created_at})
         - Timestamps from conversation history
     - Follow strict literal necessity--if stated directly in message, extract a conclusion
-    - Latest message MUST be a premise, previous messages and timestamps may be used to contextualize
+    - New turn MUST be a premise, previous messages and timestamps may be used to contextualize
     - Transforms a single message (premise) into ONE OR MULTIPLE conclusions
     - Derive EVERYTHING that can be explicitly concluded
     - Make sure EVERY conclusion is sufficiently contextualized, i.e. ensure each conclusion contains enough specific information about subjects and objects to make it self-contained and useful (e.g. instead of "Ann is nervous about the interview", use "Ann is nervous about the job interview at the pharmacy")
@@ -128,7 +126,7 @@ Based on our definitions and examples, here's a summary of the logical reasoning
 - Explicit/Deductive/Temporal/General → Further Deductive: Can use certain conclusions and known facts to deduce additional certain conclusions
 - Probabilistic Deductive ↛ Further Deductive: If a deductive conclusion includes probabilistic qualifiers (likely, potentially, typically, might, etc) it may NOT be used as a premise for further deductions
 
-**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to the latest message, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions.
+**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to all new turns, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions. You must extract observations from all new turns.
 
 
 {peer_card_section}