diff --git a/.env.template b/.env.template index a91ede16..9e90ade3 100644 --- a/.env.template +++ b/.env.template @@ -83,6 +83,7 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 # DERIVER_PROVIDER=google # DERIVER_MODEL=gemini-2.0-flash-lite +# DERIVER_DEDUPLICATE=true # DERIVER_MAX_OUTPUT_TOKENS=2500 # only applied when using Anthropic as provider # DERIVER_THINKING_BUDGET_TOKENS=1024 diff --git a/config.toml.example b/config.toml.example index 50f618ba..bc514eb6 100644 --- a/config.toml.example +++ b/config.toml.example @@ -64,6 +64,7 @@ POLLING_SLEEP_INTERVAL_SECONDS = 1.0 STALE_SESSION_TIMEOUT_MINUTES = 5 PROVIDER = "google" MODEL = "gemini-2.0-flash-lite" +DEDUPLICATE = true MAX_OUTPUT_TOKENS = 2500 THINKING_BUDGET_TOKENS = 1024 # only applied when using Anthropic diff --git a/src/config.py b/src/config.py index 6458717e..c9670508 100644 --- a/src/config.py +++ b/src/config.py @@ -199,6 +199,9 @@ class DeriverSettings(HonchoSettings): PROVIDER: SupportedProviders = "google" MODEL: str = "gemini-2.5-flash-lite" + # Whether to deduplicate documents when creating them + DEDUPLICATE: bool = True + MAX_OUTPUT_TOKENS: Annotated[int, Field(default=10_000, gt=0, le=100_000)] = 10_000 # Thinking budget tokens are only applied when using Anthropic as provider THINKING_BUDGET_TOKENS: Annotated[int, Field(default=1024, gt=0, le=5000)] = 1024 diff --git a/src/crud/document.py b/src/crud/document.py index 127b5d7b..272f3ab3 100644 --- a/src/crud/document.py +++ b/src/crud/document.py @@ -103,9 +103,10 @@ async def create_documents( *, observer: str, observed: str, + deduplicate: bool = False, ) -> int: """ - Create multiple documents with NO duplicate detection. + Create multiple documents with optional duplicate detection. Args: db: Database session @@ -120,6 +121,16 @@ async def create_documents( honcho_documents: list[models.Document] = [] for doc in documents: try: + # for each document, if deduplicate is True, perform a process + # that checks against existing documents and either rejects this document + # as a duplicate OR deletes an existing document that is a duplicate. + if deduplicate: + is_duplicate = await is_rejected_duplicate( + db, doc, workspace_name, observer=observer, observed=observed + ) + if is_duplicate: + continue + metadata_dict = doc.metadata.model_dump(exclude_none=True) honcho_documents.append( models.Document( @@ -147,3 +158,68 @@ async def create_documents( ) from e return len(honcho_documents) + + +async def is_rejected_duplicate( + db: AsyncSession, + doc: schemas.DocumentCreate, + workspace_name: str, + *, + observer: str, + observed: str, +) -> bool: + """ + Check if a document is a duplicate of an existing document. + + Uses: 1) Cosine similarity (>=0.95), 2) Token diff for retention. + + Returns True if both: + - the document is deemed a duplicate of an existing document + - the existing document is deemed a superior duplicate + + If the document is not a duplicate, returns False. + + If the document is a duplicate AND the new document is superior, + deletes the existing document and returns False. + """ + # Step 1: Find potential duplicates using cosine similarity + similar_docs = await query_documents( + db=db, + workspace_name=workspace_name, + query=doc.content, + observer=observer, + observed=observed, + max_distance=0.05, + top_k=1, + embedding=doc.embedding, + ) + + if not similar_docs: + return False + + existing_doc = similar_docs[0] + + # Step 2: Determine which has more information using token set difference + tokens_new = set(embedding_client.encoding.encode(doc.content)) + tokens_existing = set(embedding_client.encoding.encode(existing_doc.content)) + + unique_new = len(tokens_new - tokens_existing) + unique_existing = len(tokens_existing - tokens_new) + + score_new = len(tokens_new) + (unique_new * 10) + score_existing = len(tokens_existing) + (unique_existing * 10) + + # If new document has more or equal information, keep it and delete existing + if score_new >= score_existing: + logger.warning( + f"[DUPLICATE DETECTION] Deleting existing in favor of new. new='{doc.content}', existing='{existing_doc.content}'." + ) + await db.delete(existing_doc) + await db.flush() # Flush to make deletion visible in this transaction + return False # Don't reject the new document + + # Existing document has more information, reject the new one + logger.warning( + f"[DUPLICATE DETECTION] Rejecting new in favor of existing. new='{doc.content}', existing='{existing_doc.content}'." + ) + return True diff --git a/src/crud/representation.py b/src/crud/representation.py index 93fc6744..790912f1 100644 --- a/src/crud/representation.py +++ b/src/crud/representation.py @@ -160,13 +160,14 @@ class RepresentationManager: ) ) - # Use bulk creation with NO duplicate detection + # Use bulk creation with optional duplicate detection new_documents = await crud.create_documents( db, documents_to_create, self.workspace_name, observer=self.observer, observed=self.observed, + deduplicate=settings.DERIVER.DEDUPLICATE, ) try: diff --git a/src/deriver/prompts.py b/src/deriver/prompts.py index 2ef199eb..4a414f88 100644 --- a/src/deriver/prompts.py +++ b/src/deriver/prompts.py @@ -65,11 +65,9 @@ Current understanding of {peer_id}: You are an agent who critically analyzes messages from {peer_id} through rigorous logical reasoning to produce only conclusions about them that are CERTAIN. TARGET USER TO ANALYZE -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ You are analyzing: {peer_id} The conversation may include messages from multiple participants, but you MUST focus ONLY on deriving conclusions about {peer_id}. Only use other participants' messages as context for understanding {peer_id}. -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ IMPORTANT NAMING RULES • When you write a conclusion about {peer_id}, always start the sentence with their name (e.g. "Anthony is 25 years old"). @@ -87,7 +85,7 @@ Here are strict definitions for the reasoning modes you are to employ: - Current date and time (which is: {message_created_at}) - Timestamps from conversation history - Follow strict literal necessity--if stated directly in message, extract a conclusion - - Latest message MUST be a premise, previous messages and timestamps may be used to contextualize + - New turn MUST be a premise, previous messages and timestamps may be used to contextualize - Transforms a single message (premise) into ONE OR MULTIPLE conclusions - Derive EVERYTHING that can be explicitly concluded - Make sure EVERY conclusion is sufficiently contextualized, i.e. ensure each conclusion contains enough specific information about subjects and objects to make it self-contained and useful (e.g. instead of "Ann is nervous about the interview", use "Ann is nervous about the job interview at the pharmacy") @@ -128,7 +126,7 @@ Based on our definitions and examples, here's a summary of the logical reasoning - Explicit/Deductive/Temporal/General → Further Deductive: Can use certain conclusions and known facts to deduce additional certain conclusions - Probabilistic Deductive ↛ Further Deductive: If a deductive conclusion includes probabilistic qualifiers (likely, potentially, typically, might, etc) it may NOT be used as a premise for further deductions -**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to the latest message, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions. +**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to all new turns, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions. You must extract observations from all new turns. {peer_card_section}