Turbopuffer and LanceDB Integration (#287)

* feat: init turbopuffer and lanceDB

* fix: remove destructive embedding migration

* fix: bug fixes

* fix: LanceDB

* fix: turbopuffer

* fix: search and add create_observations

* fix: use Async clients

* fix: search; protect agaainst failed vector create/delete

* fix: coderabbit comments

* fix: set up compose vector store and reconciliation loop

* feat: sync docs without embeddings

* fix: reduce batch size; comments; types; add indexes for reconciliation

* fix: add message embedding resilience

* fix: clean-up and migration test

* fix: cleanup 2

* fix: centralize retry logic; bump reconciliation batch; use tracked db; fix soft-delete race condition

* fix: skip double query when pgvector is primary

* fix: down migration

* fix: remove hard-delete from critical path and make PgVectorStore deletions a no-op

* fix: use soft-delete pattern for duplicate detection

* fix: steps toward deprecating MessageEmbedding table

* fix: remove composite and pgvector store -> make more specific

* fix: migration order

* fix: shorten reconciliation cycle + fix 'IN' equality check

* fix: coderabbit comments

* fix: add test for migration 7c0d9a4e3b1f

* feat: refactor to use ReconcilerScheduler

* fix: CR / opus comments

* fix: work unit key and reserve system workspace

* fix: make workspace_name nullable

* fix: clean up sync vectors

* fix: delete syntax

* fix: hash namespace

* External Vector Store Nits (#332)

* fix: Migration naming and long held connection

* chore: Comment for potential debt

* chore: update typescript core package

---------

Co-authored-by: Vineeth Voruganti <13438633+VVoruganti@users.noreply.github.com>

.env.template

@@ -228,3 +228,29 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
 # CACHE_NAMESPACE="honcho"  # Inherits from NAMESPACE if not set
 # CACHE_DEFAULT_TTL_SECONDS=300
 # CACHE_DEFAULT_LOCK_TTL_SECONDS=5
+
+# =============================================================================
+# Vector Store Settings
+# =============================================================================
+# Vector store type: "pgvector", "turbopuffer", or "lancedb"
+VECTOR_STORE_TYPE=pgvector
+
+# Migration flag: set to true when migration from pgvector is complete
+VECTOR_STORE_MIGRATED=false
+
+# Global namespace prefix for all vector namespaces
+# Namespaces follow the pattern:
+# - Documents: {NAMESPACE}.{workspace}.{observer}.{observed}
+# - Messages: {NAMESPACE}.{workspace}.messages
+# VECTOR_STORE_NAMESPACE=honcho  # Inherits from NAMESPACE if not set
+
+# Embedding dimensions (default: 1536 for OpenAI text-embedding-3-small)
+# VECTOR_STORE_DIMENSIONS=1536
+
+# Turbopuffer-specific settings (required if TYPE is "turbopuffer")
+# VECTOR_STORE_TURBOPUFFER_API_KEY=your-turbopuffer-api-key
+# VECTOR_STORE_TURBOPUFFER_REGION=gcp-us-east4
+
+# LanceDB-specific settings (local embedded mode)
+# VECTOR_STORE_LANCEDB_PATH=./lancedb_data
+#VECTOR_STORE_RECONCILIATION_INTERVAL_SECONDS

.gitignore

@@ -189,3 +189,4 @@ CRUSH.md
 
 metrics.jsonl
 AGENTS.md
+lancedb_data/

config.toml.example

@@ -172,3 +172,16 @@ URL = "redis://localhost:6379/0?suppress=false"
 # NAMESPACE = "honcho"  # Inherits from app.NAMESPACE if not set
 DEFAULT_TTL_SECONDS = 300
 DEFAULT_LOCK_TTL_SECONDS = 5
+
+# Vector store settings
+[vector_store]
+# Vector store type: "pgvector", "turbopuffer", or "lancedb"
+TYPE = "pgvector"
+# Migration flag: set to true when migration from pgvector is complete
+MIGRATED = false
+NAMESPACE = "honcho"
+DIMENSIONS = 1536
+# TURBOPUFFER_API_KEY = "your-turbopuffer-api-key"
+# TURBOPUFFER_REGION = "us-east-1"
+LANCEDB_PATH = "./lancedb_data"
+RECONCILIATION_INTERVAL_SECONDS = 300

migrations/versions/119a52b73c60_support_external_embeddings.py

@@ -0,0 +1,359 @@
+"""make embeddings nullable, add soft delete, add vector sync state
+
+This migration:
+1. Makes embedding columns nullable in both message_embeddings and documents tables
+   since embeddings are now stored in external vector stores instead of PostgreSQL.
+2. Adds deleted_at column to documents table for soft delete support, enabling
+   hybrid sync/soft delete pattern for vector store consistency.
+3. Adds sync_state, last_sync_at, and sync_attempts columns to documents and
+   message_embeddings tables for tracking vector store synchronization status.
+4. Adds partial unique index on queue table for reconciler task deduplication,
+   ensuring only one pending reconciler task exists per work_unit_key.
+5. Makes workspace_name nullable on queue table for system-level tasks (e.g., reconciler)
+   that don't belong to any specific workspace.
+
+Revision ID: 119a52b73c60
+Revises: 7c0d9a4e3b1f
+Create Date: 2025-11-24 12:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from pgvector.sqlalchemy import Vector
+
+from migrations.utils import column_exists, constraint_exists, get_schema, index_exists
+
+# revision identifiers, used by Alembic.
+revision: str = "119a52b73c60"
+down_revision: str | None = "7c0d9a4e3b1f"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+schema = get_schema()
+
+
+def upgrade() -> None:
+    """Make embeddings nullable, add deleted_at, add sync state."""
+    inspector = sa.inspect(op.get_bind())
+
+    op.alter_column(
+        "message_embeddings",
+        "embedding",
+        existing_type=Vector(1536),
+        nullable=True,
+        schema=schema,
+    )
+
+    op.alter_column(
+        "documents",
+        "embedding",
+        existing_type=Vector(1536),
+        nullable=True,
+        schema=schema,
+    )
+
+    # Add deleted_at column to documents for soft delete support
+    if not column_exists("documents", "deleted_at", inspector):
+        op.add_column(
+            "documents",
+            sa.Column(
+                "deleted_at",
+                sa.DateTime(timezone=True),
+                nullable=True,
+            ),
+            schema=schema,
+        )
+        # Create partial index for efficient cleanup queries (only index non-null values)
+        op.create_index(
+            "ix_documents_deleted_at",
+            "documents",
+            ["deleted_at"],
+            schema=schema,
+            postgresql_where=sa.text("deleted_at IS NOT NULL"),
+        )
+
+    # Add sync state columns to documents table
+    if not column_exists("documents", "sync_state", inspector):
+        op.add_column(
+            "documents",
+            sa.Column(
+                "sync_state",
+                sa.TEXT(),
+                nullable=False,
+                server_default=sa.text(
+                    "'pending'"
+                ),  # Existing records need reconciliation
+            ),
+            schema=schema,
+        )
+        op.create_index(
+            "ix_documents_sync_state",
+            "documents",
+            ["sync_state"],
+            schema=schema,
+        )
+
+    if not column_exists("documents", "last_sync_at", inspector):
+        op.add_column(
+            "documents",
+            sa.Column(
+                "last_sync_at",
+                sa.DateTime(timezone=True),
+                nullable=True,
+            ),
+            schema=schema,
+        )
+
+    if not column_exists("documents", "sync_attempts", inspector):
+        op.add_column(
+            "documents",
+            sa.Column(
+                "sync_attempts",
+                sa.Integer(),
+                nullable=False,
+                server_default=sa.text("0"),
+            ),
+            schema=schema,
+        )
+
+    # Add composite index for efficient reconciliation queries after both columns exist
+    # Reconciliation orders by: WHERE sync_state='pending' ORDER BY last_sync_at
+    if not index_exists("documents", "ix_documents_sync_state_last_sync_at", inspector):
+        op.create_index(
+            "ix_documents_sync_state_last_sync_at",
+            "documents",
+            ["sync_state", "last_sync_at"],
+            schema=schema,
+        )
+
+    # Add sync state columns to message_embeddings table
+    if not column_exists("message_embeddings", "sync_state", inspector):
+        op.add_column(
+            "message_embeddings",
+            sa.Column(
+                "sync_state",
+                sa.TEXT(),
+                nullable=False,
+                server_default=sa.text(
+                    "'pending'"
+                ),  # Existing records need reconciliation
+            ),
+            schema=schema,
+        )
+        op.create_index(
+            "ix_message_embeddings_sync_state",
+            "message_embeddings",
+            ["sync_state"],
+            schema=schema,
+        )
+
+    if not column_exists("message_embeddings", "last_sync_at", inspector):
+        op.add_column(
+            "message_embeddings",
+            sa.Column(
+                "last_sync_at",
+                sa.DateTime(timezone=True),
+                nullable=True,
+            ),
+            schema=schema,
+        )
+
+    if not column_exists("message_embeddings", "sync_attempts", inspector):
+        op.add_column(
+            "message_embeddings",
+            sa.Column(
+                "sync_attempts",
+                sa.Integer(),
+                nullable=False,
+                server_default=sa.text("0"),
+            ),
+            schema=schema,
+        )
+
+    # Add composite index for efficient reconciliation queries after both columns exist
+    # Reconciliation orders by: WHERE sync_state='pending' ORDER BY last_sync_at
+    if not index_exists(
+        "message_embeddings", "ix_message_embeddings_sync_state_last_sync_at", inspector
+    ):
+        op.create_index(
+            "ix_message_embeddings_sync_state_last_sync_at",
+            "message_embeddings",
+            ["sync_state", "last_sync_at"],
+            schema=schema,
+        )
+
+    # Add partial unique index on queue table for reconciler task deduplication
+    # This ensures only one pending reconciler task exists per work_unit_key
+    if not index_exists(
+        "queue", "uq_queue_reconciler_pending_work_unit_key", inspector
+    ):
+        op.create_index(
+            "uq_queue_reconciler_pending_work_unit_key",
+            "queue",
+            ["work_unit_key"],
+            unique=True,
+            schema=schema,
+            postgresql_where=sa.text("task_type = 'reconciler' AND processed = false"),
+        )
+
+    # Make workspace_name nullable on queue table for system-level tasks
+    # This requires dropping and recreating the FK constraint
+    if constraint_exists("queue", "fk_queue_workspace_name", "foreignkey", inspector):
+        op.drop_constraint(
+            "fk_queue_workspace_name", "queue", type_="foreignkey", schema=schema
+        )
+    op.alter_column(
+        "queue",
+        "workspace_name",
+        existing_type=sa.TEXT(),
+        nullable=True,
+        schema=schema,
+    )
+    op.create_foreign_key(
+        "fk_queue_workspace_name",
+        "queue",
+        "workspaces",
+        ["workspace_name"],
+        ["name"],
+        source_schema=schema,
+        referent_schema=schema,
+    )
+
+
+def downgrade() -> None:
+    """Remove deleted_at columns and revert embedding columns."""
+    inspector = sa.inspect(op.get_bind())
+    conn = op.get_bind()
+
+    # Delete system-level queue items (with NULL workspace_name) before reverting to NOT NULL
+    # First delete any active_queue_sessions referencing these queue items
+    batch_size = 5000
+
+    # Delete active_queue_sessions for queue items with NULL workspace_name
+    # Use CTE-based delete since PostgreSQL doesn't support DELETE ... LIMIT
+    while True:
+        result = conn.execute(
+            sa.text(
+                f"""
+                WITH del AS (
+                    SELECT work_unit_key FROM "{schema}".queue
+                    WHERE workspace_name IS NULL
+                    LIMIT :batch_size
+                )
+                DELETE FROM "{schema}".active_queue_sessions
+                WHERE work_unit_key IN (SELECT work_unit_key FROM del)
+                """
+            ),
+            {"batch_size": batch_size},
+        )
+        if result.rowcount == 0:
+            break
+
+    # Delete queue items with NULL workspace_name
+    # Use CTE with ctid for safe batched delete in PostgreSQL
+    while True:
+        result = conn.execute(
+            sa.text(
+                f"""
+                WITH del AS (
+                    SELECT ctid FROM "{schema}".queue
+                    WHERE workspace_name IS NULL
+                    LIMIT :batch_size
+                )
+                DELETE FROM "{schema}".queue
+                WHERE ctid IN (SELECT ctid FROM del)
+                """
+            ),
+            {"batch_size": batch_size},
+        )
+        if result.rowcount == 0:
+            break
+
+    # Revert workspace_name to NOT NULL on queue table
+    op.drop_constraint(
+        "fk_queue_workspace_name", "queue", type_="foreignkey", schema=schema
+    )
+    op.alter_column(
+        "queue",
+        "workspace_name",
+        existing_type=sa.TEXT(),
+        nullable=False,
+        schema=schema,
+    )
+    op.create_foreign_key(
+        "fk_queue_workspace_name",
+        "queue",
+        "workspaces",
+        ["workspace_name"],
+        ["name"],
+        source_schema=schema,
+        referent_schema=schema,
+    )
+
+    # Drop reconciler queue index if it exists
+    if index_exists("queue", "uq_queue_reconciler_pending_work_unit_key", inspector):
+        op.drop_index(
+            "uq_queue_reconciler_pending_work_unit_key",
+            table_name="queue",
+            schema=schema,
+        )
+
+    # Drop message_embeddings indexes if they exist
+    if index_exists(
+        "message_embeddings", "ix_message_embeddings_sync_state_last_sync_at", inspector
+    ):
+        op.drop_index(
+            "ix_message_embeddings_sync_state_last_sync_at",
+            table_name="message_embeddings",
+            schema=schema,
+        )
+    if index_exists(
+        "message_embeddings", "ix_message_embeddings_sync_state", inspector
+    ):
+        op.drop_index(
+            "ix_message_embeddings_sync_state",
+            table_name="message_embeddings",
+            schema=schema,
+        )
+
+    if column_exists("message_embeddings", "sync_state", inspector):
+        op.drop_column("message_embeddings", "sync_state", schema=schema)
+
+    if column_exists("message_embeddings", "sync_attempts", inspector):
+        op.drop_column("message_embeddings", "sync_attempts", schema=schema)
+
+    if column_exists("message_embeddings", "last_sync_at", inspector):
+        op.drop_column("message_embeddings", "last_sync_at", schema=schema)
+
+    # Drop documents indexes if they exist
+    if index_exists("documents", "ix_documents_sync_state_last_sync_at", inspector):
+        op.drop_index(
+            "ix_documents_sync_state_last_sync_at",
+            table_name="documents",
+            schema=schema,
+        )
+    if index_exists("documents", "ix_documents_sync_state", inspector):
+        op.drop_index("ix_documents_sync_state", table_name="documents", schema=schema)
+
+    if column_exists("documents", "sync_state", inspector):
+        op.drop_column("documents", "sync_state", schema=schema)
+
+    # Remove sync state columns from documents
+    if column_exists("documents", "sync_attempts", inspector):
+        op.drop_column("documents", "sync_attempts", schema=schema)
+
+    if column_exists("documents", "last_sync_at", inspector):
+        op.drop_column("documents", "last_sync_at", schema=schema)
+
+    # Remove deleted_at column and index from documents
+    if index_exists("documents", "ix_documents_deleted_at", inspector):
+        op.drop_index("ix_documents_deleted_at", table_name="documents", schema=schema)
+    if column_exists("documents", "deleted_at", inspector):
+        op.drop_column("documents", "deleted_at", schema=schema)
+
+    # NOTE: This downgrade does NOT restore the NOT NULL constraint on embedding columns
+    # in message_embeddings and documents tables. This is intentional to avoid migration
+    # failures if NULL embedding values exist (which is expected when using external vector stores).

migrations/versions/7c0d9a4e3b1f_add_unique_index_for_pending_dreams.py

@@ -28,7 +28,7 @@ schema = get_schema()
 def upgrade() -> None:
     """Add a partial unique index to prevent duplicate pending dream queue items."""
     op.create_index(
-        "ux_queue_dream_pending_work_unit_key",
+        "uq_queue_dream_pending_work_unit_key",
         "queue",
         ["work_unit_key"],
         unique=True,
@@ -40,5 +40,5 @@ def upgrade() -> None:
 def downgrade() -> None:
     """Drop the partial unique index for pending dream queue items."""
     op.drop_index(
-        "ux_queue_dream_pending_work_unit_key", table_name="queue", schema=schema
+        "uq_queue_dream_pending_work_unit_key", table_name="queue", schema=schema
     )

pyproject.toml

@@ -33,6 +33,8 @@ dependencies = [
     "typing-extensions>=4.11.0",
     "prometheus-client>=0.20.0",
     "json-repair>=0.49.0",
+    "turbopuffer>=1.8.1",
+    "lancedb>=0.25.3",
     "redis>=7.0.0,<8.0.0",
     "cashews[redis]==7.4.4",
     "scikit-learn>=1.6.0",
@@ -112,7 +114,7 @@ reportUnusedCallResult = false
 reportCallInDefaultInitializer = false
 reportAny = false
 reportExplicitAny = false
-allowedUntypedLibraries = ["langfuse"]
+allowedUntypedLibraries = ["langfuse", "lancedb", "pyarrow"]
 reportImplicitOverride = false
 reportImportCycles = false
 

sdks/typescript/bun.lock

@@ -1,5 +1,6 @@
 {
   "lockfileVersion": 1,
+  "configVersion": 0,
   "workspaces": {
     "": {
       "name": "@honcho-ai/sdk",

src/config.py

@@ -67,6 +67,7 @@ class TomlConfigSettingsSource(PydanticBaseSettingsSource):
         "SUMMARY": "summary",
         "WEBHOOK": "webhook",
         "DREAM": "dream",
+        "VECTOR_STORE": "vector_store",
         "": "app",  # For AppSettings with no prefix
     }
 
@@ -530,6 +531,51 @@ class DreamSettings(BackupLLMSettingsMixin, HonchoSettings):
         return self
 
 
+class VectorStoreSettings(HonchoSettings):
+    """Settings for vector store (pgvector, Turbopuffer, or LanceDB)."""
+
+    model_config = SettingsConfigDict(env_prefix="VECTOR_STORE_", extra="ignore")  # pyright: ignore
+
+    # Vector store type to use
+    TYPE: Literal["pgvector", "turbopuffer", "lancedb"] = "pgvector"
+
+    MIGRATED: bool = False
+
+    # Global namespace prefix for all vector namespaces
+    # Namespaces follow the pattern: {NAMESPACE}.{type}.{hash}
+    # where hash is a base64url-encoded SHA-256 of the workspace/peer names
+    # - Documents: {NAMESPACE}.doc.{hash(workspace, observer, observed)}
+    # - Messages: {NAMESPACE}.msg.{hash(workspace)}
+    NAMESPACE: str = "honcho"
+
+    DIMENSIONS: Annotated[
+        int,
+        Field(
+            default=1536,
+            gt=0,
+        ),
+    ] = 1536
+
+    # Turbopuffer-specific settings
+    TURBOPUFFER_API_KEY: str | None = None
+    TURBOPUFFER_REGION: str | None = None
+
+    # LanceDB-specific settings (local embedded mode)
+    LANCEDB_PATH: str = "./lancedb_data"
+
+    RECONCILIATION_INTERVAL_SECONDS: Annotated[int, Field(default=300, gt=0)] = (
+        300  # 5 minutes
+    )
+
+    @model_validator(mode="after")
+    def _require_api_key_for_turbopuffer(self) -> "VectorStoreSettings":
+        if self.TYPE == "turbopuffer" and not self.TURBOPUFFER_API_KEY:
+            raise ValueError(
+                "VECTOR_STORE_TURBOPUFFER_API_KEY must be set when TYPE is 'turbopuffer'"
+            )
+        return self
+
+
 class AppSettings(HonchoSettings):
     # No env_prefix for app-level settings
     model_config = SettingsConfigDict(  # pyright: ignore
@@ -572,6 +618,7 @@ class AppSettings(HonchoSettings):
     METRICS: MetricsSettings = Field(default_factory=MetricsSettings)
     CACHE: CacheSettings = Field(default_factory=CacheSettings)
     DREAM: DreamSettings = Field(default_factory=DreamSettings)
+    VECTOR_STORE: VectorStoreSettings = Field(default_factory=VectorStoreSettings)
 
     @field_validator("LOG_LEVEL")
     def validate_log_level(cls, v: str) -> str:
@@ -584,13 +631,17 @@ class AppSettings(HonchoSettings):
     def propagate_namespace(self) -> "AppSettings":
         """Propagate top-level NAMESPACE to nested settings if not explicitly set.
 
-        After this validator runs, CACHE.NAMESPACE and METRICS.NAMESPACE are guaranteed
-        to exist.
+        After this validator runs, CACHE.NAMESPACE, METRICS.NAMESPACE, and
+        VECTOR_STORE.NAMESPACE are guaranteed to exist. Explicitly provided
+        nested namespaces are preserved.
         """
-        if self.CACHE.NAMESPACE is None:
+        if "NAMESPACE" not in self.CACHE.model_fields_set:
             self.CACHE.NAMESPACE = self.NAMESPACE
-        if self.METRICS.NAMESPACE is None:
+        if "NAMESPACE" not in self.METRICS.model_fields_set:
             self.METRICS.NAMESPACE = self.NAMESPACE
+        if "NAMESPACE" not in self.VECTOR_STORE.model_fields_set:
+            self.VECTOR_STORE.NAMESPACE = self.NAMESPACE
+
         return self
 
 

src/crud/document.py

@@ -1,11 +1,13 @@
+import datetime
 from collections.abc import Sequence
 from logging import getLogger
 from typing import Any
 
-from sqlalchemy import delete, select
+from sqlalchemy import delete, select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.sql import Select
+from sqlalchemy.sql.functions import func
 
 from src import models, schemas
 from src.config import settings
@@ -15,6 +17,12 @@ from src.crud.session import get_session
 from src.embedding_client import embedding_client
 from src.exceptions import ResourceNotFoundException, ValidationException
 from src.utils.filter import apply_filter
+from src.vector_store import (
+    VectorRecord,
+    VectorStore,
+    get_external_vector_store,
+    upsert_with_retry,
+)
 
 logger = getLogger(__name__)
 
@@ -49,6 +57,7 @@ def get_all_documents(
         .where(models.Document.workspace_name == workspace_name)
         .where(models.Document.observer == observer)
         .where(models.Document.observed == observed)
+        .where(models.Document.deleted_at.is_(None))  # Exclude soft-deleted
     )
 
     # Apply additional filters if provided
@@ -86,8 +95,10 @@ def get_documents_with_filters(
     Returns:
         Select query for documents
     """
-    stmt = select(models.Document).where(
-        models.Document.workspace_name == workspace_name
+    stmt = (
+        select(models.Document)
+        .where(models.Document.workspace_name == workspace_name)
+        .where(models.Document.deleted_at.is_(None))  # Exclude soft-deleted
     )
 
     # Apply additional filters if provided
@@ -129,6 +140,7 @@ async def query_documents_recent(
         models.Document.workspace_name == workspace_name,
         models.Document.observer == observer,
         models.Document.observed == observed,
+        models.Document.deleted_at.is_(None),
     )
 
     if session_name is not None:
@@ -167,6 +179,7 @@ async def query_documents_most_derived(
             models.Document.workspace_name == workspace_name,
             models.Document.observer == observer,
             models.Document.observed == observed,
+            models.Document.deleted_at.is_(None),
         )
         .order_by(models.Document.times_derived.desc())
         .limit(limit)
@@ -197,7 +210,7 @@ async def query_documents(
         query: Search query text
         observer: Name of the observing peer
         observed: Name of the observed peer
-        filters: Optional filters to apply
+        filters: Optional filters to apply at vector store level (supports: level, session_name)
         max_distance: Maximum cosine distance for results
         top_k: Number of results to return
         embedding: Optional pre-computed embedding for the query (avoids extra API call if possible)
@@ -214,22 +227,89 @@ async def query_documents(
                 f"Query exceeds maximum token limit of {settings.MAX_EMBEDDING_TOKENS}."
             ) from e
 
+    # Query Postgres directly when using pgvector OR during migration (not yet migrated)
+    # This ensures we use pgvector as source of truth until migration is complete
+    if settings.VECTOR_STORE.TYPE == "pgvector" or not settings.VECTOR_STORE.MIGRATED:
+        stmt = (
+            select(models.Document)
+            .where(models.Document.workspace_name == workspace_name)
+            .where(models.Document.observer == observer)
+            .where(models.Document.observed == observed)
+            .where(models.Document.embedding.isnot(None))
+            .where(models.Document.deleted_at.is_(None))
+        )
+
+        if max_distance is not None:
+            stmt = stmt.where(
+                models.Document.embedding.cosine_distance(embedding) <= max_distance
+            )
+
+        stmt = apply_filter(stmt, models.Document, filters)
+        stmt = stmt.order_by(
+            models.Document.embedding.cosine_distance(embedding)
+        ).limit(top_k)
+
+        result = await db.execute(stmt)
+        return list(result.scalars().all())
+
+    # FALLBACK: Use external vector store (Turbopuffer, LanceDB)
+    external_vector_store = get_external_vector_store()
+    if external_vector_store is None:
+        return []
+
+    namespace = external_vector_store.get_vector_namespace(
+        "document", workspace_name, observer, observed
+    )
+
+    # Build vector store filters
+    # Convert filter dict to vector store format (handles level, session_name, etc.)
+    vector_filters: dict[str, Any] = {}
+    if filters:
+        # Direct pass-through for simple equality filters
+        # The filters dict can contain: level, session_name, or other document fields
+        # We can push level and session_name to vector store since they're in metadata
+        for key in ["level", "session_name"]:
+            if key in filters:
+                vector_filters[key] = filters[key]
+
+    # Query external vector store for similar documents with filters applied
+    vector_results = await external_vector_store.query(
+        namespace,
+        embedding,
+        top_k=top_k,
+        max_distance=max_distance,
+        filters=vector_filters if vector_filters else None,
+    )
+
+    if not vector_results:
+        return []
+
+    # Get document IDs from vector results (vector ID = document ID for documents)
+    document_ids = [result.id for result in vector_results]
+
+    # Fetch documents from database
     stmt = (
         select(models.Document)
         .where(models.Document.workspace_name == workspace_name)
         .where(models.Document.observer == observer)
         .where(models.Document.observed == observed)
+        .where(models.Document.deleted_at.is_(None))
+        .where(models.Document.id.in_(document_ids))
     )
-    if max_distance is not None:
-        stmt = stmt.where(
-            models.Document.embedding.cosine_distance(embedding) < max_distance
-        )
+    # Re-apply all filters at the database layer to catch any constraints
+    # that aren't supported by the vector store metadata.
     stmt = apply_filter(stmt, models.Document, filters)
-    stmt = stmt.limit(top_k).order_by(
-        models.Document.embedding.cosine_distance(embedding)
-    )
+
     result = await db.execute(stmt)
-    return result.scalars().all()
+    documents = {doc.id: doc for doc in result.scalars().all()}
+
+    # Return documents in order of similarity (preserving vector store order)
+    ordered_docs: list[models.Document] = []
+    for vr in vector_results:
+        if vr.id in documents:
+            ordered_docs.append(documents[vr.id])
+
+    return ordered_docs
 
 
 async def create_documents(
@@ -255,6 +335,9 @@ async def create_documents(
         Count of new documents
     """
     honcho_documents: list[models.Document] = []
+    # Store (document_model, embedding) pairs - IDs aren't available until after commit
+    docs_with_embeddings: list[tuple[models.Document, list[float]]] = []
+
     for doc in documents:
         try:
             # for each document, if deduplicate is True, perform a process
@@ -268,8 +351,16 @@ async def create_documents(
                     continue
 
             metadata_dict = doc.metadata.model_dump(exclude_none=True)
-            honcho_documents.append(
-                models.Document(
+
+            # Determine if we need to persist embeddings to postgres
+            # True when: TYPE=pgvector OR still migrating (dual-write to both stores)
+            store_embeddings_in_postgres = (
+                settings.VECTOR_STORE.TYPE == "pgvector"
+                or not settings.VECTOR_STORE.MIGRATED
+            )
+
+            if store_embeddings_in_postgres and doc.embedding:
+                new_doc = models.Document(
                     workspace_name=workspace_name,
                     observer=observer,
                     observed=observed,
@@ -277,20 +368,120 @@ async def create_documents(
                     level=doc.level,
                     times_derived=doc.times_derived,
                     internal_metadata=metadata_dict,
+                    session_name=doc.session_name,
                     embedding=doc.embedding,
+                    # Tree linkage column
+                    source_ids=doc.source_ids,
+                )
+            else:
+                new_doc = models.Document(
+                    workspace_name=workspace_name,
+                    observer=observer,
+                    observed=observed,
+                    content=doc.content,
+                    level=doc.level,
+                    times_derived=doc.times_derived,
+                    internal_metadata=metadata_dict,
                     session_name=doc.session_name,
                     # Tree linkage column
                     source_ids=doc.source_ids,
                 )
-            )
+
+            if doc.embedding:
+                new_doc.sync_state = "pending"
+            honcho_documents.append(new_doc)
+
+            # Track embedding for vector store (ID will be available after commit)
+            if doc.embedding:
+                docs_with_embeddings.append((new_doc, doc.embedding))
+
         except Exception as e:
             logger.error(
                 f"Error adding new document to {workspace_name}/{doc.session_name}/{observer}/{observed}: {e}"
             )
             continue
+
     try:
         db.add_all(honcho_documents)
+        # NOTE
+        # If the process crashes after this commit but before vector upsert completes,
+        # documents will be left in sync_state='pending' with NULL embeddings.
+        # The reconciliation job will automatically re-embed and sync these documents,
         await db.commit()
+
+        # Store embeddings in external vector store after documents are committed (IDs now available)
+        if docs_with_embeddings:
+            doc_ids = [doc.id for doc, _ in docs_with_embeddings]
+            external_vector_store = get_external_vector_store()
+
+            # If no external vector store (pgvector mode), mark as synced immediately
+            if external_vector_store is None:
+                await db.execute(
+                    update(models.Document)
+                    .where(models.Document.id.in_(doc_ids))
+                    .values(
+                        sync_state="synced",
+                        last_sync_at=func.now(),
+                        sync_attempts=0,
+                    )
+                )
+                await db.commit()
+            else:
+                # External vector store - upsert and track sync state
+                namespace = external_vector_store.get_vector_namespace(
+                    "document",
+                    workspace_name,
+                    observer,
+                    observed,
+                )
+
+                # Build vector records with metadata for filtering
+                vector_records: list[VectorRecord] = []
+                for doc, embedding in docs_with_embeddings:
+                    vector_records.append(
+                        VectorRecord(
+                            id=doc.id,
+                            embedding=embedding,
+                            metadata={
+                                "workspace_name": workspace_name,
+                                "observer": observer,
+                                "observed": observed,
+                                "session_name": doc.session_name,
+                                "level": doc.level,
+                            },
+                        )
+                    )
+
+                # Upsert to external vector store with retry and update sync state
+                try:
+                    await upsert_with_retry(
+                        external_vector_store, namespace, vector_records
+                    )
+                    # Success: mark as synced
+                    await db.execute(
+                        update(models.Document)
+                        .where(models.Document.id.in_(doc_ids))
+                        .values(
+                            sync_state="synced",
+                            last_sync_at=func.now(),
+                            sync_attempts=0,
+                        )
+                    )
+                    await db.commit()
+
+                except Exception:
+                    # Failed after retries - increment sync_attempts for reconciliation
+                    logger.exception("Failed to upsert vectors after retries")
+                    await db.execute(
+                        update(models.Document)
+                        .where(models.Document.id.in_(doc_ids))
+                        .values(
+                            sync_attempts=models.Document.sync_attempts + 1,
+                            last_sync_at=func.now(),
+                        )
+                    )
+                    await db.commit()
+
     except IntegrityError as e:
         await db.rollback()
         raise ValidationException(
@@ -310,7 +501,10 @@ async def delete_document(
     session_name: str | None = None,
 ) -> None:
     """
-    Delete a single document by ID.
+    Soft-delete a document by ID.
+
+    Sets deleted_at timestamp to mark the document as deleted. The reconciliation
+    job handles vector store cleanup and hard deletion from the database.
 
     Args:
         db: Database session
@@ -323,25 +517,28 @@ async def delete_document(
     Raises:
         ResourceNotFoundException: If document not found or doesn't match criteria
     """
-    stmt = delete(models.Document).where(
+    conditions = [
         models.Document.id == document_id,
         models.Document.workspace_name == workspace_name,
         models.Document.observer == observer,
         models.Document.observed == observed,
-    )
-
-    # If session is specified, ensure document belongs to that session
+        models.Document.deleted_at.is_(None),
+    ]
     if session_name is not None:
-        stmt = stmt.where(models.Document.session_name == session_name)
+        conditions.append(models.Document.session_name == session_name)
 
-    result = await db.execute(stmt)
-    await db.commit()
+    update_stmt = (
+        update(models.Document).where(*conditions).values(deleted_at=func.now())
+    )
+    result = await db.execute(update_stmt)
 
     if result.rowcount == 0:
         raise ResourceNotFoundException(
             f"Document {document_id} not found or does not belong to the specified collection/session"
         )
 
+    await db.commit()
+
 
 async def delete_document_by_id(
     db: AsyncSession,
@@ -349,7 +546,10 @@ async def delete_document_by_id(
     document_id: str,
 ) -> None:
     """
-    Delete a single document by ID and workspace.
+    Soft-delete a document by ID and workspace.
+
+    Sets deleted_at timestamp to mark the document as deleted. The reconciliation
+    job handles vector store cleanup and hard deletion from the database.
 
     Args:
         db: Database session
@@ -359,19 +559,24 @@ async def delete_document_by_id(
     Raises:
         ResourceNotFoundException: If document not found or doesn't belong to the workspace
     """
-    stmt = delete(models.Document).where(
-        models.Document.id == document_id,
-        models.Document.workspace_name == workspace_name,
+    update_stmt = (
+        update(models.Document)
+        .where(
+            models.Document.id == document_id,
+            models.Document.workspace_name == workspace_name,
+            models.Document.deleted_at.is_(None),
+        )
+        .values(deleted_at=func.now())
     )
-
-    result = await db.execute(stmt)
-    await db.commit()
+    result = await db.execute(update_stmt)
 
     if result.rowcount == 0:
         raise ResourceNotFoundException(
             f"Document {document_id} not found or does not belong to workspace {workspace_name}"
         )
 
+    await db.commit()
+
 
 async def create_observations(
     db: AsyncSession,
@@ -431,11 +636,22 @@ async def create_observations(
     except ValueError as e:
         raise ValidationException(str(e)) from e
 
-    # Create document objects
+    # Create document objects and track embeddings for vector store
     honcho_documents: list[models.Document] = []
+    # Group observations by collection (observer, observed) for vector store upserts
+    collection_embeddings: dict[
+        tuple[str, str], list[tuple[models.Document, list[float]]]
+    ] = {}
+
+    # Determine if we need to persist embeddings to postgres
+    # True when: TYPE=pgvector OR still migrating (dual-write to both stores)
+    store_embeddings_in_postgres = (
+        settings.VECTOR_STORE.TYPE == "pgvector" or not settings.VECTOR_STORE.MIGRATED
+    )
+
     for obs, embedding in zip(observations, embeddings, strict=True):
-        honcho_documents.append(
-            models.Document(
+        if store_embeddings_in_postgres:
+            doc = models.Document(
                 workspace_name=workspace_name,
                 observer=obs.observer_id,
                 observed=obs.observed_id,
@@ -443,10 +659,28 @@ async def create_observations(
                 level="explicit",  # Manually created observations are always explicit
                 times_derived=1,
                 internal_metadata={},  # No message_ids since not derived from messages
+                session_name=obs.session_id,
                 embedding=embedding,
+            )
+        else:
+            doc = models.Document(
+                workspace_name=workspace_name,
+                observer=obs.observer_id,
+                observed=obs.observed_id,
+                content=obs.content,
+                level="explicit",  # Manually created observations are always explicit
+                times_derived=1,
+                internal_metadata={},  # No message_ids since not derived from messages
                 session_name=obs.session_id,
             )
-        )
+        doc.sync_state = "pending"
+        honcho_documents.append(doc)
+
+        # Track embedding for vector store (grouped by collection)
+        collection_key = (obs.observer_id, obs.observed_id)
+        if collection_key not in collection_embeddings:
+            collection_embeddings[collection_key] = []
+        collection_embeddings[collection_key].append((doc, embedding))
 
     try:
         db.add_all(honcho_documents)
@@ -454,6 +688,87 @@ async def create_observations(
         # Refresh all documents to get generated IDs and timestamps
         for doc in honcho_documents:
             await db.refresh(doc)
+
+        # Store embeddings in external vector store after documents are committed (IDs now available)
+        external_vector_store = get_external_vector_store()
+        all_doc_ids = [doc.id for doc in honcho_documents]
+
+        # If no external vector store (pgvector mode), mark as synced immediately
+        if external_vector_store is None:
+            await db.execute(
+                update(models.Document)
+                .where(models.Document.id.in_(all_doc_ids))
+                .values(
+                    sync_state="synced",
+                    last_sync_at=func.now(),
+                    sync_attempts=0,
+                )
+            )
+            await db.commit()
+        else:
+            # External vector store - upsert each collection's embeddings
+            for (
+                observer,
+                observed,
+            ), docs_with_embeddings in collection_embeddings.items():
+                namespace = external_vector_store.get_vector_namespace(
+                    "document",
+                    workspace_name,
+                    observer,
+                    observed,
+                )
+
+                # Build vector records with metadata for filtering
+                vector_records: list[VectorRecord] = []
+                doc_ids: list[str] = []
+                for doc, embedding in docs_with_embeddings:
+                    doc_ids.append(doc.id)
+                    vector_records.append(
+                        VectorRecord(
+                            id=doc.id,
+                            embedding=embedding,
+                            metadata={
+                                "workspace_name": workspace_name,
+                                "observer": observer,
+                                "observed": observed,
+                                "session_name": doc.session_name,
+                                "level": doc.level,
+                            },
+                        )
+                    )
+
+                # Upsert to external vector store with retry and update sync state
+                try:
+                    await upsert_with_retry(
+                        external_vector_store, namespace, vector_records
+                    )
+                    # Success: mark as synced
+                    await db.execute(
+                        update(models.Document)
+                        .where(models.Document.id.in_(doc_ids))
+                        .values(
+                            sync_state="synced",
+                            last_sync_at=func.now(),
+                            sync_attempts=0,
+                        )
+                    )
+                    await db.commit()
+
+                except Exception:
+                    # Failed after retries - increment sync_attempts for reconciliation
+                    logger.exception(
+                        f"Failed to upsert vectors for {namespace} after retries"
+                    )
+                    await db.execute(
+                        update(models.Document)
+                        .where(models.Document.id.in_(doc_ids))
+                        .values(
+                            sync_attempts=models.Document.sync_attempts + 1,
+                            last_sync_at=func.now(),
+                        )
+                    )
+                    await db.commit()
+
     except IntegrityError as e:
         await db.rollback()
         raise ValidationException(
@@ -522,8 +837,9 @@ async def is_rejected_duplicate(
         logger.warning(
             f"[DUPLICATE DETECTION] Deleting existing in favor of new. new='{doc.content}', existing='{existing_doc.content}'."
         )
-        await db.delete(existing_doc)
-        await db.flush()  # Flush to make deletion visible in this transaction
+        # Soft-delete the existing document - reconciliation will clean up vectors and hard-delete
+        existing_doc.deleted_at = datetime.datetime.now(datetime.timezone.utc)
+        await db.flush()
         return False  # Don't reject the new document
 
     # Existing document has more information, reject the new one
@@ -533,6 +849,89 @@ async def is_rejected_duplicate(
     return True
 
 
+async def cleanup_soft_deleted_documents(
+    db: AsyncSession,
+    external_vector_store: VectorStore,
+    batch_size: int = 100,
+    older_than_minutes: int = 5,
+) -> int:
+    """
+    Cleanup soft-deleted documents by removing their vectors and database records.
+
+    This function implements a two-phase cleanup process for documents that have been
+    soft-deleted (deleted_at is not NULL)
+
+    Args:
+        db: Database session for executing queries
+        external_vector_store: External vector store instance for deleting vectors
+        batch_size: Maximum number of documents to process per call (default 100)
+        older_than_minutes: Only process documents soft-deleted more than this many
+            minutes ago (default 5).
+
+    Returns:
+        Count of documents cleaned up (only those where vector deletion succeeded).
+    """
+    cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
+        minutes=older_than_minutes
+    )
+
+    # Find soft-deleted documents ready for cleanup
+    # Use FOR UPDATE SKIP LOCKED to prevent multiple deriver instances from
+    # processing the same documents simultaneously
+    stmt = (
+        select(models.Document)
+        .where(models.Document.deleted_at.is_not(None))
+        .where(models.Document.deleted_at < cutoff)
+        .limit(batch_size)
+        .with_for_update(skip_locked=True)
+    )
+    result = await db.execute(stmt)
+    documents = list(result.scalars().all())
+
+    if not documents:
+        return 0
+
+    # Group by namespace for batch vector deletion
+    by_namespace: dict[str, list[str]] = {}
+    for doc in documents:
+        namespace = external_vector_store.get_vector_namespace(
+            "document",
+            doc.workspace_name,
+            doc.observer,
+            doc.observed,
+        )
+        by_namespace.setdefault(namespace, []).append(doc.id)
+
+    # Delete from external vector store (per namespace) and track successful deletions
+    successfully_deleted_ids: set[str] = set()
+    for namespace, ids in by_namespace.items():
+        try:
+            await external_vector_store.delete_many(namespace, ids)
+            # Only add to successfully_deleted_ids if vector deletion succeeded
+            successfully_deleted_ids.update(ids)
+        except Exception as e:
+            # Log but continue - vectors may already be deleted or namespace may not exist
+            logger.warning(f"Failed to delete vectors from {namespace}: {e}")
+
+    # Only hard delete documents where vector deletion succeeded
+    if successfully_deleted_ids:
+        await db.execute(
+            delete(models.Document).where(
+                models.Document.id.in_(successfully_deleted_ids)
+            )
+        )
+        await db.commit()
+        logger.debug(
+            f"Cleaned up {len(successfully_deleted_ids)} soft-deleted documents"
+        )
+        return len(successfully_deleted_ids)
+
+    # No documents were successfully deleted from vector store
+    # Release FOR UPDATE locks by rolling back the transaction
+    await db.rollback()
+    return 0
+
+
 # =============================================================================
 # Tree Traversal Functions - For reasoning chain navigation
 # =============================================================================
@@ -559,6 +958,7 @@ async def get_documents_by_ids(
     stmt = select(models.Document).where(
         models.Document.workspace_name == workspace_name,
         models.Document.id.in_(document_ids),
+        models.Document.deleted_at.is_(None),
     )
     result = await db.execute(stmt)
     return result.scalars().all()
@@ -592,6 +992,7 @@ async def get_child_observations(
     stmt = select(models.Document).where(
         models.Document.workspace_name == workspace_name,
         models.Document.source_ids.contains([parent_id]),
+        models.Document.deleted_at.is_(None),
     )
     if observer:
         stmt = stmt.where(models.Document.observer == observer)

src/crud/message.py

@@ -3,7 +3,7 @@ from logging import getLogger
 from typing import Any
 
 from nanoid import generate as generate_nanoid
-from sqlalchemy import ColumnElement, Select, and_, func, select, text
+from sqlalchemy import ColumnElement, Select, and_, func, select, text, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from src import models, schemas
@@ -11,6 +11,7 @@ from src.config import settings
 from src.embedding_client import embedding_client
 from src.utils.filter import apply_filter
 from src.utils.formatting import ILIKE_ESCAPE_CHAR, escape_ilike_pattern
+from src.vector_store import VectorRecord, get_external_vector_store, upsert_with_retry
 
 from .session import get_or_create_session
 
@@ -209,25 +210,117 @@ async def create_messages(
             }
             embedding_dict = await embedding_client.batch_embed(id_resource_dict)
 
-            # Create MessageEmbedding entries for each embedded message
+            external_vector_store = get_external_vector_store()
+
+            # Determine if we need to persist embeddings to postgres
+            # True when: TYPE=pgvector OR still migrating (dual-write to both stores)
+            store_embeddings_in_postgres = (
+                settings.VECTOR_STORE.TYPE == "pgvector"
+                or not settings.VECTOR_STORE.MIGRATED
+            )
+
+            # Create MessageEmbedding entries
             embedding_objects: list[models.MessageEmbedding] = []
+            # Maps emb index -> (chunk_position, embedding vector)
+            pending_embedding_data: dict[int, tuple[int, list[float]]] = {}
             for message_obj in message_objects:
                 embeddings = embedding_dict.get(message_obj.public_id, [])
-                for embedding in embeddings:
+                for chunk_position, embedding in enumerate(embeddings):
                     embedding_obj = models.MessageEmbedding(
                         content=message_obj.content,
-                        embedding=embedding,
                         message_id=message_obj.public_id,
                         workspace_name=workspace_name,
                         session_name=session_name,
                         peer_name=message_obj.peer_name,
+                        sync_state="pending",
+                        embedding=embedding if store_embeddings_in_postgres else None,
                     )
+                    emb_idx = len(embedding_objects)
+                    pending_embedding_data[emb_idx] = (chunk_position, embedding)
                     embedding_objects.append(embedding_obj)
 
-            # Add all embedding objects to the session
+            # Always create MessageEmbedding rows so reconciliation can track sync state
+            # even when embeddings aren't stored in postgres
+            embedding_ids: list[int] = []
             if embedding_objects:
                 db.add_all(embedding_objects)
-                await db.commit()
+                await db.flush()
+                embedding_ids = [emb.id for emb in embedding_objects]
+
+            await db.commit()
+
+            # If no external vector store (pgvector-only mode), mark as synced immediately
+            if external_vector_store is None:
+                if embedding_ids:
+                    await db.execute(
+                        update(models.MessageEmbedding)
+                        .where(models.MessageEmbedding.id.in_(embedding_ids))
+                        .values(
+                            sync_state="synced",
+                            last_sync_at=func.now(),
+                            sync_attempts=0,
+                        )
+                    )
+                    await db.commit()
+            else:
+                # External vector store - build and upsert vector records
+                namespace = external_vector_store.get_vector_namespace(
+                    "message", workspace_name
+                )
+
+                # Build vector records with {message_id}_{chunk_position} as vector ID
+                vector_records: list[VectorRecord] = []
+                for emb_idx, emb in enumerate(embedding_objects):
+                    chunk_position, embedding = pending_embedding_data[emb_idx]
+                    vector_id = f"{emb.message_id}_{chunk_position}"
+                    vector_records.append(
+                        VectorRecord(
+                            id=vector_id,
+                            embedding=list(embedding),
+                            metadata={
+                                "message_id": emb.message_id,
+                                "session_name": emb.session_name,
+                                "peer_name": emb.peer_name,
+                            },
+                        )
+                    )
+
+                # Upsert to external vector store with retry and update sync state
+                if vector_records:
+                    try:
+                        await upsert_with_retry(
+                            external_vector_store, namespace, vector_records
+                        )
+                        # Success: mark as synced if we have DB rows
+                        if embedding_ids:
+                            await db.execute(
+                                update(models.MessageEmbedding)
+                                .where(models.MessageEmbedding.id.in_(embedding_ids))
+                                .values(
+                                    sync_state="synced",
+                                    last_sync_at=func.now(),
+                                    sync_attempts=0,
+                                )
+                            )
+                            await db.commit()
+
+                    except Exception:
+                        # Failed after retries - increment sync_attempts for reconciliation
+                        logger.exception(
+                            "Failed to upsert message vectors after retries"
+                        )
+                        if embedding_ids:
+                            await db.execute(
+                                update(models.MessageEmbedding)
+                                .where(models.MessageEmbedding.id.in_(embedding_ids))
+                                .values(
+                                    sync_attempts=models.MessageEmbedding.sync_attempts
+                                    + 1,
+                                    last_sync_at=func.now(),
+                                )
+                            )
+                            await db.commit()
+
     except Exception:
         logger.exception(
             "Failed to generate message embeddings for %s messages in workspace %s and session %s.",

src/crud/representation.py

@@ -24,9 +24,6 @@ from src.utils.representation import (
 
 logger = logging.getLogger(__name__)
 
-# Fetch extra documents to ensure we have enough after filtering
-FILTER_OVERSAMPLING_FACTOR = 3
-
 
 class RepresentationManager:
     """Unified manager for representation and document queries."""
@@ -403,30 +400,31 @@ class RepresentationManager:
             observed=self.observed,
             query=query,
             max_distance=max_distance,
-            top_k=count * FILTER_OVERSAMPLING_FACTOR,
+            top_k=count,
             filters=self._build_filter_conditions(level),
         )
 
-        # Sort by creation time and return top count
+        # Sort by creation time
         docs_sorted: list[models.Document] = sorted(
             list(documents), key=lambda x: x.created_at, reverse=True
         )
-        return docs_sorted[:count]
+        return docs_sorted
 
     def _build_filter_conditions(
         self,
         level: str | None = None,
     ) -> dict[str, Any]:
-        """Build complete filter conditions for document queries."""
-        conditions: list[dict[str, Any]] = []
+        """
+        Build filter conditions for document queries.
+
+        Returns a flat dict of key-value pairs for vector store filtering.
+        """
+        filters: dict[str, Any] = {}
 
         if level:
-            conditions.append({"level": level})
+            filters["level"] = level
 
-        if not conditions:
-            return {}
-
-        return conditions[0] if len(conditions) == 1 else {"AND": conditions}
+        return filters
 
 
 # Module-level functions for backward compatibility and convenience

src/crud/session.py

@@ -19,6 +19,7 @@ from src.exceptions import (
 )
 from src.utils.filter import apply_filter
 from src.utils.types import GetOrCreateResult
+from src.vector_store import get_external_vector_store
 
 from .peer import get_or_create_peers, get_peer
 from .workspace import get_or_create_workspace
@@ -425,6 +426,46 @@ async def delete_session(
             )
         )
 
+        # Delete message vectors from vector store before deleting DB records
+        # Fetch all MessageEmbedding records to build vector IDs with {message_id}_{chunk_index}
+        embedding_result = await db.execute(
+            select(models.MessageEmbedding).where(
+                models.MessageEmbedding.session_name == session_name,
+                models.MessageEmbedding.workspace_name == workspace_name,
+            )
+        )
+        embeddings = list(embedding_result.scalars().all())
+        external_vector_store = get_external_vector_store()
+
+        # Only delete from external vector store if one exists
+        if external_vector_store is not None and embeddings:
+            # Compute chunk_index for each embedding based on message_id ordering
+            message_chunks: dict[str, list[models.MessageEmbedding]] = {}
+            for emb in embeddings:
+                message_chunks.setdefault(emb.message_id, []).append(emb)
+
+            # Sort each message's chunks by id and build vector IDs
+            vector_ids: list[str] = []
+            for chunks in message_chunks.values():
+                chunks.sort(key=lambda e: e.id)
+                for chunk_idx, chunk in enumerate(chunks):
+                    vector_ids.append(f"{chunk.message_id}_{chunk_idx}")
+
+            # Try to delete from external vector store (best effort)
+            try:
+                namespace = external_vector_store.get_vector_namespace(
+                    "message", workspace_name
+                )
+                await external_vector_store.delete_many(namespace, vector_ids)
+                logger.debug(
+                    f"Deleted {len(vector_ids)} message vectors for session {session_name}"
+                )
+            except Exception as e:
+                # Log warning but continue - workspace deletion will clean up eventually
+                logger.warning(
+                    f"Failed to delete message vectors for session {session_name}: {e}"
+                )
+
         # Delete MessageEmbedding entries in batches
         await _batch_delete_matching(
             db,
@@ -436,6 +477,46 @@ async def delete_session(
             batch_size=5000,
         )
 
+        # Delete document vectors from vector store before deleting DB records
+        # Fetch all Document records to get IDs and namespaces
+        doc_result = await db.execute(
+            select(
+                models.Document.id,
+                models.Document.observer,
+                models.Document.observed,
+            ).where(
+                models.Document.session_name == session_name,
+                models.Document.workspace_name == workspace_name,
+            )
+        )
+        documents = doc_result.all()
+
+        # Only delete from external vector store if one exists
+        if external_vector_store is not None and documents:
+            # Group document IDs by namespace (observer/observed)
+            docs_by_namespace: dict[str, list[str]] = {}
+            for doc in documents:
+                namespace = external_vector_store.get_vector_namespace(
+                    "document",
+                    workspace_name,
+                    doc.observer,
+                    doc.observed,
+                )
+                docs_by_namespace.setdefault(namespace, []).append(doc.id)
+
+            # Try to delete from external vector store (best effort, per namespace)
+            for namespace, doc_ids in docs_by_namespace.items():
+                try:
+                    await external_vector_store.delete_many(namespace, doc_ids)
+                    logger.debug(
+                        f"Deleted {len(doc_ids)} document vectors from {namespace}"
+                    )
+                except Exception as e:
+                    # Log warning but continue - workspace deletion will clean up eventually
+                    logger.warning(
+                        f"Failed to delete document vectors from {namespace}: {e}"
+                    )
+
         # Delete Document entries associated with this session in batches
         await _batch_delete_matching(
             db,

src/crud/workspace.py

@@ -12,6 +12,7 @@ from src.config import settings
 from src.exceptions import ConflictException, ResourceNotFoundException
 from src.utils.filter import apply_filter
 from src.utils.types import GetOrCreateResult
+from src.vector_store import get_external_vector_store
 
 logger = getLogger(__name__)
 
@@ -274,6 +275,14 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor
             )
         )
 
+        # Get all collections for this workspace to delete their vector namespaces
+        collections_result = await db.execute(
+            select(models.Collection).where(
+                models.Collection.workspace_name == workspace_name
+            )
+        )
+        collections = collections_result.scalars().all()
+
         await db.execute(
             delete(models.MessageEmbedding).where(
                 models.MessageEmbedding.workspace_name == workspace_name
@@ -316,6 +325,51 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor
         await db.delete(honcho_workspace)
         await db.commit()
 
+        # Delete vector store namespaces for this workspace
+        external_vector_store = get_external_vector_store()
+
+        # Delete message embeddings namespace for this workspace
+        if external_vector_store:
+            message_namespace = external_vector_store.get_vector_namespace(
+                "message", workspace_name
+            )
+            try:
+                await external_vector_store.delete_namespace(message_namespace)
+                logger.debug(
+                    "Deleted message embeddings namespace %s for workspace %s",
+                    message_namespace,
+                    workspace_name,
+                )
+            except Exception as e:
+                logger.warning(
+                    "Failed to delete message embeddings namespace %s: %s",
+                    message_namespace,
+                    e,
+                )
+
+            # Delete document embeddings namespaces for each collection
+            for collection in collections:
+                doc_namespace = external_vector_store.get_vector_namespace(
+                    "document",
+                    workspace_name,
+                    collection.observer,
+                    collection.observed,
+                )
+                try:
+                    await external_vector_store.delete_namespace(doc_namespace)
+                    logger.debug(
+                        "Deleted document namespace %s for collection %s/%s",
+                        doc_namespace,
+                        collection.observer,
+                        collection.observed,
+                    )
+                except Exception as e:
+                    logger.warning(
+                        "Failed to delete document namespace %s: %s",
+                        doc_namespace,
+                        e,
+                    )
+
         cache_key = workspace_cache_key(workspace_name)
         workspace_pattern = f"{cache_key}*"
         await cache.delete_match(workspace_pattern)

src/deriver/consumer.py

@@ -10,12 +10,15 @@ from src.deriver.deriver import process_representation_tasks_batch
 from src.dreamer.dreamer import process_dream
 from src.exceptions import ResourceNotFoundException
 from src.models import Message
-from src.schemas import ResolvedConfiguration
+from src.reconciler.queue_cleanup import cleanup_queue_items
+from src.reconciler.sync_vectors import run_vector_reconciliation_cycle
+from src.schemas import ReconcilerType, ResolvedConfiguration
 from src.utils import summarizer
 from src.utils.logging import log_performance_metrics
 from src.utils.queue_payload import (
     DeletionPayload,
     DreamPayload,
+    ReconcilerPayload,
     SummaryPayload,
     WebhookPayload,
 )
@@ -31,6 +34,25 @@ async def process_item(queue_item: models.QueueItem) -> None:
     queue_payload = queue_item.payload
     workspace_name = queue_item.workspace_name
 
+    # Handle reconciler first - it's the only task type that doesn't require workspace_name
+    if task_type == "reconciler":
+        with sentry_sdk.start_transaction(name="process_reconciler_task", op="deriver"):
+            try:
+                validated = ReconcilerPayload(**queue_payload)
+            except ValidationError as e:
+                logger.error(
+                    "Invalid reconciler payload received: %s. Payload: %s",
+                    str(e),
+                    queue_payload,
+                )
+                raise ValueError(f"Invalid payload structure: {str(e)}") from e
+            await process_reconciler(validated)
+        return
+
+    # All other task types require a workspace_name
+    if workspace_name is None:
+        raise ValueError(f"{task_type} tasks require a workspace_name")
+
     if task_type == "webhook":
         try:
             validated = WebhookPayload(**queue_payload)
@@ -220,3 +242,43 @@ async def process_deletion(
 
         else:
             raise ValueError(f"Unsupported deletion type: {deletion_type}")
+
+
+async def process_reconciler(payload: ReconcilerPayload) -> None:
+    """
+    Process a reconciler task from the queue.
+
+    Currently supports:
+    - sync_vectors: Syncs pending documents/message embeddings to vector store
+      and cleans up soft-deleted documents.
+    - cleanup_queue: Removes old processed queue items.
+
+    Args:
+        payload: The reconciler payload containing the reconciler type
+    """
+    reconciler_type = payload.reconciler_type
+
+    if reconciler_type == ReconcilerType.SYNC_VECTORS:
+        logger.debug("Processing sync_vectors task")
+        metrics = await run_vector_reconciliation_cycle()
+
+        if (
+            metrics.total_synced > 0
+            or metrics.total_failed > 0
+            or metrics.total_cleaned > 0
+        ):
+            logger.info(
+                "Reconciliation complete: synced %s docs, %s message embeddings; failed %s docs, %s message embeddings; cleaned %s docs",
+                metrics.documents_synced,
+                metrics.message_embeddings_synced,
+                metrics.documents_failed,
+                metrics.message_embeddings_failed,
+                metrics.documents_cleaned,
+            )
+
+    elif reconciler_type == ReconcilerType.CLEANUP_QUEUE:
+        logger.debug("Processing cleanup_queue task")
+        await cleanup_queue_items()
+
+    else:
+        raise ValueError(f"Unsupported reconciler type: {reconciler_type}")

src/deriver/queue_manager.py

@@ -19,13 +19,21 @@ from src import models, prometheus
 from src.cache.client import close_cache, init_cache
 from src.config import settings
 from src.dependencies import tracked_db
-from src.deriver.consumer import process_item, process_representation_batch
+from src.deriver.consumer import (
+    process_item,
+    process_representation_batch,
+)
 from src.dreamer.dream_scheduler import (
     DreamScheduler,
     get_dream_scheduler,
     set_dream_scheduler,
 )
 from src.models import QueueItem
+from src.reconciler import (
+    ReconcilerScheduler,
+    get_reconciler_scheduler,
+    set_reconciler_scheduler,
+)
 from src.schemas import ResolvedConfiguration
 from src.sentry import initialize_sentry
 from src.utils.work_unit import parse_work_unit_key
@@ -52,7 +60,6 @@ class QueueManager:
         self.active_tasks: set[asyncio.Task[None]] = set()
         self.worker_ownership: dict[str, WorkerOwnership] = {}
         self.queue_empty_flag: asyncio.Event = asyncio.Event()
-        self._maintenance_task: asyncio.Task[None] | None = None
 
         # Initialize from settings
         self.workers: int = settings.DERIVER.WORKERS
@@ -66,6 +73,14 @@ class QueueManager:
         else:
             self.dream_scheduler = existing_scheduler
 
+        # Get or create the singleton reconciler scheduler
+        existing_reconciler = get_reconciler_scheduler()
+        if existing_reconciler is None:
+            self.reconciler_scheduler: ReconcilerScheduler = ReconcilerScheduler()
+            set_reconciler_scheduler(self.reconciler_scheduler)
+        else:
+            self.reconciler_scheduler = existing_reconciler
+
         # Initialize Sentry if enabled, using settings
         if settings.SENTRY.ENABLED:
             initialize_sentry(integrations=[AsyncioIntegration()])
@@ -108,11 +123,11 @@ class QueueManager:
             )
         logger.debug("Signal handlers registered")
 
-        # Start background maintenance loop
+        # Start the reconciler scheduler
         try:
-            self._maintenance_task = asyncio.create_task(self._maintenance_loop())
+            await self.reconciler_scheduler.start()
         except Exception:
-            logger.exception("Failed to start maintenance loop")
+            logger.exception("Failed to start reconciler scheduler")
 
         # Run the polling loop directly in this task
         logger.debug("Starting polling loop directly")
@@ -129,6 +144,9 @@ class QueueManager:
         # Cancel all pending dreams
         await self.dream_scheduler.shutdown()
 
+        # Stop the reconciler scheduler
+        await self.reconciler_scheduler.shutdown()
+
         if self.active_tasks:
             logger.info(
                 f"Waiting for {len(self.active_tasks)} active tasks to complete..."
@@ -160,14 +178,6 @@ class QueueManager:
             finally:
                 self.worker_ownership.clear()
 
-        # Cancel maintenance loop if running
-        if self._maintenance_task is not None:
-            from contextlib import suppress
-
-            self._maintenance_task.cancel()
-            with suppress(asyncio.CancelledError):
-                await self._maintenance_task
-
     ##########################
     # Polling and Scheduling #
     ##########################
@@ -353,55 +363,6 @@ class QueueManager:
     # Queue Worker Logic #
     ######################
 
-    async def cleanup_queue_items(self) -> None:
-        """Delete processed queue items.
-        Successfully processed queue items are deleted immediately,
-        while errored queue items are deleted after retention window."""
-        async with tracked_db("cleanup_queue_items") as db:
-            now = datetime.now(timezone.utc)
-            error_cutoff = now - timedelta(
-                seconds=settings.DERIVER.QUEUE_ERROR_RETENTION_SECONDS
-            )
-
-            await db.execute(
-                delete(models.QueueItem).where(
-                    models.QueueItem.processed
-                    & (
-                        models.QueueItem.error.is_(None)
-                        | (
-                            models.QueueItem.error.is_not(None)
-                            & (models.QueueItem.created_at < error_cutoff)
-                        )
-                    )
-                )
-            )
-            await db.commit()
-
-    async def _maintenance_loop(self) -> None:
-        """Run periodic maintenance tasks on the queue."""
-        try:
-            while not self.shutdown_event.is_set():
-                try:
-                    await self.cleanup_queue_items()
-                except Exception:
-                    logger.exception("Error during maintenance cleanup")
-                    if settings.SENTRY.ENABLED:
-                        sentry_sdk.capture_exception()
-
-                # Sleep until interval elapses or shutdown event is set
-                try:
-                    await asyncio.wait_for(
-                        self.shutdown_event.wait(),
-                        timeout=43200,  # 12 hours
-                    )
-                    break  # Shutdown event set
-                except asyncio.TimeoutError:
-                    # Timeout means it's time for next cleanup
-                    pass
-        except asyncio.CancelledError:
-            logger.debug("Maintenance loop cancelled")
-            raise
-
     async def _handle_processing_error(
         self,
         error: Exception,
@@ -545,7 +506,10 @@ class QueueManager:
                 if removed and queue_item_count > 0:
                     # Only publish webhook if we actually removed an active session
                     try:
-                        if work_unit.task_type in ["representation", "summary"]:
+                        if (
+                            work_unit.task_type in ["representation", "summary"]
+                            and work_unit.workspace_name is not None
+                        ):
                             logger.debug(
                                 f"Publishing queue.empty event for {work_unit_key} in workspace {work_unit.workspace_name}"
                             )
@@ -807,7 +771,10 @@ class QueueManager:
             )
             await db.commit()
 
-            if work_unit.task_type in ["representation", "summary"]:
+            if (
+                work_unit.task_type in ["representation", "summary"]
+                and work_unit.workspace_name is not None
+            ):
                 prometheus.DERIVER_QUEUE_ITEMS_PROCESSED.labels(
                     workspace_name=work_unit.workspace_name,
                     task_type=work_unit.task_type,
@@ -853,6 +820,7 @@ class QueueManager:
 
 async def main():
     logger.debug("Starting queue manager")
+
     try:
         await init_cache()
     except Exception as e:

src/exceptions.py

@@ -125,6 +125,14 @@ class SpecialistExecutionError(HonchoException):
     detail = "Specialist execution failed"
 
 
+@final
+class VectorStoreError(HonchoException):
+    """Exception raised when a vector store operation fails."""
+
+    status_code = 500
+    detail = "Vector store operation failed"
+
+
 class LLMError(Exception):
     """Exception raised when an LLM call fails.
 

src/main.py

@@ -125,6 +125,10 @@ async def lifespan(_: FastAPI):
     try:
         yield
     finally:
+        # Import here to avoid circular import at module load time
+        from src.vector_store import close_external_vector_store
+
+        await close_external_vector_store()
         await close_cache()
         await engine.dispose()
 

src/models.py

@@ -21,12 +21,11 @@ from sqlalchemy import (
     text,
 )
 from sqlalchemy.dialects.postgresql import JSONB, TEXT
-from sqlalchemy.orm import Mapped, mapped_column, relationship
-from sqlalchemy.orm.properties import MappedColumn
+from sqlalchemy.orm import Mapped, MappedColumn, mapped_column, relationship
 from sqlalchemy.sql import func
 from typing_extensions import override
 
-from src.utils.types import DocumentLevel, TaskType
+from src.utils.types import DocumentLevel, TaskType, VectorSyncState
 
 from .db import Base
 
@@ -279,7 +278,7 @@ class MessageEmbedding(Base):
         BigInteger, Identity(), primary_key=True, autoincrement=True
     )
     content: Mapped[str] = mapped_column(TEXT)
-    embedding: MappedColumn[Any] = mapped_column(Vector(1536))
+    embedding: MappedColumn[Any] = mapped_column(Vector(1536), nullable=True)
     message_id: Mapped[str] = mapped_column(
         ForeignKey("messages.public_id", ondelete="CASCADE"), nullable=False, index=True
     )
@@ -291,6 +290,16 @@ class MessageEmbedding(Base):
     created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), server_default=func.now(), index=True
     )
+    # Vector sync state tracking
+    sync_state: Mapped[VectorSyncState] = mapped_column(
+        TEXT, nullable=False, server_default="pending", index=True
+    )
+    last_sync_at: Mapped[datetime.datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
+    sync_attempts: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=0, server_default=text("0")
+    )
 
     __table_args__ = (
         # Compound foreign key constraints
@@ -310,6 +319,12 @@ class MessageEmbedding(Base):
             postgresql_with={"m": 16, "ef_construction": 64},
             postgresql_ops={"embedding": "vector_cosine_ops"},
         ),
+        # Composite index for efficient reconciliation queries
+        Index(
+            "ix_message_embeddings_sync_state_last_sync_at",
+            "sync_state",
+            "last_sync_at",
+        ),
     )
 
 
@@ -371,10 +386,10 @@ class Document(Base):
     times_derived: Mapped[int] = mapped_column(
         Integer, nullable=False, server_default=text("1")
     )
+    embedding: MappedColumn[Any] = mapped_column(Vector(1536), nullable=True)
     source_ids: Mapped[list[str] | None] = mapped_column(
         JSONB, nullable=True, server_default=text("NULL")
     )
-    embedding: MappedColumn[Any] = mapped_column(Vector(1536))
     created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), server_default=func.now(), index=True
     )
@@ -385,6 +400,21 @@ class Document(Base):
         ForeignKey("workspaces.name"), nullable=False, index=True
     )
     session_name: Mapped[str] = mapped_column(TEXT, index=True)
+    deleted_at: Mapped[datetime.datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True, index=True, default=None
+    )
+
+    # Vector sync state tracking
+    sync_state: Mapped[VectorSyncState] = mapped_column(
+        TEXT, nullable=False, server_default="pending", index=True
+    )
+    last_sync_at: Mapped[datetime.datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
+    sync_attempts: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=0, server_default=text("0")
+    )
+
     collection = relationship("Collection", back_populates="documents")
 
     __table_args__ = (
@@ -431,6 +461,12 @@ class Document(Base):
             "source_ids",
             postgresql_using="gin",
         ),
+        # Composite index for efficient reconciliation queries
+        Index(
+            "ix_documents_sync_state_last_sync_at",
+            "sync_state",
+            "last_sync_at",
+        ),
     )
 
 
@@ -454,8 +490,8 @@ class QueueItem(Base):
     created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), server_default=func.now(), index=True
     )
-    workspace_name: Mapped[str] = mapped_column(
-        ForeignKey("workspaces.name"), nullable=False, index=True
+    workspace_name: Mapped[str | None] = mapped_column(
+        ForeignKey("workspaces.name"), nullable=True, index=True
     )
     message_id: Mapped[int | None] = mapped_column(
         BigInteger, ForeignKey("messages.id"), nullable=True
@@ -473,8 +509,16 @@ class QueueItem(Base):
             "processed",
             "id",
         ),
+        # Partial unique index for reconciler task deduplication
         Index(
-            "ux_queue_dream_pending_work_unit_key",
+            "uq_queue_reconciler_pending_work_unit_key",
+            "work_unit_key",
+            unique=True,
+            postgresql_where=text("task_type = 'reconciler' AND processed = false"),
+        ),
+        # Partial unique index for dream task deduplication
+        Index(
+            "uq_queue_dream_pending_work_unit_key",
             "work_unit_key",
             unique=True,
             postgresql_where=text("task_type = 'dream' AND processed = false"),

src/reconciler/__init__.py

@@ -0,0 +1,13 @@
+"""Reconciler for self-healing background tasks like vector sync and cleanup."""
+
+from .scheduler import (
+    ReconcilerScheduler,
+    get_reconciler_scheduler,
+    set_reconciler_scheduler,
+)
+
+__all__ = [
+    "ReconcilerScheduler",
+    "get_reconciler_scheduler",
+    "set_reconciler_scheduler",
+]

src/reconciler/queue_cleanup.py

@@ -0,0 +1,45 @@
+"""
+Queue cleanup job.
+
+This module provides a periodic cleanup job that removes old processed queue items.
+"""
+
+import logging
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import delete
+
+from src import models
+from src.config import settings
+from src.dependencies import tracked_db
+
+logger = logging.getLogger(__name__)
+
+
+async def cleanup_queue_items() -> None:
+    """
+    Delete processed queue items.
+
+    Successfully processed queue items are deleted immediately,
+    while errored queue items are deleted after retention window.
+    """
+    async with tracked_db("cleanup_queue_items") as db:
+        now = datetime.now(timezone.utc)
+        error_cutoff = now - timedelta(
+            seconds=settings.DERIVER.QUEUE_ERROR_RETENTION_SECONDS
+        )
+
+        await db.execute(
+            delete(models.QueueItem).where(
+                models.QueueItem.processed
+                & (
+                    models.QueueItem.error.is_(None)
+                    | (
+                        models.QueueItem.error.is_not(None)
+                        & (models.QueueItem.created_at < error_cutoff)
+                    )
+                )
+            )
+        )
+        await db.commit()
+        logger.info("Queue cleanup completed")

src/reconciler/scheduler.py

@@ -0,0 +1,268 @@
+"""
+Reconciler scheduler for self-healing background tasks.
+
+This module provides a scheduler for running reconciliation and cleanup tasks
+like vector store sync and soft-delete cleanup. It ensures only one task of each
+type runs at a time across multiple deriver instances by using the queue table
+for coordination.
+"""
+
+import asyncio
+import contextlib
+import logging
+from datetime import datetime, timedelta, timezone
+
+import sentry_sdk
+from pydantic import BaseModel
+from sqlalchemy import exists, select
+from sqlalchemy.exc import IntegrityError
+
+from src import models
+from src.config import settings
+from src.dependencies import tracked_db
+from src.models import QueueItem
+
+logger = logging.getLogger(__name__)
+
+
+class ReconcilerTask(BaseModel):
+    """Definition of a reconciler task."""
+
+    name: str
+    work_unit_key: str
+    interval_seconds: int
+
+
+# Task intervals
+QUEUE_CLEANUP_INTERVAL_SECONDS = 12 * 3600  # 12 hours
+
+# Task registry - add new tasks here
+RECONCILER_TASKS: dict[str, ReconcilerTask] = {
+    "sync_vectors": ReconcilerTask(
+        name="sync_vectors",
+        work_unit_key="reconciler:sync_vectors",
+        interval_seconds=settings.VECTOR_STORE.RECONCILIATION_INTERVAL_SECONDS,
+    ),
+    "cleanup_queue": ReconcilerTask(
+        name="cleanup_queue",
+        work_unit_key="reconciler:cleanup_queue",
+        interval_seconds=QUEUE_CLEANUP_INTERVAL_SECONDS,
+    ),
+}
+
+
+_reconciler_scheduler: "ReconcilerScheduler | None" = None
+
+
+def set_reconciler_scheduler(scheduler: "ReconcilerScheduler") -> None:
+    """Set the global reconciler scheduler reference."""
+    global _reconciler_scheduler
+    _reconciler_scheduler = scheduler
+
+
+def get_reconciler_scheduler() -> "ReconcilerScheduler | None":
+    """Get the global reconciler scheduler reference."""
+    return _reconciler_scheduler
+
+
+class ReconcilerScheduler:
+    """
+    Scheduler for self-healing reconciliation and cleanup tasks.
+
+    Ensures only one task of each type runs at a time across multiple deriver
+    instances by using the queue table for coordination. This provides:
+    - Vector store synchronization (syncing pending documents/embeddings)
+    - Soft-delete cleanup (removing soft-deleted documents from vector stores)
+    - Self-healing behavior (retrying failed syncs)
+    - Extensible task registry for adding new maintenance tasks
+
+    Each task type has its own interval and work_unit_key, allowing multiple
+    different tasks to be queued simultaneously while preventing duplicates
+    of the same task type.
+    """
+
+    _instance: "ReconcilerScheduler | None" = None
+    _initialized: bool = False
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        # Only initialize once
+        if not ReconcilerScheduler._initialized:
+            self._scheduler_task: asyncio.Task[None] | None = None
+            self._shutdown_event: asyncio.Event = asyncio.Event()
+            # Track next run time for each task
+            self._next_run: dict[str, datetime] = {}
+            ReconcilerScheduler._initialized = True
+
+    @classmethod
+    def reset_singleton(cls) -> None:
+        """Reset the singleton instance. Only use this in tests."""
+        cls._instance = None
+        cls._initialized = False
+
+    async def start(self) -> None:
+        """Start the reconciler scheduler loop."""
+        if self._scheduler_task is not None:
+            logger.warning("ReconcilerScheduler already running")
+            return
+
+        self._shutdown_event.clear()
+        # Initialize next run times to first interval
+        now = datetime.now(timezone.utc)
+        for task_name, task in RECONCILER_TASKS.items():
+            self._next_run[task_name] = now + timedelta(seconds=task.interval_seconds)
+
+        self._scheduler_task = asyncio.create_task(self._scheduler_loop())
+        logger.info(
+            "ReconcilerScheduler started with %d tasks: %s",
+            len(RECONCILER_TASKS),
+            list(RECONCILER_TASKS.keys()),
+        )
+
+    async def shutdown(self) -> None:
+        """Stop the reconciler scheduler."""
+        if self._scheduler_task is None:
+            return
+
+        logger.info("Shutting down ReconcilerScheduler...")
+        self._shutdown_event.set()
+
+        try:
+            await asyncio.wait_for(self._scheduler_task, timeout=5.0)
+        except asyncio.TimeoutError:
+            logger.warning("ReconcilerScheduler shutdown timed out, cancelling task")
+            self._scheduler_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._scheduler_task
+
+        self._scheduler_task = None
+        self._next_run.clear()
+        logger.info("ReconcilerScheduler stopped")
+
+    async def _scheduler_loop(self) -> None:
+        """
+        Main scheduler loop that enqueues tasks based on their intervals.
+
+        Each task has its own interval and the loop checks all tasks on each
+        iteration, enqueueing any that are due.
+        """
+        try:
+            while not self._shutdown_event.is_set():
+                now = datetime.now(timezone.utc)
+
+                # Check each task and enqueue if due
+                for task_name, task in RECONCILER_TASKS.items():
+                    next_run = self._next_run.get(task_name, now)
+                    if now >= next_run:
+                        try:
+                            enqueued = await self._try_enqueue_task(task)
+                            if enqueued:
+                                logger.debug("Enqueued task: %s", task_name)
+                        except Exception as e:
+                            logger.exception("Error enqueueing task %s", task_name)
+                            if settings.SENTRY.ENABLED:
+                                sentry_sdk.capture_exception(e)
+
+                        # Schedule next run regardless of whether we enqueued
+                        # (if already pending/in-progress, we'll skip next time too)
+                        self._next_run[task_name] = now + timedelta(
+                            seconds=task.interval_seconds
+                        )
+
+                # Calculate sleep time until next task is due
+                if self._next_run:
+                    next_task_time = min(self._next_run.values())
+                    sleep_seconds = max(
+                        1.0,  # At least 1 second to avoid busy loop
+                        (next_task_time - datetime.now(timezone.utc)).total_seconds(),
+                    )
+                else:
+                    sleep_seconds = 60.0  # Default if no tasks
+
+                try:
+                    await asyncio.wait_for(
+                        self._shutdown_event.wait(),
+                        timeout=sleep_seconds,
+                    )
+                    break  # Shutdown event was set
+                except asyncio.TimeoutError:
+                    # Timeout means interval elapsed, continue loop
+                    pass
+
+        except asyncio.CancelledError:
+            logger.debug("ReconcilerScheduler loop cancelled")
+            raise
+
+    async def _try_enqueue_task(self, task: ReconcilerTask) -> bool:
+        """
+        Attempt to enqueue a reconciler task.
+
+        This is idempotent - if the task is already in-progress (has an
+        ActiveQueueSession) or pending in the queue, the enqueue is skipped.
+
+        Args:
+            task: The task definition to enqueue
+
+        Returns:
+            True if a task was enqueued, False if skipped
+        """
+        async with tracked_db("reconciler_enqueue") as db:
+            # Check if task is already in progress
+            in_progress_check = select(
+                exists(
+                    select(models.ActiveQueueSession.id).where(
+                        models.ActiveQueueSession.work_unit_key == task.work_unit_key
+                    )
+                )
+            )
+            is_in_progress = await db.scalar(in_progress_check)
+
+            if is_in_progress:
+                logger.debug("Task %s already in progress, skipping enqueue", task.name)
+                return False
+
+            # Check if there's already a pending task
+            pending_check = select(
+                exists(
+                    select(QueueItem.id).where(
+                        QueueItem.work_unit_key == task.work_unit_key,
+                        QueueItem.processed == False,  # noqa: E712
+                    )
+                )
+            )
+            is_pending = await db.scalar(pending_check)
+
+            if is_pending:
+                logger.debug(
+                    "Task %s already pending in queue, skipping enqueue", task.name
+                )
+                return False
+
+            # Enqueue the task using ORM
+            queue_item = QueueItem(
+                work_unit_key=task.work_unit_key,
+                payload={
+                    "reconciler_type": task.name,
+                },
+                session_id=None,
+                task_type="reconciler",
+                workspace_name=None,
+                message_id=None,
+            )
+            db.add(queue_item)
+            try:
+                await db.commit()
+            except IntegrityError:
+                # Another instance already enqueued this task (unique constraint)
+                await db.rollback()
+                logger.debug(
+                    "Task %s already enqueued by another instance, skipping", task.name
+                )
+                return False
+
+            logger.info("Enqueued reconciler task: %s", task.name)
+            return True

src/reconciler/sync_vectors.py

@@ -0,0 +1,619 @@
+"""
+Vector store reconciliation job.
+
+This module provides a periodic reconciliation job that syncs documents and message
+embeddings to the vector store on a rolling basis, healing any missed writes.
+"""
+
+import datetime
+import logging
+import time
+from dataclasses import dataclass
+from typing import cast
+
+from sqlalchemy import and_, bindparam, delete, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.functions import func
+
+from src import models
+from src.config import settings
+from src.dependencies import tracked_db
+from src.embedding_client import embedding_client
+from src.vector_store import VectorRecord, VectorStore, get_external_vector_store
+
+logger = logging.getLogger(__name__)
+
+# Constants
+RECONCILIATION_BATCH_SIZE = 50
+RECONCILIATION_TIME_BUDGET_SECONDS = 240  # Leave headroom for other maintenance work
+MAX_SYNC_ATTEMPTS = 5  # After this many failures, mark as failed
+
+
+@dataclass
+class ReconciliationMetrics:
+    """Metrics for a reconciliation cycle."""
+
+    documents_synced: int = 0
+    documents_failed: int = 0
+    documents_cleaned: int = 0
+    message_embeddings_synced: int = 0
+    message_embeddings_failed: int = 0
+
+    @property
+    def total_synced(self) -> int:
+        return self.documents_synced + self.message_embeddings_synced
+
+    @property
+    def total_failed(self) -> int:
+        return self.documents_failed + self.message_embeddings_failed
+
+    @property
+    def total_cleaned(self) -> int:
+        return self.documents_cleaned
+
+
+async def _get_documents_needing_sync(
+    db: AsyncSession,
+    batch_size: int = RECONCILIATION_BATCH_SIZE,
+) -> list[models.Document]:
+    """
+    Get documents that need to be synced to the vector store.
+
+    Finds documents where:
+    - not soft-deleted (deleted_at is NULL)
+    - sync_state is "pending" (never synced or retry needed)
+    - Note: "synced" = done forever, "failed" = permanent failure (manual intervention)
+
+    Uses FOR UPDATE SKIP LOCKED to prevent concurrent processing.
+    """
+    stmt = (
+        select(models.Document)
+        .where(
+            and_(
+                models.Document.deleted_at.is_(None),
+                models.Document.sync_state == "pending",  # Only pending items
+            )
+        )
+        .order_by(models.Document.last_sync_at.asc().nullsfirst())
+        .limit(batch_size)
+        .with_for_update(skip_locked=True)
+    )
+
+    result = await db.execute(stmt)
+    return list(result.scalars().all())
+
+
+async def _get_message_embeddings_needing_sync(
+    db: AsyncSession,
+    batch_size: int = RECONCILIATION_BATCH_SIZE,
+) -> list[models.MessageEmbedding]:
+    """
+    Get pending message embeddings that need to be synced to the vector store.
+
+    Returns only pending embeddings (with full data including embedding vectors).
+    The batch_size limits the number of embeddings returned.
+
+    Uses FOR UPDATE SKIP LOCKED to prevent concurrent processing and
+    orders by last_sync_at (nulls first) to prioritize never-synced records.
+
+    Note: "synced" = done forever, "failed" = permanent failure (manual intervention)
+    """
+    stmt = (
+        select(models.MessageEmbedding)
+        .where(models.MessageEmbedding.sync_state == "pending")
+        .order_by(models.MessageEmbedding.last_sync_at.asc().nullsfirst())
+        .limit(batch_size)
+        .with_for_update(skip_locked=True)
+    )
+
+    result = await db.execute(stmt)
+    return list(result.scalars().all())
+
+
+async def _bump_document_sync_attempts(
+    db: AsyncSession,
+    documents: list[models.Document],
+) -> None:
+    if not documents:
+        return
+
+    for doc in documents:
+        new_attempts = doc.sync_attempts + 1
+        new_state = "failed" if new_attempts >= MAX_SYNC_ATTEMPTS else "pending"
+        await db.execute(
+            update(models.Document)
+            .where(models.Document.id == doc.id)
+            .values(
+                sync_state=new_state,
+                sync_attempts=new_attempts,
+                last_sync_at=func.now(),
+            )
+        )
+
+
+async def _bump_message_embedding_sync_attempts(
+    db: AsyncSession,
+    embeddings: list[models.MessageEmbedding],
+) -> None:
+    if not embeddings:
+        return
+
+    for emb in embeddings:
+        new_attempts = emb.sync_attempts + 1
+        new_state = "failed" if new_attempts >= MAX_SYNC_ATTEMPTS else "pending"
+
+        await db.execute(
+            update(models.MessageEmbedding)
+            .where(models.MessageEmbedding.id == emb.id)
+            .values(
+                sync_state=new_state,
+                sync_attempts=new_attempts,
+                last_sync_at=func.now(),
+            )
+        )
+
+
+async def _sync_documents(
+    db: AsyncSession,
+    documents: list[models.Document],
+    external_vector_store: VectorStore,
+) -> tuple[int, int]:
+    """
+    Sync a batch of pending documents to the external vector store.
+
+    Handles three cases for each document:
+    1. Embedding exists in postgres → use it for external upsert
+    2. Embedding missing + need postgres storage → re-embed, write to both stores
+    3. Embedding missing + external-only mode → re-embed, write to external only
+
+    Returns (synced_count, failed_count).
+    """
+    if not documents:
+        return 0, 0
+
+    synced_count = 0
+    failed_count = 0
+
+    # True when using pgvector OR during migration (dual-write to both stores)
+    store_in_postgres = (
+        settings.VECTOR_STORE.TYPE == "pgvector" or not settings.VECTOR_STORE.MIGRATED
+    )
+
+    # Step 1: Re-embed documents missing embeddings in postgres (cases 2 & 3)
+    docs_needing_embed = [
+        doc for doc in documents if cast(list[float] | None, doc.embedding) is None
+    ]
+    freshly_embedded: dict[str, list[float]] = {}
+
+    if docs_needing_embed:
+        try:
+            contents = [doc.content for doc in docs_needing_embed]
+            new_embeddings = await embedding_client.simple_batch_embed(contents)
+
+            if len(new_embeddings) != len(docs_needing_embed):
+                logger.warning(
+                    "Re-embedded %s/%s documents; remaining will be retried",
+                    len(new_embeddings),
+                    len(docs_needing_embed),
+                )
+
+            for doc, emb in zip(docs_needing_embed, new_embeddings, strict=False):
+                freshly_embedded[doc.id] = emb
+
+            # Case 2: Write to postgres if needed
+            if store_in_postgres and freshly_embedded:
+                stmt = (
+                    update(models.Document)
+                    .where(models.Document.id == bindparam("doc_id"))
+                    .values(embedding=bindparam("emb"))
+                )
+                await db.execute(
+                    stmt,
+                    [
+                        {"doc_id": doc_id, "emb": emb}
+                        for doc_id, emb in freshly_embedded.items()
+                    ],
+                )
+        except Exception:
+            logger.exception("Failed to re-embed %s documents", len(docs_needing_embed))
+
+    # Mark documents that failed to get an embedding
+    failed_to_embed = [
+        doc for doc in docs_needing_embed if doc.id not in freshly_embedded
+    ]
+    if failed_to_embed:
+        await _bump_document_sync_attempts(db, failed_to_embed)
+        failed_count += len(failed_to_embed)
+
+    # Step 2: Build vector records and upsert to external store (all cases)
+    by_namespace: dict[str, list[models.Document]] = {}
+    for doc in documents:
+        ns = external_vector_store.get_vector_namespace(
+            "document", doc.workspace_name, doc.observer, doc.observed
+        )
+        by_namespace.setdefault(ns, []).append(doc)
+
+    for namespace, docs in by_namespace.items():
+        docs_to_sync: list[models.Document] = []
+        vector_records: list[VectorRecord] = []
+
+        for doc in docs:
+            # Case 1: use existing embedding, Cases 2&3: use freshly embedded
+            existing = cast(list[float] | None, doc.embedding)
+            embedding = (
+                existing if existing is not None else freshly_embedded.get(doc.id)
+            )
+            if embedding is None:
+                continue
+
+            vector_records.append(
+                VectorRecord(
+                    id=doc.id,
+                    embedding=[float(x) for x in embedding],
+                    metadata={
+                        "workspace_name": doc.workspace_name,
+                        "observer": doc.observer,
+                        "observed": doc.observed,
+                        "session_name": doc.session_name,
+                        "level": doc.level,
+                    },
+                )
+            )
+            docs_to_sync.append(doc)
+
+        if not vector_records:
+            continue
+
+        try:
+            await external_vector_store.upsert_many(namespace, vector_records)
+            await db.execute(
+                update(models.Document)
+                .where(models.Document.id.in_([d.id for d in docs_to_sync]))
+                .values(sync_state="synced", last_sync_at=func.now(), sync_attempts=0)
+            )
+            synced_count += len(docs_to_sync)
+        except Exception:
+            logger.exception("Failed to sync documents to namespace %s", namespace)
+            await _bump_document_sync_attempts(db, docs_to_sync)
+            failed_count += len(docs_to_sync)
+
+    return synced_count, failed_count
+
+
+async def _sync_message_embeddings(
+    db: AsyncSession,
+    embeddings: list[models.MessageEmbedding],
+    external_vector_store: VectorStore,
+) -> tuple[int, int]:
+    """
+    Sync a batch of pending message embeddings to the external vector store.
+
+    Handles three cases for each embedding:
+    1. Embedding exists in postgres → use it for external upsert
+    2. Embedding missing + need postgres storage → re-embed, write to both stores
+    3. Embedding missing + external-only mode → re-embed, write to external only
+
+    Returns (synced_count, failed_count).
+    """
+    if not embeddings:
+        return 0, 0
+
+    synced_count = 0
+    failed_count = 0
+
+    # True when using pgvector OR during migration (dual-write to both stores)
+    store_in_postgres = (
+        settings.VECTOR_STORE.TYPE == "pgvector" or not settings.VECTOR_STORE.MIGRATED
+    )
+
+    # Step 1: Re-embed message embeddings missing vectors in postgres (cases 2 & 3)
+    embs_needing_embed: list[models.MessageEmbedding] = [
+        emb for emb in embeddings if emb.embedding is None
+    ]
+    freshly_embedded: dict[int, list[float]] = {}
+
+    if embs_needing_embed:
+        try:
+            contents = [emb.content for emb in embs_needing_embed]
+            new_embeddings = await embedding_client.simple_batch_embed(contents)
+
+            if len(new_embeddings) != len(embs_needing_embed):
+                logger.warning(
+                    "Re-embedded %s/%s message embeddings; remaining will be retried",
+                    len(new_embeddings),
+                    len(embs_needing_embed),
+                )
+
+            for emb, new_emb in zip(embs_needing_embed, new_embeddings, strict=False):
+                freshly_embedded[emb.id] = new_emb
+
+            # Case 2: Write to postgres if needed
+            if store_in_postgres and freshly_embedded:
+                stmt = (
+                    update(models.MessageEmbedding)
+                    .where(models.MessageEmbedding.id == bindparam("emb_id"))
+                    .values(embedding=bindparam("emb"))
+                )
+                await db.execute(
+                    stmt,
+                    [
+                        {"emb_id": emb_id, "emb": emb}
+                        for emb_id, emb in freshly_embedded.items()
+                    ],
+                )
+        except Exception:
+            logger.exception(
+                "Failed to re-embed %s message embeddings", len(embs_needing_embed)
+            )
+
+    # Mark embeddings that failed to get a vector
+    failed_to_embed: list[models.MessageEmbedding] = [
+        emb for emb in embs_needing_embed if emb.id not in freshly_embedded
+    ]
+    if failed_to_embed:
+        await _bump_message_embedding_sync_attempts(db, failed_to_embed)
+        failed_count += len(failed_to_embed)
+
+    # Step 2: Compute chunk positions for vector IDs
+    # Messages can be split into multiple chunks; we need {message_id}_{chunk_position}
+    #
+    # TODO: chunk_position is computed from MessageEmbedding row ordering by ID, which is
+    # fragile. If rows are deleted and re-created (e.g., during re-embedding), IDs change
+    # and positions shift, potentially causing vector ID mismatches with the external store.
+    # This doesn't break search (metadata.message_id is used, not vector ID), but can leave
+    # stale vectors. Consider either:
+    # 1. Persisting chunk_position in the MessageEmbedding table
+    # 2. Removing MessageEmbedding table entirely if it becomes unnecessary
+    # See: https://github.com/plastic-labs/honcho/issues/XXX
+    message_ids = list({emb.message_id for emb in embeddings})
+    sibling_stmt = (
+        select(models.MessageEmbedding.id, models.MessageEmbedding.message_id)
+        .where(models.MessageEmbedding.message_id.in_(message_ids))
+        .order_by(models.MessageEmbedding.message_id, models.MessageEmbedding.id)
+    )
+    sibling_rows = (await db.execute(sibling_stmt)).all()
+
+    embs_by_message: dict[str, list[int]] = {}
+    for emb_id, msg_id in sibling_rows:
+        embs_by_message.setdefault(msg_id, []).append(emb_id)
+
+    chunk_position: dict[int, int] = {}
+    for emb_ids in embs_by_message.values():
+        for pos, emb_id in enumerate(emb_ids):
+            chunk_position[emb_id] = pos
+
+    # Step 3: Build vector records and upsert to external store (all cases)
+    by_namespace: dict[str, list[models.MessageEmbedding]] = {}
+    for emb in embeddings:
+        ns = external_vector_store.get_vector_namespace("message", emb.workspace_name)
+        by_namespace.setdefault(ns, []).append(emb)
+
+    for namespace, embs in by_namespace.items():
+        embs_to_sync: list[models.MessageEmbedding] = []
+        vector_records: list[VectorRecord] = []
+
+        for emb in embs:
+            # Case 1: use existing embedding, Cases 2&3: use freshly embedded
+            existing = emb.embedding
+            embedding = (
+                existing if existing is not None else freshly_embedded.get(emb.id)
+            )
+            if embedding is None:
+                continue
+
+            vector_records.append(
+                VectorRecord(
+                    id=f"{emb.message_id}_{chunk_position[emb.id]}",
+                    embedding=[float(x) for x in embedding],
+                    metadata={
+                        "message_id": emb.message_id,
+                        "session_name": emb.session_name,
+                        "peer_name": emb.peer_name,
+                    },
+                )
+            )
+            embs_to_sync.append(emb)
+
+        if not vector_records:
+            continue
+
+        try:
+            await external_vector_store.upsert_many(namespace, vector_records)
+            await db.execute(
+                update(models.MessageEmbedding)
+                .where(models.MessageEmbedding.id.in_([e.id for e in embs_to_sync]))
+                .values(sync_state="synced", last_sync_at=func.now(), sync_attempts=0)
+            )
+            synced_count += len(embs_to_sync)
+        except Exception:
+            logger.exception(
+                "Failed to sync message embeddings to namespace %s", namespace
+            )
+            await _bump_message_embedding_sync_attempts(db, embs_to_sync)
+            failed_count += len(embs_to_sync)
+
+    return synced_count, failed_count
+
+
+async def _cleanup_soft_deleted_documents_pgvector(
+    db: AsyncSession,
+    batch_size: int = RECONCILIATION_BATCH_SIZE,
+    older_than_minutes: int = 5,
+) -> int:
+    """
+    Cleanup soft-deleted documents
+    """
+
+    cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
+        minutes=older_than_minutes
+    )
+
+    # Find soft-deleted documents ready for cleanup
+    stmt = (
+        select(models.Document.id)
+        .where(models.Document.deleted_at.is_not(None))
+        .where(models.Document.deleted_at < cutoff)
+        .limit(batch_size)
+        .with_for_update(skip_locked=True)
+    )
+    result = await db.execute(stmt)
+    doc_ids = [row[0] for row in result.all()]
+
+    if not doc_ids:
+        return 0
+
+    # Hard delete directly (no vector store cleanup needed in pgvector mode)
+    await db.execute(delete(models.Document).where(models.Document.id.in_(doc_ids)))
+    logger.debug(f"Cleaned up {len(doc_ids)} soft-deleted documents (pgvector mode)")
+    return len(doc_ids)
+
+
+async def _reconcile_documents_batch(
+    external_vector_store: VectorStore,
+    metrics: ReconciliationMetrics,
+) -> bool:
+    """
+    Reconcile a single batch of documents.
+
+    Returns True if work was done, False otherwise.
+    """
+    async with tracked_db("reconciliation_docs") as db:
+        docs = await _get_documents_needing_sync(db)
+        if not docs:
+            return False
+
+        synced, failed = await _sync_documents(db, docs, external_vector_store)
+        metrics.documents_synced += synced
+        metrics.documents_failed += failed
+        await db.commit()
+        return True
+
+
+async def _reconcile_message_embeddings_batch(
+    external_vector_store: VectorStore,
+    metrics: ReconciliationMetrics,
+) -> bool:
+    """
+    Reconcile a single batch of message embeddings.
+
+    Returns True if work was done, False otherwise.
+    """
+    async with tracked_db("reconciliation_embs") as db:
+        embs = await _get_message_embeddings_needing_sync(db)
+        if not embs:
+            return False
+
+        try:
+            synced, failed = await _sync_message_embeddings(
+                db, embs, external_vector_store
+            )
+        except Exception:
+            logger.exception(
+                "Message embedding reconciliation failed for %s embeddings",
+                len(embs),
+            )
+            await _bump_message_embedding_sync_attempts(db, embs)
+            synced = 0
+            failed = len(embs)
+
+        metrics.message_embeddings_synced += synced
+        metrics.message_embeddings_failed += failed
+        await db.commit()
+        return True
+
+
+async def _cleanup_documents_batch(
+    external_vector_store: VectorStore,
+    metrics: ReconciliationMetrics,
+) -> bool:
+    """
+    Clean up a single batch of soft-deleted documents.
+
+    Returns True if work was done, False otherwise.
+    """
+    from src.crud.document import cleanup_soft_deleted_documents
+
+    async with tracked_db("reconciliation_cleanup") as db:
+        cleaned = await cleanup_soft_deleted_documents(
+            db,
+            external_vector_store,
+            batch_size=RECONCILIATION_BATCH_SIZE,
+        )
+        if not cleaned:
+            return False
+
+        metrics.documents_cleaned += cleaned
+        await db.commit()
+        return True
+
+
+async def _cleanup_pgvector_batch(
+    metrics: ReconciliationMetrics,
+) -> bool:
+    """
+    Clean up a single batch of soft-deleted documents in pgvector-only mode.
+
+    Returns True if work was done, False otherwise.
+    """
+    async with tracked_db("reconciliation_pgvector_cleanup") as db:
+        cleaned = await _cleanup_soft_deleted_documents_pgvector(
+            db, batch_size=RECONCILIATION_BATCH_SIZE
+        )
+        if not cleaned:
+            return False
+
+        metrics.documents_cleaned += cleaned
+        await db.commit()
+        return True
+
+
+async def run_vector_reconciliation_cycle() -> ReconciliationMetrics:
+    """
+    Run a complete reconciliation cycle.
+
+    Runs a rolling sweep to reconcile missing vectors and clean up soft deletes.
+    Uses batching and FOR UPDATE SKIP LOCKED for safe concurrent operation.
+    Each batch operation uses its own database session to avoid holding
+    connections open for the entire cycle duration.
+
+    Returns metrics about what was synced.
+    """
+    metrics = ReconciliationMetrics()
+    external_vector_store = get_external_vector_store()
+    deadline = time.monotonic() + RECONCILIATION_TIME_BUDGET_SECONDS
+
+    # If no external vector store (pgvector mode), only clean up soft-deleted documents
+    if external_vector_store is None:
+        while time.monotonic() < deadline:
+            did_work = await _cleanup_pgvector_batch(metrics)
+            if not did_work:
+                break
+        logger.info("Vector reconciliation cycle completed (pgvector mode)")
+        return metrics
+
+    # External vector store mode - reconcile documents, embeddings, and cleanup
+    while time.monotonic() < deadline:
+        # Reconcile documents
+        docs_work = await _reconcile_documents_batch(external_vector_store, metrics)
+
+        if time.monotonic() >= deadline:
+            break
+
+        # Reconcile message embeddings
+        embs_work = await _reconcile_message_embeddings_batch(
+            external_vector_store, metrics
+        )
+
+        if time.monotonic() >= deadline:
+            break
+
+        # Clean up soft-deleted documents
+        cleanup_work = await _cleanup_documents_batch(external_vector_store, metrics)
+
+        # Continue only if any operation did work
+        if not (docs_work or embs_work or cleanup_work):
+            logger.debug("No work done, breaking reconciliation loop")
+            break
+
+    logger.info("Vector reconciliation cycle completed")
+    return metrics

src/schemas.py

@@ -27,6 +27,13 @@ class DreamType(str, Enum):
     OMNI = "omni"
 
 
+class ReconcilerType(str, Enum):
+    """Types of reconciler tasks that can be performed."""
+
+    SYNC_VECTORS = "sync_vectors"
+    CLEANUP_QUEUE = "cleanup_queue"
+
+
 class ReasoningConfiguration(BaseModel):
     enabled: bool | None = Field(
         default=None,

src/utils/queue_payload.py

@@ -3,7 +3,7 @@ from typing import Any, Literal
 
 from pydantic import BaseModel, ConfigDict
 
-from src.schemas import DreamType, ResolvedConfiguration
+from src.schemas import DreamType, ReconcilerType, ResolvedConfiguration
 
 
 class BasePayload(BaseModel):
@@ -67,6 +67,13 @@ class DeletionPayload(BasePayload):
     resource_id: str
 
 
+class ReconcilerPayload(BasePayload):
+    """Payload for reconciler tasks (vector sync, queue cleanup, self-healing)."""
+
+    task_type: Literal["reconciler"] = "reconciler"
+    reconciler_type: ReconcilerType
+
+
 def create_webhook_payload(
     event_type: str,
     data: dict[str, Any],

src/utils/search.py

@@ -18,6 +18,7 @@ from src.exceptions import ValidationException
 from src.models import session_peers_table
 from src.utils.filter import apply_filter
 from src.utils.formatting import ILIKE_ESCAPE_CHAR, escape_ilike_pattern
+from src.vector_store import get_external_vector_store
 
 T = TypeVar("T")
 
@@ -67,17 +68,19 @@ def reciprocal_rank_fusion(*ranked_lists: list[T], k: int = 60, limit: int) -> l
 async def _semantic_search(
     db: AsyncSession,
     query: str,
-    stmt: Select[tuple[models.Message]],
+    workspace_name: str,
     limit: int,
+    filters: dict[str, Any] | None = None,
 ) -> list[models.Message]:
     """
-    Perform semantic search using message embeddings.
+    Perform semantic search using external vector store for message embeddings.
 
     Args:
         db: Database session
         query: Search query
-        stmt: Base SQL query conditions
+        workspace_name: Name of the workspace to search in
         limit: Maximum number of results to return
+        filters: Optional filters to apply at vector store level (supports: session_id, peer_id)
 
     Returns:
         list of messages ordered by semantic similarity
@@ -89,16 +92,153 @@ async def _semantic_search(
             f"Query exceeds maximum token limit of {settings.MAX_EMBEDDING_TOKENS}."
         ) from e
 
-    # Use cosine distance for semantic search on MessageEmbedding table
-    semantic_query = stmt.join(
-        models.MessageEmbedding,
-        models.Message.public_id == models.MessageEmbedding.message_id,
-    ).order_by(models.MessageEmbedding.embedding.cosine_distance(embedding_query))
+    # Query Postgres / pgvector directly
+    if settings.EMBED_MESSAGES and (
+        settings.VECTOR_STORE.TYPE == "pgvector" or not settings.VECTOR_STORE.MIGRATED
+    ):
+        # Join message_embeddings with messages to get full message objects
+        distance_expr = models.MessageEmbedding.embedding.cosine_distance(
+            embedding_query
+        )
 
-    semantic_query = semantic_query.limit(limit)
+        stmt = (
+            select(models.Message)
+            .join(
+                models.MessageEmbedding,
+                models.Message.public_id == models.MessageEmbedding.message_id,
+            )
+            .where(models.MessageEmbedding.embedding.isnot(None))
+            .where(models.MessageEmbedding.workspace_name == workspace_name)
+        )
+
+        # Apply all additional filters using the standard filter utility
+        # filters dict uses external names (session_id, peer_id) which apply_filter will map
+        # to internal column names (session_name, peer_name)
+        if filters:
+            # Create a copy with workspace added
+            internal_filters = filters.copy()
+            internal_filters["workspace_id"] = workspace_name
+            stmt = apply_filter(stmt, models.Message, internal_filters)
+
+        # Order by cosine distance and limit
+        stmt = stmt.order_by(distance_expr).limit(limit)
+
+        result = await db.execute(stmt)
+        return list(result.scalars().all())
+
+    # FALLBACK: Use external vector store (Turbopuffer, LanceDB)
+    external_vector_store = get_external_vector_store()
+    if external_vector_store is None:
+        return []
+
+    namespace = external_vector_store.get_vector_namespace("message", workspace_name)
+
+    # Build vector store filters from the provided filters
+    vector_filters: dict[str, Any] = {}
+    if filters:
+        # Map external filter keys to vector store metadata keys
+        if "session_id" in filters:
+            vector_filters["session_name"] = filters["session_id"]
+        if "peer_id" in filters:
+            vector_filters["peer_name"] = filters["peer_id"]
+
+    # Query external vector store for similar message embeddings
+    # Since all filters are applied at the vector store level, we don't need to oversample
+    vector_results = await external_vector_store.query(
+        namespace,
+        embedding_query,
+        top_k=limit,
+        filters=vector_filters if vector_filters else None,
+    )
+
+    if not vector_results:
+        return []
+
+    # Extract message IDs from vector metadata
+    # Use dict to deduplicate while preserving order (dict keys maintain insertion order in Python 3.7+)
+    seen_message_ids: dict[str, None] = {}
+
+    for result in vector_results:
+        message_id = result.metadata.get("message_id")
+        if message_id and message_id not in seen_message_ids:
+            seen_message_ids[message_id] = None
+
+    message_ids = list(seen_message_ids.keys())
+
+    # Fetch messages from database by the IDs from vector search and reapply filters
+    semantic_query = select(models.Message).where(
+        models.Message.public_id.in_(message_ids)
+    )
+    semantic_query = apply_filter(semantic_query, models.Message, filters)
 
     result = await db.execute(semantic_query)
-    return list(result.scalars().all())
+    messages = {msg.public_id: msg for msg in result.scalars().all()}
+
+    # Return messages in order of similarity (preserving vector store order)
+    ordered_messages: list[models.Message] = []
+    for msg_id in message_ids:
+        if msg_id in messages:
+            ordered_messages.append(messages[msg_id])
+
+    return ordered_messages
+
+
+async def _filter_by_peer_perspective(
+    db: AsyncSession,
+    messages: list[models.Message],
+    workspace_name: str,
+    peer_name: str,
+) -> list[models.Message]:
+    """
+    Filter messages by peer perspective (temporal session membership).
+
+    Only keeps messages from sessions where the peer was a member at the time
+    the message was created (between joined_at and left_at).
+
+    Args:
+        db: Database session
+        messages: List of messages to filter
+        workspace_name: Name of the workspace
+        peer_name: Name of the peer whose perspective to use
+
+    Returns:
+        Filtered list of messages
+    """
+    if not messages:
+        return []
+
+    # Get all session memberships for this peer in this workspace
+    session_memberships_query = (
+        select(session_peers_table)
+        .where(session_peers_table.c.workspace_name == workspace_name)
+        .where(session_peers_table.c.peer_name == peer_name)
+    )
+    result = await db.execute(session_memberships_query)
+    memberships = result.all()
+
+    # Build a lookup of session -> time windows
+    session_windows: dict[str, list[tuple[Any, Any]]] = {}
+    for membership in memberships:
+        session_name = membership.session_name
+        if session_name not in session_windows:
+            session_windows[session_name] = []
+        session_windows[session_name].append((membership.joined_at, membership.left_at))
+
+    # Filter messages
+    filtered_messages: list[models.Message] = []
+    for msg in messages:
+        if msg.session_name not in session_windows:
+            continue
+
+        # Check if message was created during any of the peer's active windows in this session
+        for joined_at, left_at in session_windows[msg.session_name]:
+            if msg.created_at >= joined_at and (
+                left_at is None or msg.created_at <= left_at
+            ):
+                filtered_messages.append(msg)
+                break  # Don't add the same message twice
+
+    return filtered_messages
 
 
 async def _fulltext_search(
@@ -183,9 +323,9 @@ async def search(
     Args:
         db: Database session
         query: Search query to match against message content
-        filters: Optional filters to scope search. Special filter 'peer_perspective' will search
-                across all messages from sessions that the peer is/was a member of, filtered
-                by the time window when they were actually in the session.
+        filters: Optional filters to scope search (must include workspace_id for semantic search).
+            Special filter 'peer_perspective' will search across all messages from sessions that the peer is/was a member of,
+            filtered by the time window when they were actually in the session.
         limit: Maximum number of results to return
 
     Returns:
@@ -198,8 +338,9 @@ async def search(
     stmt = select(models.Message)
 
     # Handle special peer_perspective filter
+    peer_perspective_name: str | None = None
     if filters and "peer_perspective" in filters:
-        peer_name = filters["peer_perspective"]
+        peer_perspective_name = filters["peer_perspective"]
         # Remove from filters dict so apply_filter doesn't try to handle it
         filters = {k: v for k, v in filters.items() if k != "peer_perspective"}
         # Safety: peer_perspective must be scoped to a workspace
@@ -223,19 +364,34 @@ async def search(
                     models.Message.created_at <= session_peers_table.c.left_at,
                 ),
             ),
-        ).where(session_peers_table.c.peer_name == peer_name)
+        ).where(session_peers_table.c.peer_name == peer_perspective_name)
 
     stmt = apply_filter(stmt, models.Message, filters)
 
     search_results: list[list[models.Message]] = []
 
-    # Perform semantic search if enabled
-    if settings.EMBED_MESSAGES:
-        # Get more results for fusion
-        semantic_limit = limit * 2
+    # Perform semantic search if enabled and we have workspace context
+    # workspace_id is required for semantic search to determine the vector namespace
+    workspace_name: str | None = filters.get("workspace_id") if filters else None
+    if settings.EMBED_MESSAGES and isinstance(workspace_name, str):
+        # Type narrowing: workspace_name is guaranteed to be str in this block
+        # Get more results for fusion (increase if peer_perspective filtering is applied post-search)
+        semantic_limit = limit * 4 if peer_perspective_name else limit * 2
         semantic_results = await _semantic_search(
-            db=db, query=query, stmt=stmt, limit=semantic_limit
+            db=db,
+            query=query,
+            workspace_name=workspace_name,
+            limit=semantic_limit,
+            filters=filters,
         )
+
+        # Apply peer_perspective filtering to semantic results if needed
+        # Vector store can't handle temporal filtering (joined_at/left_at), so filter post-search
+        if peer_perspective_name:
+            semantic_results = await _filter_by_peer_perspective(
+                db, semantic_results, workspace_name, peer_perspective_name
+            )
+
         search_results.append(semantic_results)
 
     # Perform full-text search

src/utils/types.py

@@ -11,5 +11,8 @@ class GetOrCreateResult(NamedTuple, Generic[T]):
 
 
 SupportedProviders = Literal["anthropic", "openai", "google", "groq", "custom", "vllm"]
-TaskType = Literal["webhook", "summary", "representation", "dream", "deletion"]
+TaskType = Literal[
+    "webhook", "summary", "representation", "dream", "deletion", "reconciler"
+]
+VectorSyncState = Literal["synced", "pending", "failed"]
 DocumentLevel = Literal["explicit", "deductive", "inductive", "contradiction"]

src/utils/work_unit.py

@@ -9,7 +9,7 @@ class ParsedWorkUnit(BaseModel):
     """Parsed work unit components."""
 
     task_type: str
-    workspace_name: str
+    workspace_name: str | None
     session_name: str | None
     observer: str | None
     observed: str | None
@@ -64,6 +64,12 @@ def construct_work_unit_key(
             )
         return f"deletion:{workspace_name}:{deletion_type}:{resource_id}"
 
+    if task_type == "reconciler":
+        reconciler_type = payload.get("reconciler_type")
+        if not reconciler_type:
+            raise ValueError("reconciler_type is required for reconciler tasks")
+        return f"reconciler:{reconciler_type}"
+
     raise ValueError(f"Invalid task type: {task_type}")
 
 
@@ -136,4 +142,17 @@ def parse_work_unit_key(work_unit_key: str) -> ParsedWorkUnit:
             observed=None,
         )
 
+    if task_type == "reconciler":
+        if len(parts) != 2:
+            raise ValueError(
+                f"Invalid work_unit_key format for task_type {task_type}: {work_unit_key}"
+            )
+        return ParsedWorkUnit(
+            task_type=task_type,
+            workspace_name=None,
+            session_name=None,
+            observer=None,
+            observed=None,
+        )
+
     raise ValueError(f"Invalid task type in work_unit_key: {task_type}")

src/vector_store/__init__.py

@@ -0,0 +1,258 @@
+"""
+Vector store abstraction layer for Honcho.
+"""
+
+import base64
+import hashlib
+from abc import ABC, abstractmethod
+from functools import cache
+from typing import Any, ClassVar, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from src.config import settings
+
+
+def _hash_namespace_components(*parts: str) -> str:
+    """
+    Hash namespace components to create a fixed-length, valid namespace suffix.
+
+    Turbopuffer requires namespaces to match [A-Za-z0-9-_.]{1,128}.
+    This function hashes the variable parts (workspace, observer, observed)
+    to ensure the namespace fits within length limits and uses only valid chars.
+
+    Returns:
+        A 43-character base64url-encoded SHA-256 hash (no padding).
+        (SHA-256 = 32 bytes, base64 = ceil(32 * 4 / 3) = 43 chars without padding)
+    """
+    combined = ".".join(parts)
+    hash_bytes = hashlib.sha256(combined.encode("utf-8")).digest()
+    return base64.urlsafe_b64encode(hash_bytes).decode("ascii").rstrip("=")
+
+
+class VectorRecord(BaseModel):
+    """A single vector record to be stored in the vector store."""
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid", frozen=True)
+
+    id: str
+    embedding: list[float]
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class VectorQueryResult(BaseModel):
+    """A single result from a vector query."""
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid", frozen=True)
+
+    id: str
+    score: float  # Distance/similarity score (lower = more similar for cosine distance)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class VectorUpsertResult(BaseModel):
+    """Result for a vector upsert operation."""
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(
+        extra="forbid",
+        frozen=True,
+    )
+
+    ok: bool
+
+
+class VectorStore(ABC):
+    """
+    Abstract base class for vector store implementations.
+
+    All vector operations are namespace-scoped. Namespaces are generated via
+    get_vector_namespace() which hashes workspace/peer names to ensure:
+    - Total length fits within Turbopuffer's 128 char limit
+    - Only valid characters [A-Za-z0-9-_.] are used
+
+    Namespace format: {prefix}.{type}.{hash}
+    - Document embeddings: {prefix}.doc.{hash}
+    - Message embeddings: {prefix}.msg.{hash}
+    """
+
+    namespace_prefix: str
+
+    def __init__(self):
+        """
+        Initialize the vector store.
+        """
+        self.namespace_prefix = settings.VECTOR_STORE.NAMESPACE
+
+    # === Namespace helpers ===
+    def get_vector_namespace(
+        self,
+        namespace_type: Literal["document", "message"],
+        workspace_name: str,
+        observer: str | None = None,
+        observed: str | None = None,
+    ) -> str:
+        """
+        Get the namespace for document or message embeddings.
+
+        Args:
+            namespace_type: "document" or "message"
+            workspace_name: Name of the workspace
+            observer: Name of the observing peer (document only)
+            observed: Name of the observed peer (document only)
+
+        Returns:
+            Namespace string in format:
+            - document: {prefix}.doc.{hash}
+            - message: {prefix}.msg.{hash}
+            where hash is derived from the workspace/peer names.
+        """
+        if namespace_type == "document":
+            if observer is None or observed is None:
+                raise ValueError(
+                    "observer and observed are required for document namespaces"
+                )
+            hash_suffix = _hash_namespace_components(workspace_name, observer, observed)
+            return f"{self.namespace_prefix}.doc.{hash_suffix}"
+        elif namespace_type == "message":
+            hash_suffix = _hash_namespace_components(workspace_name)
+            return f"{self.namespace_prefix}.msg.{hash_suffix}"
+
+    # === Core operations ===
+    @abstractmethod
+    async def upsert_many(
+        self,
+        namespace: str,
+        vectors: list[VectorRecord],
+    ) -> VectorUpsertResult:
+        """
+        Upsert multiple vectors into the store.
+
+        Args:
+            namespace: The namespace to store the vectors in
+            vectors: List of VectorRecord objects to upsert
+
+        Returns:
+            Result describing primary/secondary outcomes.
+        """
+        ...
+
+    @abstractmethod
+    async def query(
+        self,
+        namespace: str,
+        embedding: list[float],
+        *,
+        top_k: int = 10,
+        filters: dict[str, Any] | None = None,
+        max_distance: float | None = None,
+    ) -> list[VectorQueryResult]:
+        """
+        Query for similar vectors.
+
+        Args:
+            namespace: The namespace to query
+            embedding: The query embedding vector
+            top_k: Maximum number of results to return
+            filters: Optional metadata filters
+            max_distance: Optional maximum distance threshold (cosine distance)
+
+        Returns:
+            List of VectorQueryResult objects, ordered by similarity (most similar first)
+        """
+        ...
+
+    @abstractmethod
+    async def delete_many(self, namespace: str, ids: list[str]) -> None:
+        """
+        Delete multiple vectors from the store.
+
+        Args:
+            namespace: The namespace containing the vectors
+            ids: List of vector identifiers to delete
+        """
+        ...
+
+    @abstractmethod
+    async def delete_namespace(self, namespace: str) -> None:
+        """
+        Delete an entire namespace and all its vectors.
+
+        Args:
+            namespace: The namespace to delete
+        """
+        ...
+
+    @abstractmethod
+    async def close(self) -> None:
+        """
+        Close any open connections and release resources.
+
+        Subclasses should override this if they maintain persistent connections.
+        """
+        ...
+
+
+# Import implementations after base classes are defined to avoid circular imports
+from src.vector_store.lancedb import LanceDBVectorStore  # noqa: E402
+from src.vector_store.turbopuffer import TurbopufferVectorStore  # noqa: E402
+from src.vector_store.utils import upsert_with_retry  # noqa: E402
+
+
+def _create_store_by_type(store_type: str) -> VectorStore:
+    """Create a vector store instance by type name."""
+    if store_type == "turbopuffer":
+        return TurbopufferVectorStore()
+    elif store_type == "lancedb":
+        return LanceDBVectorStore()
+    else:
+        raise ValueError(f"Unknown vector store type: {store_type}")
+
+
+@cache
+def get_external_vector_store() -> VectorStore | None:
+    """
+    Get the configured external vector store instance (singleton).
+
+    Returns None if TYPE='pgvector' since pgvector operations happen via ORM directly.
+    External vector stores include Turbopuffer and LanceDB.
+
+    Returns:
+        The external vector store instance, or None if using pgvector (ORM handles it).
+
+    Raises:
+        ValueError: If the configured vector store type is invalid.
+    """
+    if settings.VECTOR_STORE.TYPE == "pgvector":
+        return None
+    return _create_store_by_type(settings.VECTOR_STORE.TYPE)
+
+
+async def close_external_vector_store() -> None:
+    """
+    Close the external vector store and release resources.
+
+    Call this during application shutdown to cleanly close connections.
+    After calling this, you must call get_external_vector_store.cache_clear() if you
+    want to create a new instance.
+    """
+    # Check if an instance was ever created
+    if (
+        get_external_vector_store.cache_info().hits > 0
+        or get_external_vector_store.cache_info().misses > 0
+    ):
+        store = get_external_vector_store()
+        if store is not None:
+            await store.close()
+        get_external_vector_store.cache_clear()
+
+
+__all__ = [
+    "VectorStore",
+    "VectorRecord",
+    "VectorQueryResult",
+    "VectorUpsertResult",
+    "get_external_vector_store",
+    "close_external_vector_store",
+    "upsert_with_retry",
+    "_hash_namespace_components",
+]

src/vector_store/lancedb.py

@@ -0,0 +1,371 @@
+"""
+LanceDB vector store implementation.
+
+This module provides a LanceDB-based implementation of the VectorStore interface.
+"""
+
+import asyncio
+import logging
+import re
+from collections.abc import Sequence
+from typing import Any, cast
+
+import lancedb
+import pyarrow as pa
+from lancedb import AsyncConnection, AsyncTable
+
+from src.config import settings
+from src.exceptions import VectorStoreError
+
+from . import VectorQueryResult, VectorRecord, VectorStore, VectorUpsertResult
+
+logger = logging.getLogger(__name__)
+
+# Pattern for valid SQL identifiers (alphanumeric + underscore, not starting with digit)
+_VALID_IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+
+# Schema for LanceDB tables
+# id: string, vector: fixed_size_list of float32 (1536 dimensions for OpenAI embeddings)
+# Additional metadata columns are added dynamically
+
+# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownParameterType=false
+
+
+class LanceDBVectorStore(VectorStore):
+    """
+    LanceDB implementation of the VectorStore interface.
+
+    Uses LanceDB's async embedded mode for local vector storage.
+    Each namespace corresponds to a LanceDB table.
+    """
+
+    _db: AsyncConnection | None = None
+    _db_path: str
+    _db_lock: asyncio.Lock
+
+    def __init__(self) -> None:
+        """Initialize the LanceDB vector store."""
+        super().__init__()
+        self._db_path = settings.VECTOR_STORE.LANCEDB_PATH
+        self._db = None
+        self._db_lock = asyncio.Lock()
+
+    async def _get_db(self) -> AsyncConnection:
+        """Get or create the async database connection (asyncio-safe)."""
+        if self._db is not None:
+            return self._db
+
+        async with self._db_lock:
+            # Double-check after acquiring lock
+            if self._db is None:
+                self._db = await lancedb.connect_async(self._db_path)
+        return self._db
+
+    async def _get_table(self, namespace: str) -> AsyncTable | None:
+        """Get a table if it exists, otherwise return None."""
+        db = await self._get_db()
+        table_names = await db.table_names()
+        if namespace in table_names:
+            return await db.open_table(namespace)
+        return None
+
+    async def _get_or_create_table(
+        self,
+        namespace: str,
+    ) -> AsyncTable:
+        """
+        Get existing table or create if not exists.
+
+        Args:
+            namespace: Table name (namespace)
+
+        Returns:
+            LanceDB async table
+        """
+        db = await self._get_db()
+        table_names = await db.table_names()
+        if namespace in table_names:
+            return await db.open_table(namespace)
+
+        # Create empty table with base schema
+        # Handle race condition: another worker may have created the table
+        # between our check and create_table call
+        fields: list[pa.Field] = [
+            pa.field("id", pa.string()),
+            pa.field(
+                "vector", pa.list_(pa.float32(), settings.VECTOR_STORE.DIMENSIONS)
+            ),
+        ]
+        fields.extend(self._metadata_fields_for_namespace(namespace))
+        schema = pa.schema(fields)
+        try:
+            table = await db.create_table(namespace, schema=schema)  # pyright: ignore[reportUnknownArgumentType]
+            return table
+        except Exception:
+            # Table may have been created by another worker, try to open it
+            return await db.open_table(namespace)
+
+    def _metadata_fields_for_namespace(self, namespace: str) -> list[pa.Field]:
+        """
+        Infer standard metadata columns based on namespace structure.
+
+        Namespaces use format: {prefix}.{type}.{hash}
+            - Documents: {prefix}.doc.{hash}
+            - Messages:  {prefix}.msg.{hash}
+        """
+        parts = namespace.split(".")
+        if len(parts) < 3:
+            return []
+
+        # Second-to-last part indicates the type (doc or msg)
+        ns_type = parts[-2]
+
+        if ns_type == "msg":
+            return [
+                pa.field("message_id", pa.string(), nullable=True),
+                pa.field("session_name", pa.string(), nullable=True),
+                pa.field("peer_name", pa.string(), nullable=True),
+            ]
+
+        if ns_type == "doc":
+            return [
+                pa.field("workspace_name", pa.string(), nullable=True),
+                pa.field("observer", pa.string(), nullable=True),
+                pa.field("observed", pa.string(), nullable=True),
+                pa.field("session_name", pa.string(), nullable=True),
+                pa.field("level", pa.string(), nullable=True),
+            ]
+
+        return []
+
+    def _row_to_dict(self, vector: VectorRecord) -> dict[str, Any]:
+        """Convert a VectorRecord to a dict for LanceDB."""
+        row: dict[str, Any] = {
+            "id": vector.id,
+            "vector": vector.embedding,
+        }
+        # Add metadata fields
+        if vector.metadata:
+            reserved_keys = {"id", "vector", "_distance"}
+            for key in vector.metadata:
+                if key not in reserved_keys:
+                    row[key] = vector.metadata[key]
+        return row
+
+    async def upsert_many(
+        self,
+        namespace: str,
+        vectors: list[VectorRecord],
+    ) -> VectorUpsertResult:
+        """
+        Upsert multiple vectors into LanceDB.
+
+        Args:
+            namespace: The namespace (table) to store the vectors in
+            vectors: List of VectorRecord objects to upsert
+        """
+        if not vectors:
+            return VectorUpsertResult(ok=True)
+
+        try:
+            rows = [self._row_to_dict(v) for v in vectors]
+            table = await self._get_or_create_table(namespace)
+
+            # Use merge_insert for upsert behavior
+            await (
+                table.merge_insert("id")
+                .when_matched_update_all()
+                .when_not_matched_insert_all()
+                .execute(rows)
+            )
+
+            logger.debug(f"Upserted {len(vectors)} vectors to namespace {namespace}")
+            return VectorUpsertResult(ok=True)
+        except Exception as e:
+            logger.exception(
+                f"Failed to upsert {len(vectors)} vectors to namespace {namespace}"
+            )
+            raise VectorStoreError(
+                f"Failed to upsert {len(vectors)} vectors to namespace {namespace}"
+            ) from e
+
+    async def query(
+        self,
+        namespace: str,
+        embedding: list[float],
+        *,
+        top_k: int = 10,
+        filters: dict[str, Any] | None = None,
+        max_distance: float | None = None,
+    ) -> list[VectorQueryResult]:
+        """
+        Query for similar vectors in LanceDB.
+
+        Args:
+            namespace: The namespace (table) to query
+            embedding: The query embedding vector
+            top_k: Maximum number of results to return
+            filters: Optional metadata filters
+            max_distance: Optional maximum distance threshold (cosine distance)
+
+        Returns:
+            List of VectorQueryResult objects, ordered by similarity (most similar first)
+        """
+        table = await self._get_table(namespace)
+        if table is None:
+            logger.debug(f"Table {namespace} does not exist, returning empty results")
+            return []
+
+        try:
+            # Build query
+            query = table.vector_search(embedding).distance_type("cosine").limit(top_k)
+
+            # Apply filters if provided
+            if filters:
+                where_clause = self._build_where_clause(filters)
+                if where_clause:
+                    query = query.where(where_clause)
+
+            # Execute query
+            # LanceDB async API returns list of dicts with incomplete type annotations
+            results = cast(list[dict[str, Any]], await query.to_list())
+
+            # Convert to VectorQueryResult objects
+            query_results: list[VectorQueryResult] = []
+            for row in results:
+                dist = float(row.get("_distance", 0.0))
+
+                # Filter by max_distance if specified
+                if max_distance is not None and dist > max_distance:
+                    continue
+
+                # Extract metadata (everything except id, vector, _distance)
+                metadata: dict[str, Any] = {
+                    k: v
+                    for k, v in row.items()
+                    if k not in ("id", "vector", "_distance")
+                }
+
+                query_results.append(
+                    VectorQueryResult(
+                        id=str(row["id"]),
+                        score=dist,
+                        metadata=metadata,
+                    )
+                )
+
+            logger.debug(
+                f"Query returned {len(query_results)} results from namespace {namespace}"
+            )
+            return query_results
+
+        except Exception:
+            logger.exception(f"Failed to query namespace {namespace}")
+            raise
+
+    def _build_where_clause(self, filters: dict[str, Any]) -> str | None:
+        """
+        Convert a filter dict to SQL WHERE clause syntax.
+
+        Supports filter formats:
+        - {"key": "value"} -> key = 'value'
+        - {"key": {"in": ["a", "b"]}} -> key IN ('a', 'b')
+
+        Args:
+            filters: Dictionary of attribute -> value filters
+
+        Returns:
+            SQL WHERE clause string or None if no filters
+
+        Raises:
+            ValueError: If a filter key is not a valid SQL identifier
+        """
+        if not filters:
+            return None
+
+        conditions: list[str] = []
+        for key, value in filters.items():
+            # Validate key is a safe SQL identifier to prevent injection
+            if not _VALID_IDENTIFIER_PATTERN.match(key):
+                raise ValueError(f"Invalid filter key: {key!r}")
+
+            # Check if value is a dict with "in" operator
+            if isinstance(value, dict) and "in" in value:
+                # IN clause for list membership
+                in_values = cast(Sequence[Any], value["in"])
+                if in_values:
+                    escaped_values = [
+                        f"'{str(v).replace(chr(39), chr(39) + chr(39))}'"
+                        if isinstance(v, str)
+                        else str(v)
+                        for v in in_values
+                    ]
+                    conditions.append(f"{key} IN ({', '.join(escaped_values)})")
+            # Handle string values with proper quoting
+            elif isinstance(value, str):
+                # Escape single quotes in the value
+                escaped_value = value.replace("'", "''")
+                conditions.append(f"{key} = '{escaped_value}'")
+            elif isinstance(value, bool):
+                conditions.append(f"{key} = {str(value).lower()}")
+            elif value is None:
+                conditions.append(f"{key} IS NULL")
+            else:
+                conditions.append(f"{key} = {value}")
+
+        return " AND ".join(conditions) if conditions else None
+
+    async def delete_many(self, namespace: str, ids: list[str]) -> None:
+        """
+        Delete multiple vectors from LanceDB.
+
+        Args:
+            namespace: The namespace (table) containing the vectors
+            ids: List of vector identifiers to delete
+        """
+        if not ids:
+            return
+
+        table = await self._get_table(namespace)
+        if table is None:
+            logger.debug(f"Table {namespace} does not exist, nothing to delete")
+            return
+
+        try:
+            # Build IN clause with properly escaped IDs
+            escaped_ids = [f"'{id.replace(chr(39), chr(39) + chr(39))}'" for id in ids]
+            in_clause = ", ".join(escaped_ids)
+            await table.delete(f"id IN ({in_clause})")
+            logger.debug(f"Deleted {len(ids)} vectors from namespace {namespace}")
+        except Exception:
+            logger.exception(
+                f"Failed to delete {len(ids)} vectors from namespace {namespace}"
+            )
+            raise
+
+    async def delete_namespace(self, namespace: str) -> None:
+        """
+        Delete an entire namespace (table) and all its vectors from LanceDB.
+
+        Args:
+            namespace: The namespace (table) to delete
+        """
+        try:
+            db = await self._get_db()
+            table_names = await db.table_names()
+            if namespace in table_names:
+                await db.drop_table(namespace)
+            else:
+                logger.debug(f"Namespace {namespace} does not exist, nothing to delete")
+        except Exception:
+            logger.exception(f"Failed to delete namespace {namespace}")
+            raise
+
+    async def close(self) -> None:
+        """Close the LanceDB connection and release resources."""
+        if self._db is not None:
+            # AsyncConnection provides an explicit close() method (synchronous)
+            # which we invoke to ensure proper cleanup of resources
+            self._db.close()
+            self._db = None
+            logger.debug("LanceDB connection closed")

src/vector_store/turbopuffer.py

@@ -0,0 +1,278 @@
+"""
+Turbopuffer vector store implementation.
+
+This module provides a Turbopuffer-based implementation of the VectorStore interface.
+"""
+
+import logging
+from collections.abc import Sequence
+from typing import Any, Literal, cast
+
+from turbopuffer import AsyncTurbopuffer, NotFoundError
+from turbopuffer.lib.namespace import AsyncNamespace
+from turbopuffer.types import Filter
+
+from src.config import settings
+
+from . import VectorQueryResult, VectorRecord, VectorStore, VectorUpsertResult
+
+logger = logging.getLogger(__name__)
+
+# Type aliases for Turbopuffer's filter formats
+EqFilter = tuple[str, Literal["Eq"], Any]
+InFilter = tuple[str, Literal["In"], Sequence[Any]]
+AndFilter = tuple[Literal["And"], Sequence[Filter]]
+
+DISTANCE_METRIC = "cosine_distance"
+
+
+class TurbopufferVectorStore(VectorStore):
+    """
+    Turbopuffer implementation of the VectorStore interface.
+
+    Namespaces are generated via get_vector_namespace() which hashes
+    the variable components to fit Turbopuffer's [A-Za-z0-9-_.]{1,128} limit.
+    """
+
+    tpuf: AsyncTurbopuffer
+
+    def __init__(self):
+        """
+        Initialize the Turbopuffer vector store.
+        """
+        super().__init__()
+
+        # Configure Turbopuffer client
+        api_key = settings.VECTOR_STORE.TURBOPUFFER_API_KEY
+        if not api_key:
+            raise ValueError(
+                "VECTOR_STORE_TURBOPUFFER_API_KEY must be set for Turbopuffer vector store"
+            )
+
+        # Initialize the async Turbopuffer client
+        # Region can be configured via VECTOR_STORE_TURBOPUFFER_REGION or TURBOPUFFER_REGION env var
+        region = settings.VECTOR_STORE.TURBOPUFFER_REGION or "gcp-us-east4"
+        self.tpuf = AsyncTurbopuffer(api_key=api_key, region=region)
+
+    def _get_namespace(self, namespace: str) -> AsyncNamespace:
+        """Get a Turbopuffer namespace object."""
+        return self.tpuf.namespace(namespace)
+
+    async def upsert_many(
+        self,
+        namespace: str,
+        vectors: list[VectorRecord],
+    ) -> VectorUpsertResult:
+        """
+        Upsert multiple vectors into Turbopuffer.
+
+        Args:
+            namespace: The namespace to store the vectors in
+            vectors: List of VectorRecord objects to upsert
+        """
+        if not vectors:
+            return VectorUpsertResult(ok=True)
+
+        ns = self._get_namespace(namespace)
+
+        rows: list[dict[str, Any]] = [
+            {
+                "id": v.id,
+                "vector": v.embedding,
+                **(v.metadata or {}),
+            }
+            for v in vectors
+        ]
+
+        try:
+            await ns.write(
+                upsert_rows=rows,
+                distance_metric=DISTANCE_METRIC,
+            )
+            return VectorUpsertResult(ok=True)
+        except Exception:
+            logger.exception(
+                f"Failed to upsert {len(vectors)} vectors to namespace {namespace}"
+            )
+            raise
+
+    async def query(
+        self,
+        namespace: str,
+        embedding: list[float],
+        *,
+        top_k: int = 10,
+        filters: dict[str, Any] | None = None,
+        max_distance: float | None = None,
+    ) -> list[VectorQueryResult]:
+        """
+        Query for similar vectors in Turbopuffer.
+
+        Args:
+            namespace: The namespace to query
+            embedding: The query embedding vector
+            top_k: Maximum number of results to return
+            filters: Optional metadata filters
+            max_distance: Optional maximum distance threshold (cosine distance)
+
+        Returns:
+            List of VectorQueryResult objects, ordered by similarity (most similar first)
+        """
+        ns = self._get_namespace(namespace)
+
+        try:
+            # Build filter conditions for Turbopuffer
+            filter_condition = self._build_filters(filters) if filters else None
+
+            # Query using rank_by for vector similarity
+            # rank_by must be a tuple: (attribute, "ANN", vector)
+            rank_by: tuple[str, Literal["ANN"], Sequence[float]] = (
+                "vector",
+                "ANN",
+                embedding,
+            )
+
+            # Only pass filters if we have them (avoid passing None)
+            query_kwargs: dict[str, Any] = {
+                "rank_by": rank_by,
+                "top_k": top_k,
+                "distance_metric": DISTANCE_METRIC,
+                "include_attributes": True,
+            }
+            if filter_condition is not None:
+                query_kwargs["filters"] = filter_condition
+
+            response = await ns.query(**query_kwargs)
+
+            query_results: list[VectorQueryResult] = []
+            for row in response.rows or []:
+                # Distance is accessed via row["$dist"]
+                dist: float = float(row["$dist"]) if "$dist" in row else 0.0
+                # Filter by max_distance if specified
+                if max_distance is not None and dist > max_distance:
+                    continue
+
+                # Extract attributes from model_extra (excludes id, vector, $dist)
+                row_metadata: dict[str, Any] = {}
+                if row.model_extra:
+                    # Filter out internal fields like $dist
+                    row_metadata = {
+                        k: v
+                        for k, v in row.model_extra.items()
+                        if not k.startswith("$")
+                    }
+
+                query_results.append(
+                    VectorQueryResult(
+                        id=str(row.id),
+                        score=dist,
+                        metadata=row_metadata,
+                    )
+                )
+
+            logger.debug(
+                f"Query returned {len(query_results)} results from namespace {namespace}"
+            )
+            return query_results
+
+        except NotFoundError:
+            # Namespace doesn't exist yet - no vectors have been written
+            # Return empty results (same behavior as LanceDB for missing tables)
+            logger.debug(
+                f"Namespace {namespace} does not exist, returning empty results"
+            )
+            return []
+
+        except Exception:
+            logger.exception(f"Failed to query namespace {namespace}")
+            raise
+
+    def _build_filters(self, filters: dict[str, Any]) -> Filter | None:
+        """
+        Convert a filter dict to Turbopuffer filter format.
+
+        Turbopuffer uses tuples like (attribute, "Eq", value) for filters,
+        (attribute, "In", [values]) for membership filters,
+        and ("And", [filters]) for combining multiple filters.
+
+        Supports filter formats:
+        - {"key": "value"} -> ("key", "Eq", "value")
+        - {"key": {"in": ["a", "b"]}} -> ("key", "In", ["a", "b"])
+
+        Args:
+            filters: Dictionary of attribute -> value filters
+
+        Returns:
+            Turbopuffer Filter or None if no filters
+        """
+        if not filters:
+            return None
+
+        filter_list: list[EqFilter | InFilter] = []
+        for key, value in filters.items():
+            # Check if value is a dict with "in" operator
+            if isinstance(value, dict) and "in" in value:
+                # Membership filter using "In" operator
+                in_values = cast(Sequence[Any], value["in"])
+                filter_list.append((key, "In", in_values))
+            else:
+                # Simple equality filter using "Eq" operator
+                filter_list.append((key, "Eq", cast(Any, value)))
+
+        if not filter_list:
+            return None
+
+        if len(filter_list) == 1:
+            return filter_list[0]
+
+        # Combine multiple filters with AND
+        and_filter: AndFilter = ("And", filter_list)
+        return and_filter
+
+    async def delete_many(self, namespace: str, ids: list[str]) -> None:
+        """
+        Delete multiple vectors from Turbopuffer.
+
+        Args:
+            namespace: The namespace containing the vectors
+            ids: List of vector identifiers to delete
+        """
+        if not ids:
+            return
+
+        ns = self._get_namespace(namespace)
+
+        try:
+            await ns.write(deletes=ids)
+        except NotFoundError:
+            # Namespace doesn't exist - nothing to delete
+            logger.debug(f"Namespace {namespace} does not exist, nothing to delete")
+        except Exception:
+            logger.exception(
+                f"Failed to delete {len(ids)} vectors from namespace {namespace}"
+            )
+            raise
+
+    async def delete_namespace(self, namespace: str) -> None:
+        """
+        Delete an entire namespace and all its vectors from Turbopuffer.
+
+        Args:
+            namespace: The namespace to delete
+        """
+        ns = self._get_namespace(namespace)
+
+        try:
+            await ns.delete_all()
+            logger.debug(f"Deleted all vectors from namespace {namespace}")
+        except NotFoundError:
+            # Namespace doesn't exist - nothing to delete
+            logger.debug(f"Namespace {namespace} does not exist, nothing to delete")
+        except Exception:
+            logger.exception(f"Failed to delete namespace {namespace}")
+            raise
+
+    async def close(self) -> None:
+        """Close the Turbopuffer client and release resources."""
+        await self.tpuf.close()
+        logger.debug("Turbopuffer client closed")

src/vector_store/utils.py

@@ -0,0 +1,57 @@
+"""
+Vector store utility functions.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from tenacity import (
+    AsyncRetrying,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+if TYPE_CHECKING:
+    from src.vector_store import VectorRecord, VectorStore, VectorUpsertResult
+
+logger = logging.getLogger(__name__)
+
+
+async def upsert_with_retry(
+    vector_store: VectorStore,
+    namespace: str,
+    vector_records: list[VectorRecord],
+    max_attempts: int = 3,
+) -> VectorUpsertResult | None:
+    """
+    Upsert vectors with exponential backoff retry.
+
+    Args:
+        vector_store: The vector store to upsert into
+        namespace: The namespace for the vectors
+        vector_records: List of VectorRecord objects to upsert
+        max_attempts: Maximum number of retry attempts (default 3)
+
+    Returns:
+        VectorUpsertResult on success, or None if vector_records is empty
+
+    Raises:
+        Exception: If all retries fail
+    """
+    if not vector_records:
+        return None
+
+    result: VectorUpsertResult | None = None
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(max_attempts),
+        wait=wait_exponential(multiplier=0.5, min=0.5, max=2.0),
+        retry=retry_if_exception_type(Exception),
+        reraise=True,
+    ):
+        with attempt:
+            result = await vector_store.upsert_many(namespace, vector_records)
+
+    return result

tests/alembic/revisions/__init__.py

@@ -4,12 +4,14 @@ from . import (
     test_05486ce795d5_make_session_name_required_on_messages,
     test_066e87ca5b07_align_schema_with_declarative_models,
     test_08894082221a_replace_collection_name_with_observer_,
+    test_7c0d9a4e3b1f_add_unique_index_for_pending_dreams,
     test_20f89a421aff_rename_metamessage_type_to_label,
     test_29ade7350c19_remove_document_level_valid_constraint,
     test_66e63cf2cf77_add_indexes_to_documents_table,
     test_76ffba56fe8c_add_error_field_to_queueitem,
     test_88b0fb10906f_add_webhooks_table,
     test_110bdf470272_rename_deriver_disabled_to_deriver_,
+    test_119a52b73c60_support_external_embeddings,
     test_556a16564f50_add_user_id_and_app_id_to_tables,
     test_564ba40505c5_add_session_name_column_to_documents,
     test_917195d9b5e9_add_messageembedding_table,
@@ -22,6 +24,7 @@ from . import (
     test_d429de0e5338_adopt_peer_paradigm,
     test_e9b705f9adf9_add_server_defaults_to_timestamp_,
     test_ec8f94139b02_codify_workspace_name_and_message_id_in_,
+    test_f1a2b3c4d5e6_add_reasoning_tree_columns,
 )
 
 __all__ = [
@@ -29,12 +32,14 @@ __all__ = [
     "test_066e87ca5b07_align_schema_with_declarative_models",
     "test_08894082221a_replace_collection_name_with_observer_",
     "test_110bdf470272_rename_deriver_disabled_to_deriver_",
+    "test_119a52b73c60_support_external_embeddings",
     "test_20f89a421aff_rename_metamessage_type_to_label",
     "test_29ade7350c19_remove_document_level_valid_constraint",
     "test_556a16564f50_add_user_id_and_app_id_to_tables",
     "test_564ba40505c5_add_session_name_column_to_documents",
     "test_66e63cf2cf77_add_indexes_to_documents_table",
     "test_76ffba56fe8c_add_error_field_to_queueitem",
+    "test_7c0d9a4e3b1f_add_unique_index_for_pending_dreams",
     "test_88b0fb10906f_add_webhooks_table",
     "test_917195d9b5e9_add_messageembedding_table",
     "test_a1b2c3d4e5f6_initial_schema",
@@ -46,4 +51,5 @@ __all__ = [
     "test_d429de0e5338_adopt_peer_paradigm",
     "test_e9b705f9adf9_add_server_defaults_to_timestamp_",
     "test_ec8f94139b02_codify_workspace_name_and_message_id_in_",
+    "test_f1a2b3c4d5e6_add_reasoning_tree_columns",
 ]

tests/alembic/revisions/test_119a52b73c60_support_external_embeddings.py

@@ -0,0 +1,74 @@
+"""Hooks for revision 119a52b73c60 (support_external_embeddings)."""
+
+from __future__ import annotations
+
+from tests.alembic.registry import register_after_upgrade, register_before_upgrade
+from tests.alembic.verifier import MigrationVerifier
+
+# Indexes created by this migration
+INDEXES = (
+    ("documents", "ix_documents_deleted_at"),
+    ("documents", "ix_documents_sync_state"),
+    ("documents", "ix_documents_sync_state_last_sync_at"),
+    ("message_embeddings", "ix_message_embeddings_sync_state"),
+    ("message_embeddings", "ix_message_embeddings_sync_state_last_sync_at"),
+    ("queue", "uq_queue_reconciler_pending_work_unit_key"),
+)
+
+
+@register_before_upgrade("119a52b73c60")
+def prepare_support_external_embeddings(
+    verifier: MigrationVerifier,
+) -> None:
+    """Seed state and assertions before upgrading to 119a52b73c60."""
+    # Embedding columns should be NOT NULL before migration
+    verifier.assert_column_exists("message_embeddings", "embedding", nullable=False)
+    verifier.assert_column_exists("documents", "embedding", nullable=False)
+
+    # Soft delete column should not exist
+    verifier.assert_column_exists("documents", "deleted_at", exists=False)
+
+    # Sync state columns should not exist on documents
+    verifier.assert_column_exists("documents", "sync_state", exists=False)
+    verifier.assert_column_exists("documents", "last_sync_at", exists=False)
+    verifier.assert_column_exists("documents", "sync_attempts", exists=False)
+
+    # Sync state columns should not exist on message_embeddings
+    verifier.assert_column_exists("message_embeddings", "sync_state", exists=False)
+    verifier.assert_column_exists("message_embeddings", "last_sync_at", exists=False)
+    verifier.assert_column_exists("message_embeddings", "sync_attempts", exists=False)
+
+    # Queue workspace_name should be NOT NULL before migration
+    verifier.assert_column_exists("queue", "workspace_name", nullable=False)
+
+    # Indexes should not exist
+    verifier.assert_indexes_not_exist(INDEXES)
+
+
+@register_after_upgrade("119a52b73c60")
+def verify_support_external_embeddings(
+    verifier: MigrationVerifier,
+) -> None:
+    """Add assertions validating the effects of 119a52b73c60."""
+    # Embedding columns should now be nullable
+    verifier.assert_column_exists("message_embeddings", "embedding", nullable=True)
+    verifier.assert_column_exists("documents", "embedding", nullable=True)
+
+    # Soft delete column should exist and be nullable
+    verifier.assert_column_exists("documents", "deleted_at", nullable=True)
+
+    # Sync state columns should exist on documents
+    verifier.assert_column_exists("documents", "sync_state", nullable=False)
+    verifier.assert_column_exists("documents", "last_sync_at", nullable=True)
+    verifier.assert_column_exists("documents", "sync_attempts", nullable=False)
+
+    # Sync state columns should exist on message_embeddings
+    verifier.assert_column_exists("message_embeddings", "sync_state", nullable=False)
+    verifier.assert_column_exists("message_embeddings", "last_sync_at", nullable=True)
+    verifier.assert_column_exists("message_embeddings", "sync_attempts", nullable=False)
+
+    # Queue workspace_name should now be nullable for system-level tasks
+    verifier.assert_column_exists("queue", "workspace_name", nullable=True)
+
+    # All indexes should exist
+    verifier.assert_indexes_exist(INDEXES)

tests/alembic/revisions/test_7c0d9a4e3b1f_add_unique_index_for_pending_dreams.py

@@ -0,0 +1,23 @@
+"""Hooks for revision 7c0d9a4e3b1f (add_unique_index_for_pending_dreams)."""
+
+from __future__ import annotations
+
+from tests.alembic.registry import register_after_upgrade, register_before_upgrade
+from tests.alembic.verifier import MigrationVerifier
+
+# The partial unique index created by this migration
+INDEX = ("queue", "uq_queue_dream_pending_work_unit_key")
+
+
+@register_before_upgrade("7c0d9a4e3b1f")
+def prepare_add_unique_index_for_pending_dreams(verifier: MigrationVerifier) -> None:
+    """Seed state and assertions before upgrading to 7c0d9a4e3b1f."""
+    # The unique index should not exist before the migration
+    verifier.assert_indexes_not_exist([INDEX])
+
+
+@register_after_upgrade("7c0d9a4e3b1f")
+def verify_add_unique_index_for_pending_dreams(verifier: MigrationVerifier) -> None:
+    """Add assertions validating the effects of 7c0d9a4e3b1f."""
+    # The partial unique index should now exist
+    verifier.assert_indexes_exist([INDEX])

tests/conftest.py

@@ -399,6 +399,106 @@ def mock_openai_embeddings():
         yield {"embed": mock_embed, "batch_embed": mock_batch_embed}
 
 
+@pytest.fixture(autouse=True)
+def mock_vector_store():
+    """Mock vector store operations for testing"""
+    from unittest.mock import AsyncMock, MagicMock
+
+    from src.vector_store import (
+        VectorQueryResult,
+        VectorRecord,
+        VectorUpsertResult,
+        _hash_namespace_components,  # pyright: ignore[reportPrivateUsage]
+    )
+
+    # Create a mock vector store that stores vectors in memory
+    vector_storage: dict[str, dict[str, tuple[list[float], dict[str, Any]]]] = {}
+
+    async def mock_upsert_many(
+        namespace: str, vectors: list[VectorRecord]
+    ) -> VectorUpsertResult:
+        if namespace not in vector_storage:
+            vector_storage[namespace] = {}
+        for vector in vectors:
+            vector_storage[namespace][vector.id] = (vector.embedding, vector.metadata)
+        return VectorUpsertResult(ok=True)
+
+    async def mock_query(
+        namespace: str, embedding: list[float], **kwargs: Any
+    ) -> list[VectorQueryResult]:
+        _ = embedding  # unused in mock
+        if namespace not in vector_storage:
+            return []
+
+        # Simple mock: return all vectors in the namespace as results
+        results: list[VectorQueryResult] = []
+        for vec_id, (_vec_embedding, metadata) in vector_storage[namespace].items():
+            results.append(
+                VectorQueryResult(
+                    id=vec_id,
+                    score=0.1,  # Mock score
+                    metadata=metadata,
+                )
+            )
+        top_k: int = kwargs.get("top_k", 10)
+        return results[:top_k]
+
+    async def mock_delete_many(namespace: str, ids: list[str]) -> None:
+        if namespace in vector_storage:
+            for vec_id in ids:
+                vector_storage[namespace].pop(vec_id, None)
+
+    async def mock_delete_namespace(namespace: str) -> None:
+        vector_storage.pop(namespace, None)
+
+    # Clear the cache on get_external_vector_store before patching
+    from src.vector_store import get_external_vector_store
+
+    get_external_vector_store.cache_clear()  # type: ignore
+
+    # Create the mock vector store
+    mock_vs = MagicMock()
+    mock_vs.upsert_many = AsyncMock(side_effect=mock_upsert_many)
+    mock_vs.query = AsyncMock(side_effect=mock_query)
+    mock_vs.delete_many = AsyncMock(side_effect=mock_delete_many)
+    mock_vs.delete_namespace = AsyncMock(side_effect=mock_delete_namespace)
+
+    def mock_get_vector_namespace(
+        namespace_type: str,
+        workspace_name: str,
+        observer: str | None = None,
+        observed: str | None = None,
+    ) -> str:
+        # Uses real hash function for consistency with production
+        if namespace_type == "document":
+            if observer is None or observed is None:
+                raise ValueError(
+                    "observer and observed are required for document namespaces"
+                )
+            return f"honcho2345.doc.{_hash_namespace_components(workspace_name, observer, observed)}"
+        if namespace_type == "message":
+            return f"honcho2345.msg.{_hash_namespace_components(workspace_name)}"
+        raise ValueError(f"Unknown namespace type: {namespace_type}")
+
+    mock_vs.get_vector_namespace = mock_get_vector_namespace
+
+    with (
+        patch("src.crud.document.get_external_vector_store", return_value=mock_vs),
+        patch("src.crud.workspace.get_external_vector_store", return_value=mock_vs),
+        patch("src.crud.session.get_external_vector_store", return_value=mock_vs),
+        patch("src.crud.message.get_external_vector_store", return_value=mock_vs),
+        patch(
+            "src.reconciler.sync_vectors.get_external_vector_store",
+            return_value=mock_vs,
+        ),
+        patch("src.utils.search.get_external_vector_store", return_value=mock_vs),
+    ):
+        yield mock_vs
+
+        # Clear cache after test as well for cleanliness
+        get_external_vector_store.cache_clear()  # type: ignore
+
+
 @pytest.fixture(autouse=True)
 def mock_llm_call_functions():
     """Mock LLM functions to avoid needing API keys during tests"""

tests/crud/test_document.py

@@ -1,3 +1,5 @@
+import datetime
+
 import pytest
 from nanoid import generate as generate_nanoid
 from sqlalchemy import select
@@ -60,7 +62,6 @@ class TestDocumentCRUD:
             observer=test_peer.name,
             observed=test_peer2.name,
             content="Test observation 1",
-            embedding=[0.1] * 1536,
             session_name=test_session.name,
         )
         doc2 = models.Document(
@@ -68,7 +69,6 @@ class TestDocumentCRUD:
             observer=test_peer.name,
             observed=test_peer2.name,
             content="Test observation 2",
-            embedding=[0.2] * 1536,
             session_name=test_session.name,
         )
         db_session.add_all([doc1, doc2])
@@ -100,25 +100,34 @@ class TestDocumentCRUD:
             db_session, test_workspace, test_peer
         )
 
-        # Create test documents with different embeddings
-        doc1 = models.Document(
+        # Create test documents using create_documents to ensure they're in vector store
+        doc_schemas = [
+            schemas.DocumentCreate(
+                content="User likes pizza",
+                embedding=[0.9] * 1536,
+                session_name=test_session.name,
+                metadata=schemas.DocumentMetadata(
+                    message_ids=[1],
+                    message_created_at="2025-01-01T00:00:00Z",
+                ),
+            ),
+            schemas.DocumentCreate(
+                content="User dislikes vegetables",
+                embedding=[0.1] * 1536,
+                session_name=test_session.name,
+                metadata=schemas.DocumentMetadata(
+                    message_ids=[2],
+                    message_created_at="2025-01-01T00:00:00Z",
+                ),
+            ),
+        ]
+        await crud.create_documents(
+            db_session,
+            doc_schemas,
             workspace_name=test_workspace.name,
             observer=test_peer.name,
             observed=test_peer2.name,
-            content="User likes pizza",
-            embedding=[0.9] * 1536,
-            session_name=test_session.name,
         )
-        doc2 = models.Document(
-            workspace_name=test_workspace.name,
-            observer=test_peer.name,
-            observed=test_peer2.name,
-            content="User dislikes vegetables",
-            embedding=[0.1] * 1536,
-            session_name=test_session.name,
-        )
-        db_session.add_all([doc1, doc2])
-        await db_session.flush()
 
         # Query documents
         results = await crud.query_documents(
@@ -132,6 +141,139 @@ class TestDocumentCRUD:
 
         assert len(results) == 2
 
+    @pytest.mark.asyncio
+    async def test_query_documents_excludes_soft_deleted(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ):
+        """Query results should not include soft-deleted documents even if vectors remain"""
+        test_workspace, test_peer = sample_data
+        test_peer2, test_session, _ = await self._setup_test_data(
+            db_session, test_workspace, test_peer
+        )
+
+        # Create two documents and persist embeddings
+        doc_schemas = [
+            schemas.DocumentCreate(
+                content="User likes pizza",
+                embedding=[0.9] * 1536,
+                session_name=test_session.name,
+                metadata=schemas.DocumentMetadata(
+                    message_ids=[1],
+                    message_created_at="2025-01-01T00:00:00Z",
+                ),
+            ),
+            schemas.DocumentCreate(
+                content="User dislikes vegetables",
+                embedding=[0.1] * 1536,
+                session_name=test_session.name,
+                metadata=schemas.DocumentMetadata(
+                    message_ids=[2],
+                    message_created_at="2025-01-01T00:00:00Z",
+                ),
+            ),
+        ]
+        await crud.create_documents(
+            db_session,
+            doc_schemas,
+            workspace_name=test_workspace.name,
+            observer=test_peer.name,
+            observed=test_peer2.name,
+        )
+
+        # Soft-delete one document without touching vectors
+        stmt = select(models.Document).where(
+            models.Document.workspace_name == test_workspace.name,
+            models.Document.observer == test_peer.name,
+            models.Document.observed == test_peer2.name,
+        )
+        result = await db_session.execute(stmt)
+        docs = {doc.content: doc for doc in result.scalars().all()}
+        deleted_doc = docs["User likes pizza"]
+        kept_doc = docs["User dislikes vegetables"]
+
+        deleted_doc.deleted_at = datetime.datetime.now(datetime.timezone.utc)
+        await db_session.commit()
+
+        results = await crud.query_documents(
+            db_session,
+            workspace_name=test_workspace.name,
+            query="food preferences",
+            observer=test_peer.name,
+            observed=test_peer2.name,
+            top_k=10,
+        )
+
+        assert len(results) == 1
+        assert results[0].id == kept_doc.id
+
+    @pytest.mark.asyncio
+    async def test_query_documents_applies_additional_filters(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ):
+        """Filters beyond vector metadata should be enforced at the DB layer"""
+        test_workspace, test_peer = sample_data
+        test_peer2, test_session, _ = await self._setup_test_data(
+            db_session, test_workspace, test_peer
+        )
+
+        doc_schemas = [
+            schemas.DocumentCreate(
+                content="Observation one",
+                embedding=[0.5] * 1536,
+                session_name=test_session.name,
+                times_derived=1,
+                metadata=schemas.DocumentMetadata(
+                    message_ids=[1],
+                    message_created_at="2025-01-01T00:00:00Z",
+                ),
+            ),
+            schemas.DocumentCreate(
+                content="Observation two",
+                embedding=[0.5] * 1536,
+                session_name=test_session.name,
+                times_derived=2,
+                metadata=schemas.DocumentMetadata(
+                    message_ids=[2],
+                    message_created_at="2025-01-01T00:00:00Z",
+                ),
+            ),
+        ]
+        await crud.create_documents(
+            db_session,
+            doc_schemas,
+            workspace_name=test_workspace.name,
+            observer=test_peer.name,
+            observed=test_peer2.name,
+        )
+
+        result = await db_session.execute(
+            select(models.Document).where(
+                models.Document.workspace_name == test_workspace.name,
+                models.Document.observer == test_peer.name,
+                models.Document.observed == test_peer2.name,
+            )
+        )
+        docs = result.scalars().all()
+        times_derived_map = {doc.times_derived: doc.id for doc in docs}
+
+        results = await crud.query_documents(
+            db_session,
+            workspace_name=test_workspace.name,
+            query="any query",
+            observer=test_peer.name,
+            observed=test_peer2.name,
+            top_k=10,
+            filters={"times_derived": 2},
+            embedding=[0.5] * 1536,
+        )
+
+        assert len(results) == 1
+        assert results[0].id == times_derived_map[2]
+
     @pytest.mark.asyncio
     async def test_delete_document_success(
         self,
@@ -150,7 +292,6 @@ class TestDocumentCRUD:
             observer=test_peer.name,
             observed=test_peer2.name,
             content="Test observation",
-            embedding=[0.1] * 1536,
             session_name=test_session.name,
         )
         db_session.add(doc)
@@ -172,9 +313,11 @@ class TestDocumentCRUD:
             observed=test_peer2.name,
         )
 
-        # Verify document is deleted
+        # Verify document is soft-deleted
         result = await db_session.execute(stmt)
-        assert result.scalar_one_or_none() is None
+        doc = result.scalar_one_or_none()
+        assert doc is not None
+        assert doc.deleted_at is not None
 
     @pytest.mark.asyncio
     async def test_delete_document_not_found(
@@ -214,8 +357,8 @@ class TestDocumentCRUD:
         doc_schemas = [
             schemas.DocumentCreate(
                 content="Observation 1",
-                session_name=test_session.name,
                 embedding=[0.1] * 1536,
+                session_name=test_session.name,
                 level="explicit",
                 metadata=schemas.DocumentMetadata(
                     message_ids=[1, 2, 3, 4, 5],
@@ -224,8 +367,8 @@ class TestDocumentCRUD:
             ),
             schemas.DocumentCreate(
                 content="Observation 2",
-                session_name=test_session.name,
                 embedding=[0.2] * 1536,
+                session_name=test_session.name,
                 level="deductive",
                 metadata=schemas.DocumentMetadata(
                     message_ids=[6, 7, 8, 9, 10],

tests/crud/test_workspace.py

@@ -199,7 +199,6 @@ class TestWorkspaceCRUD:
             observed=test_peer.name,
             session_name=session.name,
             content="Test document content",
-            embedding=[0.1] * 1536,  # Mock embedding vector
         )
         db_session.add(document)
         await db_session.flush()
@@ -451,7 +450,6 @@ class TestWorkspaceCRUD:
             observed=peer2.name,
             session_name=session1.name,
             content="Test document",
-            embedding=[0.1] * 1536,  # Mock embedding vector
         )
         db_session.add(document)
 

tests/deriver/test_vector_reconciliation.py

@@ -0,0 +1,815 @@
+"""
+Tests for vector store reconciliation.
+
+This module tests the vector reconciliation system that syncs documents and
+message embeddings to the vector store, handling failures and retries.
+"""
+
+import datetime
+from typing import cast
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from nanoid import generate as generate_nanoid
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src import models
+from src.reconciler.sync_vectors import (
+    MAX_SYNC_ATTEMPTS,
+    ReconciliationMetrics,
+    _get_documents_needing_sync,  # pyright: ignore[reportPrivateUsage]
+    _get_message_embeddings_needing_sync,  # pyright: ignore[reportPrivateUsage]
+    _sync_documents,  # pyright: ignore[reportPrivateUsage]
+    _sync_message_embeddings,  # pyright: ignore[reportPrivateUsage]
+    run_vector_reconciliation_cycle,
+)
+from src.vector_store import (
+    VectorRecord,
+    VectorStore,
+    VectorUpsertResult,
+    _hash_namespace_components,  # pyright: ignore[reportPrivateUsage]
+)
+
+
+@pytest.mark.asyncio
+class TestStateTransitions:
+    """Test document sync_state transitions."""
+
+    async def test_pending_to_synced_on_success(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test documents transition from pending → synced on successful sync."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create documents in pending state with embeddings
+        docs = [
+            models.Document(
+                content=f"doc_{i}",
+                workspace_name=workspace.name,
+                observer=peer1.name,
+                observed=peer1.name,
+                session_name=session.name,
+                sync_state="pending",
+                sync_attempts=0,
+                embedding=[float(i)] * 1536,  # Mock embedding
+            )
+            for i in range(3)
+        ]
+        db_session.add_all(docs)
+        await db_session.commit()
+        for doc in docs:
+            await db_session.refresh(doc)
+
+        # Mock vector store to succeed
+        mock_vector_store = MagicMock(spec=VectorStore)
+        mock_vector_store.get_vector_namespace = MagicMock(
+            return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
+        )
+        mock_vector_store.upsert_many = AsyncMock(
+            return_value=VectorUpsertResult(ok=True)
+        )
+
+        # Run sync
+        synced, failed = await _sync_documents(db_session, docs, mock_vector_store)
+
+        # Verify results
+        assert synced == 3
+        assert failed == 0
+
+        # Check state transitions
+        for doc in docs:
+            await db_session.refresh(doc)
+            assert doc.sync_state == "synced"
+            assert doc.sync_attempts == 0
+            assert doc.last_sync_at is not None
+
+    async def test_pending_to_pending_with_incremented_attempts(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test documents remain pending with incremented attempts on partial failure."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create document in pending state
+        doc = models.Document(
+            content="test doc",
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+            session_name=session.name,
+            sync_state="pending",
+            sync_attempts=2,  # Already failed twice
+            embedding=[1.0] * 1536,
+        )
+        db_session.add(doc)
+        await db_session.commit()
+        await db_session.refresh(doc)
+
+        # Mock vector store to fail with exception
+        mock_vector_store = MagicMock(spec=VectorStore)
+        mock_vector_store.get_vector_namespace = MagicMock(
+            return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
+        )
+        mock_vector_store.upsert_many = AsyncMock(
+            side_effect=Exception("Vector store failed")
+        )
+
+        # Run sync
+        synced, failed = await _sync_documents(db_session, [doc], mock_vector_store)
+
+        # Verify failure recorded
+        assert synced == 0
+        assert failed == 1
+
+        # Check sync_attempts incremented
+        await db_session.refresh(doc)
+        assert doc.sync_state == "pending"  # Still pending
+        assert doc.sync_attempts == 3  # Incremented
+        assert doc.last_sync_at is not None  # Updated timestamp
+
+    async def test_pending_to_failed_after_max_attempts(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test documents transition to failed after MAX_SYNC_ATTEMPTS failures."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create document at max attempts - 1
+        doc = models.Document(
+            content="failing doc",
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+            session_name=session.name,
+            sync_state="pending",
+            sync_attempts=MAX_SYNC_ATTEMPTS - 1,  # One more attempt will hit limit
+            embedding=[1.0] * 1536,
+        )
+        db_session.add(doc)
+        await db_session.commit()
+        await db_session.refresh(doc)
+
+        # Mock vector store to fail with exception
+        mock_vector_store = MagicMock(spec=VectorStore)
+        mock_vector_store.get_vector_namespace = MagicMock(
+            return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
+        )
+        mock_vector_store.upsert_many = AsyncMock(
+            side_effect=Exception("Vector store failed")
+        )
+
+        # Run sync - this should be the final attempt
+        synced, failed = await _sync_documents(db_session, [doc], mock_vector_store)
+
+        # Verify marked as failed
+        assert synced == 0
+        assert failed == 1
+
+        await db_session.refresh(doc)
+        assert doc.sync_state == "failed"  # Permanently failed
+        assert doc.sync_attempts == MAX_SYNC_ATTEMPTS
+
+
+@pytest.mark.asyncio
+class TestBatchProcessing:
+    """Test batch processing and namespace grouping."""
+
+    async def test_documents_grouped_by_namespace(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test documents from different collections are grouped by namespace."""
+        workspace, peer1 = sample_data
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create another peer for different observer/observed combinations
+        peer2 = models.Peer(
+            name="peer2",
+            workspace_name=workspace.name,
+        )
+        db_session.add(peer2)
+        await db_session.commit()
+
+        # Create collections for both observer/observed combinations
+        collection1 = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        collection2 = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer2.name,
+        )
+        db_session.add_all([collection1, collection2])
+        await db_session.commit()
+
+        # Create documents for different namespaces
+        docs = [
+            # Namespace 1: peer1 → peer1
+            models.Document(
+                content="doc1",
+                workspace_name=workspace.name,
+                observer=peer1.name,
+                observed=peer1.name,
+                session_name=session.name,
+                sync_state="pending",
+                embedding=[1.0] * 1536,
+            ),
+            # Namespace 2: peer1 → peer2
+            models.Document(
+                content="doc2",
+                workspace_name=workspace.name,
+                observer=peer1.name,
+                observed=peer2.name,
+                session_name=session.name,
+                sync_state="pending",
+                embedding=[2.0] * 1536,
+            ),
+            # Namespace 1 again: peer1 → peer1
+            models.Document(
+                content="doc3",
+                workspace_name=workspace.name,
+                observer=peer1.name,
+                observed=peer1.name,
+                session_name=session.name,
+                sync_state="pending",
+                embedding=[3.0] * 1536,
+            ),
+        ]
+        db_session.add_all(docs)
+        await db_session.commit()
+
+        # Mock vector store to track calls by namespace
+        mock_vector_store = MagicMock(spec=VectorStore)
+        namespace_calls: dict[str, list[VectorRecord]] = {}
+
+        def mock_get_namespace(
+            _namespace_type: str, workspace: str, observer: str, observed: str
+        ) -> str:
+            return f"honcho.doc.{_hash_namespace_components(workspace, observer, observed)}"
+
+        async def mock_upsert(
+            namespace: str, vectors: list[VectorRecord]
+        ) -> VectorUpsertResult:
+            if namespace not in namespace_calls:
+                namespace_calls[namespace] = []
+            namespace_calls[namespace].extend(vectors)
+            return VectorUpsertResult(ok=True)
+
+        mock_vector_store.get_vector_namespace = mock_get_namespace
+        mock_vector_store.upsert_many = mock_upsert
+
+        # Run sync
+        synced, failed = await _sync_documents(db_session, docs, mock_vector_store)
+
+        # Verify all synced
+        assert synced == 3
+        assert failed == 0
+
+        # Verify namespaces
+        expected_ns1 = f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
+        expected_ns2 = f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer2.name)}"
+
+        assert expected_ns1 in namespace_calls
+        assert expected_ns2 in namespace_calls
+        assert len(namespace_calls[expected_ns1]) == 2  # doc1 and doc3
+        assert len(namespace_calls[expected_ns2]) == 1  # doc2
+
+    async def test_batch_size_respected(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test that batch size limits are respected when fetching pending documents."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create more documents than batch size
+        docs = [
+            models.Document(
+                content=f"doc_{i}",
+                workspace_name=workspace.name,
+                observer=peer1.name,
+                observed=peer1.name,
+                session_name=session.name,
+                sync_state="pending",
+                embedding=[float(i)] * 1536,
+            )
+            for i in range(150)  # More than RECONCILIATION_BATCH_SIZE (100)
+        ]
+        db_session.add_all(docs)
+        await db_session.commit()
+
+        # Fetch documents with batch size limit
+        batch = await _get_documents_needing_sync(db_session, batch_size=100)
+
+        # Verify batch size respected
+        assert len(batch) == 100
+
+
+@pytest.mark.asyncio
+class TestReEmbedding:
+    """Test re-embedding logic for documents with NULL embeddings."""
+
+    async def test_documents_without_embeddings_are_reembedded(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test documents with NULL embeddings are re-embedded during reconciliation."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create documents without embeddings
+        docs = [
+            models.Document(
+                content=f"doc_{i}",
+                workspace_name=workspace.name,
+                observer=peer1.name,
+                observed=peer1.name,
+                session_name=session.name,
+                sync_state="pending",
+                embedding=None,  # NULL embedding
+            )
+            for i in range(3)
+        ]
+        db_session.add_all(docs)
+        await db_session.commit()
+        for doc in docs:
+            await db_session.refresh(doc)
+
+        # Mock embedding client
+        with patch("src.reconciler.sync_vectors.embedding_client") as mock_embed_client:
+            mock_embed_client.simple_batch_embed = AsyncMock(
+                return_value=[[float(i)] * 1536 for i in range(3)]
+            )
+
+            # Mock vector store
+            mock_vector_store = MagicMock(spec=VectorStore)
+            mock_vector_store.get_vector_namespace = MagicMock(
+                return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
+            )
+            mock_vector_store.upsert_many = AsyncMock(
+                return_value=VectorUpsertResult(ok=True)
+            )
+
+            # Run sync
+            synced, failed = await _sync_documents(db_session, docs, mock_vector_store)
+
+            # Verify embedding was called
+            mock_embed_client.simple_batch_embed.assert_called_once()
+
+            # Verify documents were synced
+            assert synced == 3
+            assert failed == 0
+
+    async def test_large_documents_embedded_in_single_batch(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test that documents are embedded in a single batch (no sub-batching)."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create documents without embeddings
+        docs = [
+            models.Document(
+                content=f"doc_{i}",
+                workspace_name=workspace.name,
+                observer=peer1.name,
+                observed=peer1.name,
+                session_name=session.name,
+                sync_state="pending",
+                embedding=None,  # Will be re-embedded
+            )
+            for i in range(3)
+        ]
+        db_session.add_all(docs)
+        await db_session.commit()
+        for doc in docs:
+            await db_session.refresh(doc)
+
+        # Mock embedding client to track batch calls
+        batch_call_count = 0
+
+        async def track_batch_embed(contents: list[str]) -> list[list[float]]:
+            nonlocal batch_call_count
+            batch_call_count += 1
+            return [[1.0] * 1536 for _ in contents]
+
+        with patch("src.reconciler.sync_vectors.embedding_client") as mock_embed_client:
+            mock_embed_client.simple_batch_embed = track_batch_embed
+
+            # Mock vector store
+            mock_vector_store = MagicMock(spec=VectorStore)
+            mock_vector_store.get_vector_namespace = MagicMock(
+                return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
+            )
+            mock_vector_store.upsert_many = AsyncMock(
+                return_value=VectorUpsertResult(ok=True)
+            )
+
+            # Run sync
+            await _sync_documents(db_session, docs, mock_vector_store)
+
+            # Verify single batch call (no sub-batching)
+            assert batch_call_count == 1
+
+
+@pytest.mark.asyncio
+class TestSoftDeleteCleanup:
+    """Test soft delete cleanup functionality."""
+
+    async def test_cleanup_respects_grace_period(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test cleanup only processes documents deleted_at older than threshold."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create recently soft-deleted document (within grace period)
+        recent_doc = models.Document(
+            content="recent",
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+            session_name=session.name,
+            deleted_at=datetime.datetime.now(datetime.timezone.utc)
+            - datetime.timedelta(minutes=2),  # Only 2 minutes ago
+        )
+        db_session.add(recent_doc)
+
+        # Create old soft-deleted document (outside grace period)
+        old_doc = models.Document(
+            content="old",
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+            session_name=session.name,
+            deleted_at=datetime.datetime.now(datetime.timezone.utc)
+            - datetime.timedelta(minutes=10),  # 10 minutes ago
+        )
+        db_session.add(old_doc)
+
+        await db_session.commit()
+
+        # Query for documents ready for cleanup (older than 5 minutes)
+        cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
+            minutes=5
+        )
+        stmt = (
+            select(models.Document)
+            .where(models.Document.deleted_at.is_not(None))
+            .where(models.Document.deleted_at < cutoff)
+        )
+        result = await db_session.execute(stmt)
+        ready_for_cleanup = result.scalars().all()
+
+        # Only old_doc should be ready
+        assert len(ready_for_cleanup) == 1
+        assert ready_for_cleanup[0].id == old_doc.id
+
+
+@pytest.mark.asyncio
+class TestMetricsTracking:
+    """Test reconciliation metrics collection."""
+
+    async def test_metrics_track_sync_results(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Test ReconciliationMetrics tracks synced/failed counts."""
+        workspace, peer1 = sample_data
+
+        # Create collection (required for documents)
+        collection = models.Collection(
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+        )
+        db_session.add(collection)
+        await db_session.commit()
+
+        # Create session
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        # Create mix of documents that will succeed and fail
+        success_doc = models.Document(
+            content="success",
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+            session_name=session.name,
+            sync_state="pending",
+            embedding=[1.0] * 1536,
+        )
+        fail_doc = models.Document(
+            content="fail",
+            workspace_name=workspace.name,
+            observer=peer1.name,
+            observed=peer1.name,
+            session_name=session.name,
+            sync_state="pending",
+            sync_attempts=MAX_SYNC_ATTEMPTS - 1,  # Will fail on next attempt
+            embedding=[2.0] * 1536,
+        )
+        db_session.add_all([success_doc, fail_doc])
+        await db_session.commit()
+
+        # Note: Since both docs have same namespace, they'll be in one batch
+        # This test structure needs adjustment for the actual grouping logic
+        # For simplicity, let's test metrics at the function level
+
+        metrics = ReconciliationMetrics()
+        metrics.documents_synced = 5
+        metrics.documents_failed = 2
+        metrics.message_embeddings_synced = 3
+
+        assert metrics.total_synced == 8
+        assert metrics.total_failed == 2
+
+
+@pytest.mark.asyncio
+class TestMessageEmbeddings:
+    """Test message embedding reconciliation paths."""
+
+    async def _create_pending_message_embedding(
+        self,
+        db_session: AsyncSession,
+        workspace: models.Workspace,
+        peer: models.Peer,
+    ) -> models.MessageEmbedding:
+        """Helper to create a pending message embedding with no stored vector."""
+        session = models.Session(
+            name=str(generate_nanoid()), workspace_name=workspace.name
+        )
+        db_session.add(session)
+        await db_session.commit()
+
+        message = models.Message(
+            public_id=str(generate_nanoid()),
+            session_name=session.name,
+            workspace_name=workspace.name,
+            peer_name=peer.name,
+            content="hello world",
+            seq_in_session=1,
+        )
+        db_session.add(message)
+        await db_session.commit()
+
+        emb = models.MessageEmbedding(
+            content=message.content,
+            message_id=message.public_id,
+            workspace_name=workspace.name,
+            session_name=session.name,
+            peer_name=peer.name,
+            sync_state="pending",
+            embedding=None,
+        )
+        db_session.add(emb)
+        await db_session.commit()
+        await db_session.refresh(emb)
+        return emb
+
+    async def test_pending_embeddings_are_selected_without_vectors(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+    ) -> None:
+        """Pending rows with NULL embeddings should still be reconciled."""
+        workspace, peer = sample_data
+        pending_emb = await self._create_pending_message_embedding(
+            db_session, workspace, peer
+        )
+
+        pending = await _get_message_embeddings_needing_sync(db_session)
+        assert any(emb.id == pending_emb.id for emb in pending)
+
+    async def test_missing_embeddings_reembedded_and_synced(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+        mock_vector_store: VectorStore,
+    ) -> None:
+        """Reconciliation should re-embed missing payloads and mark them synced."""
+        workspace, peer = sample_data
+        pending_emb = await self._create_pending_message_embedding(
+            db_session, workspace, peer
+        )
+
+        synced, failed = await _sync_message_embeddings(
+            db_session, [pending_emb], mock_vector_store
+        )
+
+        await db_session.refresh(pending_emb)
+
+        assert synced == 1
+        assert failed == 0
+        assert pending_emb.sync_state == "synced"
+        assert pending_emb.sync_attempts == 0
+
+        # Ensure vector upsert was attempted with a populated embedding
+        upsert_mock: AsyncMock = cast(AsyncMock, mock_vector_store.upsert_many)
+        await_args = upsert_mock.await_args
+        assert await_args is not None
+        args = await_args.args
+        assert len(args) == 2
+        vector_records: list[VectorRecord] = args[1]
+        assert vector_records
+        assert vector_records[0].embedding
+
+    async def test_upsert_failure_marks_attempt_and_continues(
+        self,
+        db_session: AsyncSession,
+        sample_data: tuple[models.Workspace, models.Peer],
+        mock_vector_store: VectorStore,
+    ) -> None:
+        """Failures during upsert should bump attempts and keep row pending/failed."""
+        workspace, peer = sample_data
+        pending_emb = await self._create_pending_message_embedding(
+            db_session, workspace, peer
+        )
+
+        upsert_mock: AsyncMock = cast(AsyncMock, mock_vector_store.upsert_many)
+        upsert_mock.side_effect = Exception("boom")
+
+        synced, failed = await _sync_message_embeddings(
+            db_session, [pending_emb], mock_vector_store
+        )
+
+        await db_session.refresh(pending_emb)
+
+        assert synced == 0
+        assert failed == 1
+        assert pending_emb.sync_state in {"pending", "failed"}
+        assert pending_emb.sync_attempts == 1
+
+
+@pytest.mark.asyncio
+class TestEndToEndReconciliation:
+    """Test full reconciliation cycle."""
+
+    async def test_reconciliation_cycle_completes(
+        self,
+        db_session: AsyncSession,
+    ) -> None:
+        """Test full reconciliation cycle processes documents and embeddings."""
+        # This would be an integration test with the full cycle
+        # For now, we verify the function signature and return type
+        with (
+            patch("src.reconciler.sync_vectors.tracked_db") as mock_tracked_db,
+            patch("src.reconciler.sync_vectors.get_external_vector_store"),
+            patch(
+                "src.reconciler.sync_vectors._get_documents_needing_sync"
+            ) as mock_get_docs,
+            patch(
+                "src.reconciler.sync_vectors._get_message_embeddings_needing_sync"
+            ) as mock_get_embs,
+            patch("src.crud.document.cleanup_soft_deleted_documents") as mock_cleanup,
+        ):
+            # Mock to return empty results (no work to do)
+            mock_get_docs.return_value = []
+            mock_get_embs.return_value = []
+            mock_cleanup.return_value = 0
+
+            # Mock context manager
+            mock_db_context = MagicMock()
+            mock_db_context.__aenter__ = AsyncMock(return_value=db_session)
+            mock_db_context.__aexit__ = AsyncMock(return_value=None)
+            mock_tracked_db.return_value = mock_db_context
+
+            # Run cycle
+            metrics = await run_vector_reconciliation_cycle()
+
+            # Verify metrics returned
+            assert isinstance(metrics, ReconciliationMetrics)
+            assert metrics.total_synced == 0  # No work done

tests/integration/test_message_embeddings.py

@@ -64,16 +64,13 @@ async def test_message_embedding_created_when_setting_enabled(
     result = await db_session.execute(stmt)
     embedding_record = result.scalar_one_or_none()
 
-    # Verify the embedding was created
+    # Verify the embedding record was created (embedding vectors are now stored externally)
     assert embedding_record is not None
     assert embedding_record.message_id == created_message.public_id
     assert embedding_record.content == test_message_content
     assert embedding_record.workspace_name == test_workspace.name
     assert embedding_record.session_name == test_session.name
     assert embedding_record.peer_name == test_peer.name
-    # Verify embedding vector exists and is not empty
-    assert embedding_record.embedding is not None
-    assert len(embedding_record.embedding) > 0
 
 
 @pytest.mark.asyncio
@@ -176,15 +173,13 @@ async def test_multiple_message_embeddings_created_when_setting_enabled(
         result = await db_session.execute(stmt)
         embedding_record = result.scalar_one_or_none()
 
-        # Verify the embedding was created
+        # Verify the embedding record was created (embedding vectors are now stored externally)
         assert embedding_record is not None
         assert embedding_record.message_id == created_message.public_id
         assert embedding_record.content == messages[i].content
         assert embedding_record.workspace_name == test_workspace.name
         assert embedding_record.session_name == test_session.name
         assert embedding_record.peer_name == test_peer.name
-        assert embedding_record.embedding is not None
-        assert len(embedding_record.embedding) > 0
 
 
 @pytest.mark.asyncio
@@ -322,7 +317,8 @@ async def test_message_chunking_creates_multiple_embeddings(
     result = await db_session.execute(stmt)
     embedding_records = list(result.scalars().all())
 
-    # Verify multiple embeddings were created (one per chunk)
+    # Verify multiple embedding records were created (one per chunk)
+    # Embedding vectors are now stored externally in the vector store
     assert len(embedding_records) == 3  # Should have 3 embeddings for 3 chunks
 
     for _, embedding_record in enumerate(embedding_records):
@@ -333,7 +329,3 @@ async def test_message_chunking_creates_multiple_embeddings(
         assert embedding_record.workspace_name == test_workspace.name
         assert embedding_record.session_name == test_session.name
         assert embedding_record.peer_name == test_peer.name
-        assert embedding_record.embedding is not None
-        assert len(embedding_record.embedding) == 1536
-        # Each chunk should have a different embedding vector (0.1, 0.2, 0.3)
-        assert embedding_record.embedding[0] in [0.1, 0.2, 0.3]

tests/integration/test_representation.py

@@ -277,7 +277,6 @@ class TestDocumentCreationWorkflow:
             level="explicit",
             times_derived=5,
             internal_metadata={},
-            embedding=[0.1] * 1536,
         )
         db_session.add(highly_derived_doc)
 
@@ -291,7 +290,6 @@ class TestDocumentCreationWorkflow:
             level="explicit",
             times_derived=2,
             internal_metadata={},
-            embedding=[0.2] * 1536,
         )
         db_session.add(less_derived_doc)
 
@@ -325,7 +323,6 @@ class TestDocumentCreationWorkflow:
                 "message_ids": [1],
             },
             session_name="test_session",
-            embedding=[0.1] * 1536,
             created_at=datetime(2025, 1, 1, 10, 0, 0, tzinfo=timezone.utc),
         )
 
@@ -341,7 +338,6 @@ class TestDocumentCreationWorkflow:
                 "premises": ["User said they like programming"],
             },
             session_name="test_session",
-            embedding=[0.2] * 1536,
             created_at=datetime(2025, 1, 1, 10, 1, 0, tzinfo=timezone.utc),
         )
 

tests/routes/test_conclusions.py

@@ -62,7 +62,6 @@ class TestConclusionRoutes:
             observer=test_peer.name,
             observed=test_peer2.name,
             content="User prefers dark mode",
-            embedding=[0.1] * 1536,
             session_name=test_session.name,
         )
         doc2 = models.Document(
@@ -70,7 +69,6 @@ class TestConclusionRoutes:
             observer=test_peer.name,
             observed=test_peer2.name,
             content="User works late at night",
-            embedding=[0.2] * 1536,
             session_name=test_session.name,
         )
         db_session.add_all([doc1, doc2])
@@ -170,7 +168,6 @@ class TestConclusionRoutes:
             observer=test_peer.name,
             observed=test_peer2.name,
             content="Peer1 observes Peer2",
-            embedding=[0.1] * 1536,
             session_name=test_session.name,
         )
         doc2 = models.Document(
@@ -178,7 +175,6 @@ class TestConclusionRoutes:
             observer=test_peer2.name,
             observed=test_peer3.name,
             content="Peer2 observes Peer3",
-            embedding=[0.2] * 1536,
             session_name=test_session.name,
         )
         db_session.add_all([doc1, doc2])
@@ -350,9 +346,25 @@ class TestConclusionRoutes:
         db_session.add(test_session)
         await db_session.commit()
 
-        # Create collection
-        await self._create_collection(
-            db_session, test_workspace.name, test_peer.name, test_peer2.name
+        # Create test conclusions via API (this populates the vector store)
+        _create_response = client.post(
+            f"/v2/workspaces/{test_workspace.name}/conclusions",
+            json={
+                "conclusions": [
+                    {
+                        "content": "User loves pizza and pasta",
+                        "observer_id": test_peer.name,
+                        "observed_id": test_peer2.name,
+                        "session_id": test_session.name,
+                    },
+                    {
+                        "content": "User dislikes vegetables",
+                        "observer_id": test_peer.name,
+                        "observed_id": test_peer2.name,
+                        "session_id": test_session.name,
+                    },
+                ]
+            },
         )
 
         # Create test conclusions
@@ -640,7 +652,9 @@ class TestConclusionRoutes:
 
         stmt = select(models.Document).where(models.Document.id == conclusion_id)
         result = await db_session.execute(stmt)
-        assert result.scalar_one_or_none() is None
+        doc = result.scalar_one_or_none()
+        assert doc is not None
+        assert doc.deleted_at is not None
 
     @pytest.mark.asyncio
     async def test_delete_conclusion_not_found(

tests/utils/test_agent_tools.py

@@ -11,6 +11,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from src import crud, models
+from src.config import settings
 from src.utils.agent_tools import (
     ToolContext,
     _handle_create_observations,  # pyright: ignore[reportPrivateUsage]
@@ -252,7 +253,7 @@ class TestDeleteObservations:
         tool_test_data: Any,
         make_tool_context: Callable[..., ToolContext],
     ):
-        """Successfully deletes observation by ID."""
+        """Successfully soft-deletes observation by ID."""
         _, _, _, _, _, documents = tool_test_data
         ctx = make_tool_context(include_observation_ids=True)
 
@@ -261,10 +262,11 @@ class TestDeleteObservations:
 
         assert "Deleted 1 observations" in result
 
-        # Verify deletion
+        # Verify soft-deletion (document still exists but has deleted_at timestamp)
         stmt = select(models.Document).where(models.Document.id == doc_id)
         doc = (await db_session.execute(stmt)).scalar_one_or_none()
-        assert doc is None
+        assert doc is not None
+        assert doc.deleted_at is not None
 
     async def test_delete_invalid_id_handled_gracefully(
         self, make_tool_context: Callable[..., ToolContext]
@@ -311,9 +313,14 @@ class TestSearchMemory:
     """Tests for _handle_search_memory."""
 
     async def test_returns_matching_observations(
-        self, make_tool_context: Callable[..., ToolContext]
+        self,
+        make_tool_context: Callable[..., ToolContext],
+        monkeypatch: pytest.MonkeyPatch,
     ):
         """Returns observations matching semantic query."""
+        # Force pgvector queries since test documents are created directly in postgres
+        monkeypatch.setattr(settings.VECTOR_STORE, "MIGRATED", False)
+
         ctx = make_tool_context()
 
         result = await _handle_search_memory(ctx, {"query": "coffee preferences"})