fix: clean-up and migration test

.env.template

@@ -189,3 +189,27 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
 # VECTOR_STORE_TURBOPUFFER_API_KEY=
 # VECTOR_STORE_TURBOPUFFER_REGION=
 # VECTOR_STORE_LANCEDB_PATH="./lancedb_data"
+
+# Vector store settings
+# Primary vector store type: "pgvector", "turbopuffer", or "lancedb"
+VECTOR_STORE_PRIMARY_TYPE=pgvector
+
+# Secondary vector store type (optional)
+# When set, enables dual-write and fallback read
+# VECTOR_STORE_SECONDARY_TYPE=
+
+# Global namespace prefix for all vector namespaces
+# Namespaces follow the pattern:
+# - Documents: {NAMESPACE}.{workspace}.{observer}.{observed}
+# - Messages: {NAMESPACE}.{workspace}.messages
+VECTOR_STORE_NAMESPACE=honcho
+
+# Embedding dimensions (default: 1536 for OpenAI text-embedding-3-small)
+VECTOR_STORE_DIMENSIONS=1536
+
+# Turbopuffer-specific settings (required if PRIMARY_TYPE or SECONDARY_TYPE is "turbopuffer")
+# VECTOR_STORE_TURBOPUFFER_API_KEY=your-turbopuffer-api-key
+# VECTOR_STORE_TURBOPUFFER_REGION=us-east-1
+
+# LanceDB-specific settings (local embedded mode)
+VECTOR_STORE_LANCEDB_PATH=./lancedb_data

config.toml.example

@@ -143,8 +143,11 @@ DEFAULT_TTL_SECONDS = 300
 DEFAULT_LOCK_TTL_SECONDS = 5
 
 # Vector store settings
-# VECTOR_STORE_TYPE = "lancedb"
-# VECTOR_STORE_NAMESPACE = "honcho"
-# VECTOR_STORE_TURBOPUFFER_API_KEY =
-# VECTOR_STORE_TURBOPUFFER_REGION =
-# VECTOR_STORE_LANCEDB_PATH = "./lancedb_data"
+[vector_store]
+PRIMARY_TYPE = "pgvector"
+# SECONDARY_TYPE = "lancedb"
+NAMESPACE = "honcho"
+DIMENSIONS = 1536
+# TURBOPUFFER_API_KEY = "your-turbopuffer-api-key"
+# TURBOPUFFER_REGION = "us-east-1"
+LANCEDB_PATH = "./lancedb_data"

migrations/versions/f1a2b3c4d5e6_support_external_embeddings.py

@@ -20,7 +20,7 @@ import sqlalchemy as sa
 from alembic import op
 from pgvector.sqlalchemy import Vector
 
-from migrations.utils import column_exists, get_schema
+from migrations.utils import column_exists, get_schema, index_exists
 
 # revision identifiers, used by Alembic.
 revision: str = "f1a2b3c4d5e6"
@@ -115,19 +115,13 @@ def upgrade() -> None:
 
     # Add composite index for efficient reconciliation queries after both columns exist
     # Reconciliation orders by: WHERE sync_state='pending' ORDER BY last_sync_at
-    if column_exists("documents", "sync_state", inspector) and column_exists(
-        "documents", "last_sync_at", inspector
-    ):
-        # Check if index already exists
-        indexes = inspector.get_indexes("documents", schema=schema)
-        index_names = [idx["name"] for idx in indexes]
-        if "ix_documents_sync_state_last_sync_at" not in index_names:
-            op.create_index(
-                "ix_documents_sync_state_last_sync_at",
-                "documents",
-                ["sync_state", "last_sync_at"],
-                schema=schema,
-            )
+    if not index_exists("documents", "ix_documents_sync_state_last_sync_at", inspector):
+        op.create_index(
+            "ix_documents_sync_state_last_sync_at",
+            "documents",
+            ["sync_state", "last_sync_at"],
+            schema=schema,
+        )
 
     # Add sync state columns to message_embeddings table
     if not column_exists("message_embeddings", "sync_state", inspector):
@@ -173,19 +167,15 @@ def upgrade() -> None:
 
     # Add composite index for efficient reconciliation queries after both columns exist
     # Reconciliation orders by: WHERE sync_state='pending' ORDER BY last_sync_at
-    if column_exists("message_embeddings", "sync_state", inspector) and column_exists(
-        "message_embeddings", "last_sync_at", inspector
+    if not index_exists(
+        "message_embeddings", "ix_message_embeddings_sync_state_last_sync_at", inspector
     ):
-        # Check if index already exists
-        indexes = inspector.get_indexes("message_embeddings", schema=schema)
-        index_names = [idx["name"] for idx in indexes]
-        if "ix_message_embeddings_sync_state_last_sync_at" not in index_names:
-            op.create_index(
-                "ix_message_embeddings_sync_state_last_sync_at",
-                "message_embeddings",
-                ["sync_state", "last_sync_at"],
-                schema=schema,
-            )
+        op.create_index(
+            "ix_message_embeddings_sync_state_last_sync_at",
+            "message_embeddings",
+            ["sync_state", "last_sync_at"],
+            schema=schema,
+        )
 
 
 def downgrade() -> None:

src/config.py

@@ -394,13 +394,11 @@ class VectorStoreSettings(HonchoSettings):
 
     @model_validator(mode="after")
     def _require_api_key_for_turbopuffer(self) -> "VectorStoreSettings":
-        if self.PRIMARY_TYPE == "turbopuffer" and not self.TURBOPUFFER_API_KEY:
+        if (
+            self.PRIMARY_TYPE == "turbopuffer" or self.SECONDARY_TYPE == "turbopuffer"
+        ) and not self.TURBOPUFFER_API_KEY:
             raise ValueError(
-                "VECTOR_STORE_TURBOPUFFER_API_KEY must be set when PRIMARY_TYPE is 'turbopuffer'"
-            )
-        if self.SECONDARY_TYPE == "turbopuffer" and not self.TURBOPUFFER_API_KEY:
-            raise ValueError(
-                "VECTOR_STORE_TURBOPUFFER_API_KEY must be set when SECONDARY_TYPE is 'turbopuffer'"
+                "VECTOR_STORE_TURBOPUFFER_API_KEY must be set when PRIMARY_TYPE or SECONDARY_TYPE is 'turbopuffer'"
             )
         return self
 

src/exceptions.py

@@ -109,26 +109,6 @@ class FileProcessingError(HonchoException):
     detail = "File processing error"
 
 
-@final
-class PartialVectorSyncException(HonchoException):
-    """
-    Exception raised when vector upsert partially succeeds.
-
-    This indicates the primary store succeeded but secondary store failed.
-    The data is queryable but not fully replicated.
-    """
-
-    status_code = 500
-    detail = "Vector partially synced to primary store only"
-
-    def __init__(self, primary_success: bool, secondary_error: Exception):
-        self.primary_success = primary_success
-        self.secondary_error = secondary_error
-        super().__init__(
-            f"Partial sync: primary={'succeeded' if primary_success else 'failed'}, secondary failed with: {secondary_error}"
-        )
-
-
 class LLMError(Exception):
     """Exception raised when an LLM call fails.
 

src/vector_store/lancedb.py

@@ -1,8 +1,7 @@
 """
 LanceDB vector store implementation.
 
-This module provides a LanceDB-based implementation of the VectorStore interface
-for use in self-hosted deployments of Honcho.
+This module provides a LanceDB-based implementation of the VectorStore interface.
 """
 
 import asyncio

src/vector_store/turbopuffer.py

@@ -1,8 +1,7 @@
 """
 Turbopuffer vector store implementation.
 
-This module provides a Turbopuffer-based implementation of the VectorStore interface
-for use in managed deployments of Honcho.
+This module provides a Turbopuffer-based implementation of the VectorStore interface.
 """
 
 import logging
@@ -30,7 +29,6 @@ class TurbopufferVectorStore(VectorStore):
     """
     Turbopuffer implementation of the VectorStore interface.
 
-    Uses Turbopuffer's async Python SDK for vector operations.
     Each namespace corresponds to either:
     - A document collection: {prefix}.{workspace}.{observer}.{observed}
     - A workspace's message embeddings: {prefix}.{workspace}.messages

tests/alembic/revisions/test_f1a2b3c4d5e6_support_external_embeddings.py

@@ -5,13 +5,40 @@ from __future__ import annotations
 from tests.alembic.registry import register_after_upgrade, register_before_upgrade
 from tests.alembic.verifier import MigrationVerifier
 
+# Indexes created by this migration
+INDEXES = (
+    ("documents", "ix_documents_deleted_at"),
+    ("documents", "ix_documents_sync_state"),
+    ("documents", "ix_documents_sync_state_last_sync_at"),
+    ("message_embeddings", "ix_message_embeddings_sync_state"),
+    ("message_embeddings", "ix_message_embeddings_sync_state_last_sync_at"),
+)
+
 
 @register_before_upgrade("f1a2b3c4d5e6")
 def prepare_support_external_embeddings(
     verifier: MigrationVerifier,
 ) -> None:
     """Seed state and assertions before upgrading to f1a2b3c4d5e6."""
+    # Embedding columns should be NOT NULL before migration
     verifier.assert_column_exists("message_embeddings", "embedding", nullable=False)
+    verifier.assert_column_exists("documents", "embedding", nullable=False)
+
+    # Soft delete column should not exist
+    verifier.assert_column_exists("documents", "deleted_at", exists=False)
+
+    # Sync state columns should not exist on documents
+    verifier.assert_column_exists("documents", "sync_state", exists=False)
+    verifier.assert_column_exists("documents", "last_sync_at", exists=False)
+    verifier.assert_column_exists("documents", "sync_attempts", exists=False)
+
+    # Sync state columns should not exist on message_embeddings
+    verifier.assert_column_exists("message_embeddings", "sync_state", exists=False)
+    verifier.assert_column_exists("message_embeddings", "last_sync_at", exists=False)
+    verifier.assert_column_exists("message_embeddings", "sync_attempts", exists=False)
+
+    # Indexes should not exist
+    verifier.assert_indexes_not_exist(INDEXES)
 
 
 @register_after_upgrade("f1a2b3c4d5e6")
@@ -19,4 +46,22 @@ def verify_support_external_embeddings(
     verifier: MigrationVerifier,
 ) -> None:
     """Add assertions validating the effects of f1a2b3c4d5e6."""
+    # Embedding columns should now be nullable
     verifier.assert_column_exists("message_embeddings", "embedding", nullable=True)
+    verifier.assert_column_exists("documents", "embedding", nullable=True)
+
+    # Soft delete column should exist and be nullable
+    verifier.assert_column_exists("documents", "deleted_at", nullable=True)
+
+    # Sync state columns should exist on documents
+    verifier.assert_column_exists("documents", "sync_state", nullable=False)
+    verifier.assert_column_exists("documents", "last_sync_at", nullable=True)
+    verifier.assert_column_exists("documents", "sync_attempts", nullable=False)
+
+    # Sync state columns should exist on message_embeddings
+    verifier.assert_column_exists("message_embeddings", "sync_state", nullable=False)
+    verifier.assert_column_exists("message_embeddings", "last_sync_at", nullable=True)
+    verifier.assert_column_exists("message_embeddings", "sync_attempts", nullable=False)
+
+    # All indexes should exist
+    verifier.assert_indexes_exist(INDEXES)