diff --git a/src/config.py b/src/config.py index 5fb2769d..7d3854d8 100644 --- a/src/config.py +++ b/src/config.py @@ -543,9 +543,10 @@ class VectorStoreSettings(HonchoSettings): MIGRATED: bool = False # Global namespace prefix for all vector namespaces - # Namespaces follow the pattern: - # - Documents: {NAMESPACE}.{workspace}.{observer}.{observed} - # - Messages: {NAMESPACE}.{workspace}.messages + # Namespaces follow the pattern: {NAMESPACE}.{type}.{hash} + # where hash is a base64url-encoded SHA-256 of the workspace/peer names + # - Documents: {NAMESPACE}.doc.{hash(workspace, observer, observed)} + # - Messages: {NAMESPACE}.msg.{hash(workspace)} NAMESPACE: str = "honcho" DIMENSIONS: Annotated[ diff --git a/src/vector_store/__init__.py b/src/vector_store/__init__.py index 3d19b6e8..7a04eaa2 100644 --- a/src/vector_store/__init__.py +++ b/src/vector_store/__init__.py @@ -2,6 +2,8 @@ Vector store abstraction layer for Honcho. """ +import base64 +import hashlib from abc import ABC, abstractmethod from functools import cache from typing import Any, ClassVar, Literal @@ -11,6 +13,23 @@ from pydantic import BaseModel, ConfigDict, Field from src.config import settings +def _hash_namespace_components(*parts: str) -> str: + """ + Hash namespace components to create a fixed-length, valid namespace suffix. + + Turbopuffer requires namespaces to match [A-Za-z0-9-_.]{1,128}. + This function hashes the variable parts (workspace, observer, observed) + to ensure the namespace fits within length limits and uses only valid chars. + + Returns: + A 43-character base64url-encoded SHA-256 hash (no padding). + (SHA-256 = 32 bytes, base64 = ceil(32 * 4 / 3) = 43 chars without padding) + """ + combined = ".".join(parts) + hash_bytes = hashlib.sha256(combined.encode("utf-8")).digest() + return base64.urlsafe_b64encode(hash_bytes).decode("ascii").rstrip("=") + + class VectorRecord(BaseModel): """A single vector record to be stored in the vector store.""" @@ -46,13 +65,14 @@ class VectorStore(ABC): """ Abstract base class for vector store implementations. - All vector operations are namespace-scoped. Namespaces map to: - - Document embeddings: {prefix}.{workspace}.{observer}.{observed} (per collection) - - Message embeddings: {prefix}.{workspace}.messages (per workspace) + All vector operations are namespace-scoped. Namespaces are generated via + get_vector_namespace() which hashes workspace/peer names to ensure: + - Total length fits within Turbopuffer's 128 char limit + - Only valid characters [A-Za-z0-9-_.] are used - Note: Period (.) is used as the delimiter since vector stores (Turbopuffer, LanceDB) - only allow [A-Za-z0-9-_.] in namespace names, and period is not allowed in - workspace/peer IDs (which only allow [A-Za-z0-9_-]). + Namespace format: {prefix}.{type}.{hash} + - Document embeddings: {prefix}.doc.{hash} + - Message embeddings: {prefix}.msg.{hash} """ namespace_prefix: str @@ -82,17 +102,20 @@ class VectorStore(ABC): Returns: Namespace string in format: - - document: {prefix}.{workspace}.{observer}.{observed} - - message: {prefix}.{workspace}.messages + - document: {prefix}.doc.{hash} + - message: {prefix}.msg.{hash} + where hash is derived from the workspace/peer names. """ if namespace_type == "document": if observer is None or observed is None: raise ValueError( "observer and observed are required for document namespaces" ) - return f"{self.namespace_prefix}.{workspace_name}.{observer}.{observed}" - if namespace_type == "message": - return f"{self.namespace_prefix}.{workspace_name}.messages" + hash_suffix = _hash_namespace_components(workspace_name, observer, observed) + return f"{self.namespace_prefix}.doc.{hash_suffix}" + elif namespace_type == "message": + hash_suffix = _hash_namespace_components(workspace_name) + return f"{self.namespace_prefix}.msg.{hash_suffix}" # === Core operations === @abstractmethod @@ -231,4 +254,5 @@ __all__ = [ "get_external_vector_store", "close_external_vector_store", "upsert_with_retry", + "_hash_namespace_components", ] diff --git a/src/vector_store/lancedb.py b/src/vector_store/lancedb.py index 3c35e2c8..0853a98d 100644 --- a/src/vector_store/lancedb.py +++ b/src/vector_store/lancedb.py @@ -109,22 +109,25 @@ class LanceDBVectorStore(VectorStore): """ Infer standard metadata columns based on namespace structure. - Namespaces: - - Documents: {prefix}.{workspace}.{observer}.{observed} - - Messages: {prefix}.{workspace}.messages + Namespaces use format: {prefix}.{type}.{hash} + - Documents: {prefix}.doc.{hash} + - Messages: {prefix}.msg.{hash} """ parts = namespace.split(".") if len(parts) < 3: return [] - if parts[-1] == "messages": + # Second-to-last part indicates the type (doc or msg) + ns_type = parts[-2] + + if ns_type == "msg": return [ pa.field("message_id", pa.string(), nullable=True), pa.field("session_name", pa.string(), nullable=True), pa.field("peer_name", pa.string(), nullable=True), ] - if len(parts) == 4: + if ns_type == "doc": return [ pa.field("workspace_name", pa.string(), nullable=True), pa.field("observer", pa.string(), nullable=True), diff --git a/src/vector_store/turbopuffer.py b/src/vector_store/turbopuffer.py index fe209d77..39b93f5f 100644 --- a/src/vector_store/turbopuffer.py +++ b/src/vector_store/turbopuffer.py @@ -30,9 +30,8 @@ class TurbopufferVectorStore(VectorStore): """ Turbopuffer implementation of the VectorStore interface. - Each namespace corresponds to either: - - A document collection: {prefix}.{workspace}.{observer}.{observed} - - A workspace's message embeddings: {prefix}.{workspace}.messages + Namespaces are generated via get_vector_namespace() which hashes + the variable components to fit Turbopuffer's [A-Za-z0-9-_.]{1,128} limit. """ tpuf: AsyncTurbopuffer diff --git a/tests/conftest.py b/tests/conftest.py index 3dc4d7db..75aade07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -404,7 +404,12 @@ def mock_vector_store(): """Mock vector store operations for testing""" from unittest.mock import AsyncMock, MagicMock - from src.vector_store import VectorQueryResult, VectorRecord, VectorUpsertResult + from src.vector_store import ( + VectorQueryResult, + VectorRecord, + VectorUpsertResult, + _hash_namespace_components, # pyright: ignore[reportPrivateUsage] + ) # Create a mock vector store that stores vectors in memory vector_storage: dict[str, dict[str, tuple[list[float], dict[str, Any]]]] = {} @@ -464,14 +469,15 @@ def mock_vector_store(): observer: str | None = None, observed: str | None = None, ) -> str: + # Uses real hash function for consistency with production if namespace_type == "document": if observer is None or observed is None: raise ValueError( "observer and observed are required for document namespaces" ) - return f"honcho2345.{workspace_name}.{observer}.{observed}" + return f"honcho2345.doc.{_hash_namespace_components(workspace_name, observer, observed)}" if namespace_type == "message": - return f"honcho2345.{workspace_name}.messages" + return f"honcho2345.msg.{_hash_namespace_components(workspace_name)}" raise ValueError(f"Unknown namespace type: {namespace_type}") mock_vs.get_vector_namespace = mock_get_vector_namespace diff --git a/tests/deriver/test_vector_reconciliation.py b/tests/deriver/test_vector_reconciliation.py index 4ee93e1e..857a0fda 100644 --- a/tests/deriver/test_vector_reconciliation.py +++ b/tests/deriver/test_vector_reconciliation.py @@ -24,7 +24,12 @@ from src.reconciler.sync_vectors import ( _sync_message_embeddings, # pyright: ignore[reportPrivateUsage] run_vector_reconciliation_cycle, ) -from src.vector_store import VectorRecord, VectorStore, VectorUpsertResult +from src.vector_store import ( + VectorRecord, + VectorStore, + VectorUpsertResult, + _hash_namespace_components, # pyright: ignore[reportPrivateUsage] +) @pytest.mark.asyncio @@ -77,7 +82,7 @@ class TestStateTransitions: # Mock vector store to succeed mock_vector_store = MagicMock(spec=VectorStore) mock_vector_store.get_vector_namespace = MagicMock( - return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}" + return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}" ) mock_vector_store.upsert_many = AsyncMock( return_value=VectorUpsertResult(ok=True) @@ -139,7 +144,7 @@ class TestStateTransitions: # Mock vector store to fail with exception mock_vector_store = MagicMock(spec=VectorStore) mock_vector_store.get_vector_namespace = MagicMock( - return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}" + return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}" ) mock_vector_store.upsert_many = AsyncMock( side_effect=Exception("Vector store failed") @@ -200,7 +205,7 @@ class TestStateTransitions: # Mock vector store to fail with exception mock_vector_store = MagicMock(spec=VectorStore) mock_vector_store.get_vector_namespace = MagicMock( - return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}" + return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}" ) mock_vector_store.upsert_many = AsyncMock( side_effect=Exception("Vector store failed") @@ -302,7 +307,7 @@ class TestBatchProcessing: def mock_get_namespace( _namespace_type: str, workspace: str, observer: str, observed: str ) -> str: - return f"honcho.{workspace}.{observer}.{observed}" + return f"honcho.doc.{_hash_namespace_components(workspace, observer, observed)}" async def mock_upsert( namespace: str, vectors: list[VectorRecord] @@ -323,8 +328,8 @@ class TestBatchProcessing: assert failed == 0 # Verify namespaces - expected_ns1 = f"honcho.{workspace.name}.{peer1.name}.{peer1.name}" - expected_ns2 = f"honcho.{workspace.name}.{peer1.name}.{peer2.name}" + expected_ns1 = f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}" + expected_ns2 = f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer2.name)}" assert expected_ns1 in namespace_calls assert expected_ns2 in namespace_calls @@ -433,7 +438,7 @@ class TestReEmbedding: # Mock vector store mock_vector_store = MagicMock(spec=VectorStore) mock_vector_store.get_vector_namespace = MagicMock( - return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}" + return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}" ) mock_vector_store.upsert_many = AsyncMock( return_value=VectorUpsertResult(ok=True) @@ -505,7 +510,7 @@ class TestReEmbedding: # Mock vector store mock_vector_store = MagicMock(spec=VectorStore) mock_vector_store.get_vector_namespace = MagicMock( - return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}" + return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}" ) mock_vector_store.upsert_many = AsyncMock( return_value=VectorUpsertResult(ok=True)