RomanNum3ral
/
honcho
mirror of https://github.com/plastic-labs/honcho.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: hash namespace

This commit is contained in:
Rajat Ahuja 2026-01-16 11:49:03 -05:00
parent 670ab6b7d5
commit bc0e8efd2b
6 changed files with 72 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/config.py
View File

@ -543,9 +543,10 @@ class VectorStoreSettings(HonchoSettings):
MIGRATED: bool = False
# Global namespace prefix for all vector namespaces
# Namespaces follow the pattern:
# - Documents: {NAMESPACE}.{workspace}.{observer}.{observed}
# - Messages: {NAMESPACE}.{workspace}.messages
# Namespaces follow the pattern: {NAMESPACE}.{type}.{hash}
# where hash is a base64url-encoded SHA-256 of the workspace/peer names
# - Documents: {NAMESPACE}.doc.{hash(workspace, observer, observed)}
# - Messages: {NAMESPACE}.msg.{hash(workspace)}
NAMESPACE: str = "honcho"
DIMENSIONS: Annotated[

46
src/vector_store/__init__.py
View File

@ -2,6 +2,8 @@
Vector store abstraction layer for Honcho.
"""
import base64
import hashlib
from abc import ABC, abstractmethod
from functools import cache
from typing import Any, ClassVar, Literal
@ -11,6 +13,23 @@ from pydantic import BaseModel, ConfigDict, Field
from src.config import settings
def _hash_namespace_components(*parts: str) -> str:
"""
Hash namespace components to create a fixed-length, valid namespace suffix.
Turbopuffer requires namespaces to match [A-Za-z0-9-_.]{1,128}.
This function hashes the variable parts (workspace, observer, observed)
to ensure the namespace fits within length limits and uses only valid chars.
Returns:
A 43-character base64url-encoded SHA-256 hash (no padding).
(SHA-256 = 32 bytes, base64 = ceil(32 * 4 / 3) = 43 chars without padding)
"""
combined = ".".join(parts)
hash_bytes = hashlib.sha256(combined.encode("utf-8")).digest()
return base64.urlsafe_b64encode(hash_bytes).decode("ascii").rstrip("=")
class VectorRecord(BaseModel):
"""A single vector record to be stored in the vector store."""
@ -46,13 +65,14 @@ class VectorStore(ABC):
"""
Abstract base class for vector store implementations.
All vector operations are namespace-scoped. Namespaces map to:
- Document embeddings: {prefix}.{workspace}.{observer}.{observed} (per collection)
- Message embeddings: {prefix}.{workspace}.messages (per workspace)
All vector operations are namespace-scoped. Namespaces are generated via
get_vector_namespace() which hashes workspace/peer names to ensure:
- Total length fits within Turbopuffer's 128 char limit
- Only valid characters [A-Za-z0-9-_.] are used
Note: Period (.) is used as the delimiter since vector stores (Turbopuffer, LanceDB)
only allow [A-Za-z0-9-_.] in namespace names, and period is not allowed in
workspace/peer IDs (which only allow [A-Za-z0-9_-]).
Namespace format: {prefix}.{type}.{hash}
- Document embeddings: {prefix}.doc.{hash}
- Message embeddings: {prefix}.msg.{hash}
"""
namespace_prefix: str
@ -82,17 +102,20 @@ class VectorStore(ABC):
Returns:
Namespace string in format:
- document: {prefix}.{workspace}.{observer}.{observed}
- message: {prefix}.{workspace}.messages
- document: {prefix}.doc.{hash}
- message: {prefix}.msg.{hash}
where hash is derived from the workspace/peer names.
"""
if namespace_type == "document":
if observer is None or observed is None:
raise ValueError(
"observer and observed are required for document namespaces"
)
return f"{self.namespace_prefix}.{workspace_name}.{observer}.{observed}"
if namespace_type == "message":
return f"{self.namespace_prefix}.{workspace_name}.messages"
hash_suffix = _hash_namespace_components(workspace_name, observer, observed)
return f"{self.namespace_prefix}.doc.{hash_suffix}"
elif namespace_type == "message":
hash_suffix = _hash_namespace_components(workspace_name)
return f"{self.namespace_prefix}.msg.{hash_suffix}"
# === Core operations ===
@abstractmethod
@ -231,4 +254,5 @@ __all__ = [
"get_external_vector_store",
"close_external_vector_store",
"upsert_with_retry",
"_hash_namespace_components",
]

13
src/vector_store/lancedb.py
View File

@ -109,22 +109,25 @@ class LanceDBVectorStore(VectorStore):
"""
Infer standard metadata columns based on namespace structure.
Namespaces:
- Documents: {prefix}.{workspace}.{observer}.{observed}
- Messages: {prefix}.{workspace}.messages
Namespaces use format: {prefix}.{type}.{hash}
- Documents: {prefix}.doc.{hash}
- Messages: {prefix}.msg.{hash}
"""
parts = namespace.split(".")
if len(parts) < 3:
return []
if parts[-1] == "messages":
# Second-to-last part indicates the type (doc or msg)
ns_type = parts[-2]
if ns_type == "msg":
return [
pa.field("message_id", pa.string(), nullable=True),
pa.field("session_name", pa.string(), nullable=True),
pa.field("peer_name", pa.string(), nullable=True),
]
if len(parts) == 4:
if ns_type == "doc":
return [
pa.field("workspace_name", pa.string(), nullable=True),
pa.field("observer", pa.string(), nullable=True),

5
src/vector_store/turbopuffer.py
View File

@ -30,9 +30,8 @@ class TurbopufferVectorStore(VectorStore):
"""
Turbopuffer implementation of the VectorStore interface.
Each namespace corresponds to either:
- A document collection: {prefix}.{workspace}.{observer}.{observed}
- A workspace's message embeddings: {prefix}.{workspace}.messages
Namespaces are generated via get_vector_namespace() which hashes
the variable components to fit Turbopuffer's [A-Za-z0-9-_.]{1,128} limit.
"""
tpuf: AsyncTurbopuffer

12
tests/conftest.py
View File

@ -404,7 +404,12 @@ def mock_vector_store():
"""Mock vector store operations for testing"""
from unittest.mock import AsyncMock, MagicMock
from src.vector_store import VectorQueryResult, VectorRecord, VectorUpsertResult
from src.vector_store import (
VectorQueryResult,
VectorRecord,
VectorUpsertResult,
_hash_namespace_components, # pyright: ignore[reportPrivateUsage]
)
# Create a mock vector store that stores vectors in memory
vector_storage: dict[str, dict[str, tuple[list[float], dict[str, Any]]]] = {}
@ -464,14 +469,15 @@ def mock_vector_store():
observer: str | None = None,
observed: str | None = None,
) -> str:
# Uses real hash function for consistency with production
if namespace_type == "document":
if observer is None or observed is None:
raise ValueError(
"observer and observed are required for document namespaces"
)
return f"honcho2345.{workspace_name}.{observer}.{observed}"
return f"honcho2345.doc.{_hash_namespace_components(workspace_name, observer, observed)}"
if namespace_type == "message":
return f"honcho2345.{workspace_name}.messages"
return f"honcho2345.msg.{_hash_namespace_components(workspace_name)}"
raise ValueError(f"Unknown namespace type: {namespace_type}")
mock_vs.get_vector_namespace = mock_get_vector_namespace

23
tests/deriver/test_vector_reconciliation.py
View File

@ -24,7 +24,12 @@ from src.reconciler.sync_vectors import (
_sync_message_embeddings, # pyright: ignore[reportPrivateUsage]
run_vector_reconciliation_cycle,
)
from src.vector_store import VectorRecord, VectorStore, VectorUpsertResult
from src.vector_store import (
VectorRecord,
VectorStore,
VectorUpsertResult,
_hash_namespace_components, # pyright: ignore[reportPrivateUsage]
)
@pytest.mark.asyncio
@ -77,7 +82,7 @@ class TestStateTransitions:
# Mock vector store to succeed
mock_vector_store = MagicMock(spec=VectorStore)
mock_vector_store.get_vector_namespace = MagicMock(
return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}"
return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
)
mock_vector_store.upsert_many = AsyncMock(
return_value=VectorUpsertResult(ok=True)
@ -139,7 +144,7 @@ class TestStateTransitions:
# Mock vector store to fail with exception
mock_vector_store = MagicMock(spec=VectorStore)
mock_vector_store.get_vector_namespace = MagicMock(
return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}"
return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
)
mock_vector_store.upsert_many = AsyncMock(
side_effect=Exception("Vector store failed")
@ -200,7 +205,7 @@ class TestStateTransitions:
# Mock vector store to fail with exception
mock_vector_store = MagicMock(spec=VectorStore)
mock_vector_store.get_vector_namespace = MagicMock(
return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}"
return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
)
mock_vector_store.upsert_many = AsyncMock(
side_effect=Exception("Vector store failed")
@ -302,7 +307,7 @@ class TestBatchProcessing:
def mock_get_namespace(
_namespace_type: str, workspace: str, observer: str, observed: str
) -> str:
return f"honcho.{workspace}.{observer}.{observed}"
return f"honcho.doc.{_hash_namespace_components(workspace, observer, observed)}"
async def mock_upsert(
namespace: str, vectors: list[VectorRecord]
@ -323,8 +328,8 @@ class TestBatchProcessing:
assert failed == 0
# Verify namespaces
expected_ns1 = f"honcho.{workspace.name}.{peer1.name}.{peer1.name}"
expected_ns2 = f"honcho.{workspace.name}.{peer1.name}.{peer2.name}"
expected_ns1 = f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
expected_ns2 = f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer2.name)}"
assert expected_ns1 in namespace_calls
assert expected_ns2 in namespace_calls
@ -433,7 +438,7 @@ class TestReEmbedding:
# Mock vector store
mock_vector_store = MagicMock(spec=VectorStore)
mock_vector_store.get_vector_namespace = MagicMock(
return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}"
return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
)
mock_vector_store.upsert_many = AsyncMock(
return_value=VectorUpsertResult(ok=True)
@ -505,7 +510,7 @@ class TestReEmbedding:
# Mock vector store
mock_vector_store = MagicMock(spec=VectorStore)
mock_vector_store.get_vector_namespace = MagicMock(
return_value=f"honcho.{workspace.name}.{peer1.name}.{peer1.name}"
return_value=f"honcho.doc.{_hash_namespace_components(workspace.name, peer1.name, peer1.name)}"
)
mock_vector_store.upsert_many = AsyncMock(
return_value=VectorUpsertResult(ok=True)