RomanNum3ral
/
honcho
mirror of https://github.com/plastic-labs/honcho.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into ben/dev-1163

This commit is contained in:
Benjamin McCormick 2025-11-13 12:24:39 -05:00
parent 090d871c42 c8d75c77df
commit 5226b84cf3
93 changed files with 4666 additions and 835 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.env.template
View File

@ -21,6 +21,8 @@ LOG_LEVEL=INFO
# LANGFUSE_HOST=
# LANGFUSE_PUBLIC_KEY=
# NAMESPACE="honcho"
# =============================================================================
# Database Settings (REQUIRED)
# =============================================================================
@ -83,6 +85,7 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
# DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5
# DERIVER_PROVIDER=google
# DERIVER_MODEL=gemini-2.0-flash-lite
# DERIVER_DEDUPLICATE=true
# DERIVER_MAX_OUTPUT_TOKENS=2500
# only applied when using Anthropic as provider
# DERIVER_THINKING_BUDGET_TOKENS=1024
@ -166,5 +169,14 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
# =============================================================================
# Metrics (Optional)
# =============================================================================
# ENABLED=false
# NAMESPACE=honcho
# METRICS_ENABLED=false
# METRICS_NAMESPACE=honcho
# =============================================================================
# Cache
# =============================================================================
# CACHE_ENABLED=false
# CACHE_URL="redis://localhost:6379/0"
# CACHE_NAMESPACE="honcho"
# CACHE_DEFAULT_TTL_SECONDS=300
# CACHE_DEFAULT_LOCK_TTL_SECONDS=5

1
.gitignore vendored
View File

@ -4,6 +4,7 @@ api/docker-compose.yml
*.db
data
redis-data
docker-compose.yml
compose.yml

6
.pre-commit-config.yaml
View File

@ -74,11 +74,11 @@ repos:
# Run Alembic tests only when migrations change
- id: pytest-alembic
name: pytest (alembic migrations)
entry: uv run pytest tests/alembic/
entry: uv run python scripts/run_alembic_tests.py
language: system
files: ^(migrations/.*\.py|tests/alembic/.*\.py)$
files: ^(migrations/versions/.*\.py|tests/alembic/.*\.py)$
stages: [pre-push]
pass_filenames: false
pass_filenames: true
require_serial: true
# Ensure each alembic migration revision has a corresponding test file

13
CHANGELOG.md
View File

@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
## [2.4.2] - 2025-11-03
### Fixed
- Langfuse tracing to have readable waterfalls
- Alembic Migrations to match models.py
- message_in_seq correctly included in webhook payload
### Changed
- Alembic to always use a session pooler
- Statement timeout during alembic operations to 5 min
## [2.4.1] - 2025-10-24
### Added

9
README.md
View File

@ -8,7 +8,7 @@
---
![Static Badge](https://img.shields.io/badge/Version-2.4.1-blue)
![Static Badge](https://img.shields.io/badge/Version-2.4.2-blue)
[![PyPI version](https://img.shields.io/pypi/v/honcho-ai.svg)](https://pypi.org/project/honcho-ai/)
[![NPM version](https://img.shields.io/npm/v/@honcho-ai/sdk.svg)](https://npmjs.org/package/@honcho-ai/sdk)
[![Discord](https://img.shields.io/discord/1016845111637839922?style=flat&logo=discord&logoColor=23ffffff&label=Plastic%20Labs&labelColor=235865F2)](https://discord.gg/plasticlabs)
@ -396,13 +396,18 @@ cp config.toml.example config.toml
Then modify the values as needed. The TOML file is organized into sections:
- `[app]` - Application-level settings (log level, host, port, embedding settings)
- `[app]` - Application-level settings (log level, session limits, embedding settings, metrics)
- `[db]` - Database connection and pool settings
- `[auth]` - Authentication configuration
- `[cache]` - Redis cache configuration
- `[llm]` - LLM provider API keys and general settings
- `[dialectic]` - Dialectic API configuration (provider, model, search settings)
- `[deriver]` - Background worker settings and theory of mind configuration
- `[peer_card]` - Peer card generation settings
- `[summary]` - Session summarization settings
- `[dream]` - Dream processing configuration
- `[webhook]` - Webhook configuration
- `[metrics]` - Metrics collection settings
- `[sentry]` - Error tracking and monitoring settings
### Using Environment Variables

11
config.toml.example
View File

@ -15,7 +15,7 @@ MAX_EMBEDDING_TOKENS = 8192
MAX_EMBEDDING_TOKENS_PER_REQUEST = 300000
# LANGFUSE_HOST = "https://api.langfuse.com"
# LANGFUSE_PUBLIC_KEY = "your-public-key-here"
NAMESPACE="honcho"
# Database settings
[db]
@ -64,6 +64,7 @@ POLLING_SLEEP_INTERVAL_SECONDS = 1.0
STALE_SESSION_TIMEOUT_MINUTES = 5
PROVIDER = "google"
MODEL = "gemini-2.5-flash-lite"
DEDUPLICATE = true
MAX_OUTPUT_TOKENS = 2500
THINKING_BUDGET_TOKENS = 1024 # only applied when using Anthropic
# BACKUP_PROVIDER = "google"
@ -132,3 +133,11 @@ MAX_WORKSPACE_LIMIT = 10
[metrics]
ENABLED = false
NAMESPACE = "honcho"
# Cache settings
[cache]
ENABLED = false
URL = "redis://localhost:6379/0"
NAMESPACE="honcho"
DEFAULT_TTL_SECONDS = 300
DEFAULT_LOCK_TTL_SECONDS = 5

12
docker-compose.yml.example
View File

@ -45,3 +45,15 @@ services:
interval: 5s
timeout: 5s
retries: 5
redis:
image: redis:8.2
restart: always
ports:
- 6379:6379
volumes:
- ./redis-data:/data
healthcheck:
test: ["CMD-SHELL", "redis-cli ping"]
interval: 5s
timeout: 5s
retries: 5

6
docs/changelog/compatibility-guide.mdx
View File

@ -8,7 +8,7 @@ This guide helps you understand which versions of Honcho's API are compatible wi
## Version Compatibility
### Honcho API v2.4.1 (Current)
### Honcho API v2.4.2 (Current)
<CardGroup cols={2}>
<Card title="TypeScript SDK" icon="js">
@ -30,12 +30,12 @@ This guide helps you understand which versions of Honcho's API are compatible wi
</CardGroup>
## Version Compatibility Table
| Honcho API Version | TypeScript SDK | Python SDK |
|-------------------|---------------|------------|
| v2.4.1 (Current) | v1.5.0 | v1.5.0 |
| v2.4.2 (Current) | v1.5.0 | v1.5.0 |
| v2.4.1 | v1.5.0 | v1.5.0 |
| v2.4.0 | v1.5.0 | v1.5.0 |
| v2.3.3 | v1.4.1 | v1.4.1 |
| v2.3.2 | v1.4.0 | v1.4.0 |

16
docs/changelog/introduction.mdx
View File

@ -27,7 +27,21 @@ Welcome to the Honcho changelog! This section documents all notable changes to t
### Honcho API and SDK Changelogs
<Tabs>
<Tab title="Honcho API">
<Update label="v2.4.1 (Current)">
<Update label="v2.4.2 (Current)">
### Fixed
- Langfuse tracing to have readable waterfalls
- Alembic Migrations to match models.py
- message_in_seq correctly included in webhook payload
### Changed
- Alembic to always use a session pooler
- Statement timeout during alembic operations to 5 min
</Update>
<Update label="v2.4.1">
### Added
- Alembic migration validation test suite

117
docs/docs.json
View File

@ -3,27 +3,20 @@
"theme": "mint",
"name": "Honcho",
"colors": {
"primary": "#86BCF2",
"primary": "#66AAFF",
"dark": "#151E27",
"light": "#B5D9FD"
"light": "#86BCF2"
},
"favicon": "/favicon.svg",
"contextual": {
"options": [
"copy",
"view",
"chatgpt",
"claude"
]
"options": ["copy", "view", "chatgpt", "claude"]
},
"navigation": {
"versions": [
{
"version": "v2.4.1",
"version": "v2.4.2",
"api": {
"openapi": [
"openapi.documented.yml"
]
"openapi": ["openapi.documented.yml"]
},
"tabs": [
{
@ -41,10 +34,19 @@
"group": "Core Concepts",
"pages": [
"v2/documentation/core-concepts/architecture",
"v2/documentation/core-concepts/glossary",
"v2/documentation/core-concepts/features",
"v2/documentation/core-concepts/features/storing-data",
"v2/documentation/core-concepts/features/dialectic-endpoint",
"v2/documentation/core-concepts/features/get-context",
"v2/documentation/core-concepts/features/search",
"v2/documentation/core-concepts/features/working-rep",
"v2/documentation/core-concepts/features/streaming-response",
"v2/documentation/core-concepts/features/using-filters",
"v2/documentation/core-concepts/features/file-uploads",
"v2/documentation/core-concepts/features/queue-status",
"v2/documentation/core-concepts/features/local-vs-global",
"v2/documentation/core-concepts/configuration",
"v2/documentation/core-concepts/summarizer"
"v2/documentation/core-concepts/summarizer",
"v2/documentation/core-concepts/glossary"
]
},
{
@ -61,54 +63,21 @@
"groups": [
{
"group": "Getting Started",
"pages": [
"v2/guides/overview",
"v2/guides/mcp"
]
"pages": ["v2/guides/overview", "v2/guides/mcp"]
},
{
"group": "Application Interfaces",
"pages": [
"v2/guides/discord",
"v2/guides/telegram"
]
},
{
"group": "Design Patterns",
"pages": [
"v2/guides/dialectic-endpoint",
"v2/guides/get-context",
"v2/guides/search",
"v2/guides/working-rep",
"v2/guides/streaming-response",
"v2/guides/using-filters",
"v2/guides/file-uploads"
]
}
]
},
{
"tab": "Contributing",
"groups": [
{
"group": "Contributing",
"pages": [
"v2/contributing/guidelines",
"v2/contributing/self-hosting",
"v2/contributing/configuration",
"v2/contributing/license"
]
"pages": ["v2/guides/discord", "v2/guides/telegram"]
}
]
},
{
"tab": "API Reference",
"groups": [
{
"group": "API Documentation",
"pages": [
"v2/api-reference/introduction"
]
"pages": ["v2/api-reference/introduction"]
},
{
"group": "workspaces",
@ -192,15 +161,27 @@
]
}
]
},
{
"tab": "Contributing",
"groups": [
{
"group": "Contributing",
"pages": [
"v2/contributing/guidelines",
"v2/contributing/self-hosting",
"v2/contributing/configuration",
"v2/contributing/license"
]
}
]
}
]
},
{
"version": "v1.1.0",
"api": {
"openapi": [
"openapi.json"
]
"openapi": ["openapi.json"]
},
"tabs": [
{
@ -230,23 +211,15 @@
"groups": [
{
"group": "Getting Started",
"pages": [
"v1/guides/overview",
"v1/guides/streaming-response"
]
"pages": ["v1/guides/overview", "v1/guides/streaming-response"]
},
{
"group": "Application Interfaces",
"pages": [
"v1/guides/discord",
"v1/guides/honcho-mcp"
]
"pages": ["v1/guides/discord", "v1/guides/honcho-mcp"]
},
{
"group": "Personal Memory",
"pages": [
"v1/guides/dialectic-endpoint"
]
"pages": ["v1/guides/dialectic-endpoint"]
}
]
},
@ -255,9 +228,7 @@
"groups": [
{
"group": "API Documentation",
"pages": [
"v1/api-reference/introduction"
]
"pages": ["v1/api-reference/introduction"]
},
{
"group": "apps",
@ -305,9 +276,7 @@
},
{
"group": "keys",
"pages": [
"v1/api-reference/endpoint/keys/create-key"
]
"pages": ["v1/api-reference/endpoint/keys/create-key"]
},
{
"group": "metamessages",
@ -360,9 +329,9 @@
"global": {
"anchors": [
{
"anchor": "Managed Platform",
"anchor": "Dashboard",
"href": "https://app.honcho.dev",
"icon": "book-open-cover"
"icon": "table-columns"
},
{
"anchor": "Community",

4
docs/images/overview/honcho-overview-dark.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 38 KiB

BIN
docs/images/overview/honcho-overview-light.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 414 KiB

4
docs/images/overview/honcho-overview-light.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 38 KiB

BIN
docs/images/overview/honcho-overview-no-background.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 418 KiB

4
docs/images/overview/honcho-overview.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 37 KiB

0
docs/_snippets/overview-shields.mdx → docs/snippets/overview-shields.mdx
View File

346
docs/v2/contributing/configuration.mdx
View File

@ -46,14 +46,19 @@ cp config.toml.example config.toml
Then modify the values as needed. The TOML file is organized into sections:
- `[app]` - Application-level settings (log level, host, port, embedding settings)
- `[db]` - Database connection and pool settings
- `[auth]` - Authentication configuration
- `[llm]` - LLM provider API keys and general settings
- `[dialectic]` - Dialectic API configuration (provider, model, search settings)
- `[deriver]` - Background worker settings and theory of mind configuration
- `[summary]` - Session summarization settings
- `[sentry]` - Error tracking and monitoring settings
- `[app]` - Application-level settings (log level, session limits, embedding settings, Langfuse integration, local metrics collection)
- `[db]` - Database connection and pool settings (connection URI, pool size, timeouts, connection recycling)
- `[auth]` - Authentication configuration (enable/disable auth, JWT secret)
- `[cache]` - Redis cache configuration (enable/disable caching, Redis URL, TTL settings, lock configuration for cache stampede prevention)
- `[llm]` - LLM provider API keys (Anthropic, OpenAI, Gemini, Groq, OpenAI-compatible endpoints) and general LLM settings
- `[dialectic]` - Dialectic API configuration (provider, model, query generation settings, semantic search parameters, context window size)
- `[deriver]` - Background worker settings (worker count, polling intervals, queue management) and theory of mind configuration (model, tokens, observation limits)
- `[peer_card]` - Peer card generation settings (provider, model, token limits)
- `[summary]` - Session summarization settings (frequency thresholds, provider, model, token limits for short and long summaries)
- `[dream]` - Dream processing configuration (enable/disable, thresholds, idle timeouts, dream types, LLM settings)
- `[webhook]` - Webhook configuration (webhook secret, workspace limits)
- `[metrics]` - Metrics collection settings (enable/disable metrics, namespace)
- `[sentry]` - Error tracking and monitoring settings (enable/disable, DSN, environment, sample rates)
### Using Environment Variables
@ -107,28 +112,33 @@ The application will use the production connection URI while keeping the pool si
### Application Settings
Application-level settings control core behavior of the Honcho server including logging, session limits, message handling, and optional integrations.
**Basic Application Configuration:**
```bash
# Logging and server settings
LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR
SESSION_PEERS_LIMIT=10
GET_CONTEXT_MAX_TOKENS=100000
LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR, CRITICAL
# Embedding settings (optional)
EMBED_MESSAGES=false
MAX_EMBEDDING_TOKENS=8192
MAX_EMBEDDING_TOKENS_PER_REQUEST=300000
# Session and context limits
SESSION_OBSERVERS_LIMIT=10 # Maximum number of observers per session
GET_CONTEXT_MAX_TOKENS=100000 # Maximum tokens for context retrieval
MAX_MESSAGE_SIZE=25000 # Maximum message size in characters
# Embedding settings
EMBED_MESSAGES=true # Enable vector embeddings for messages
MAX_EMBEDDING_TOKENS=8192 # Maximum tokens per embedding
MAX_EMBEDDING_TOKENS_PER_REQUEST=300000 # Batch embedding limit
```
**Environment-specific settings:**
**Optional Integrations:**
```bash
# Development
LOG_LEVEL=DEBUG
FASTAPI_HOST=127.0.0.1
# Langfuse integration for LLM observability
LANGFUSE_HOST=https://cloud.langfuse.com
LANGFUSE_PUBLIC_KEY=your-langfuse-public-key
# Production
LOG_LEVEL=WARNING
FASTAPI_HOST=0.0.0.0
# Local metrics collection
COLLECT_METRICS_LOCAL=false
LOCAL_METRICS_FILE=metrics.jsonl
```
### Database Configuration
@ -196,6 +206,36 @@ AUTH_JWT_SECRET=your-super-secret-jwt-key
python scripts/generate_jwt_secret.py
```
### Cache Configuration
Honcho supports Redis caching to improve performance by caching frequently accessed data like peers, sessions, and working representations. Caching also includes lock mechanisms to prevent cache stampede scenarios.
**Redis Cache Settings:**
```bash
# Enable/disable Redis caching
CACHE_ENABLED=false # Set to true to enable caching
# Redis connection
CACHE_URL=redis://localhost:6379/0?suppress=false
# Cache namespace and TTL
CACHE_NAMESPACE=honcho # Prefix for all cache keys
CACHE_DEFAULT_TTL_SECONDS=300 # How long items stay in cache (5 minutes)
# Lock settings for preventing cache stampede
CACHE_DEFAULT_LOCK_TTL_SECONDS=5 # Lock duration when fetching from DB on cache miss
```
**When to Enable Caching:**
- High-traffic production environments
- Applications with many repeated reads of the same data
- When you need to reduce database load
**Note:** Caching requires a Redis instance. You can run Redis locally with Docker:
```bash
docker run -d -p 6379:6379 redis:latest
```
## LLM Provider Configuration
Honcho supports multiple LLM providers for different tasks. API keys are configured in the `[llm]` section, while specific features use their own configuration sections.
@ -221,6 +261,9 @@ LLM_OPENAI_COMPATIBLE_BASE_URL=https://your-openai-compatible-endpoint.com
```bash
# Default settings for all LLM calls
LLM_DEFAULT_MAX_TOKENS=2500
# Embedding provider (used when EMBED_MESSAGES=true)
LLM_EMBEDDING_PROVIDER=openai # Options: openai, gemini
```
### Feature-Specific Model Configuration
@ -228,67 +271,147 @@ LLM_DEFAULT_MAX_TOKENS=2500
Different features can use different providers and models:
**Dialectic API:**
The Dialectic API provides theory-of-mind informed responses by integrating long-term facts with current context.
```bash
# Main dialectic model (default: Anthropic)
DIALECTIC_PROVIDER=anthropic
DIALECTIC_MODEL=claude-sonnet-4-20250514
DIALECTIC_MAX_OUTPUT_TOKENS=2500
DIALECTIC_THINKING_BUDGET_TOKENS=1024
DIALECTIC_THINKING_BUDGET_TOKENS=1024 # Only used with Anthropic provider
DIALECTIC_CONTEXT_WINDOW_SIZE=100000 # Maximum context window tokens
# Query generation for dialectic (default: Groq)
# Query generation for dialectic searches
DIALECTIC_PERFORM_QUERY_GENERATION=false # Enable query generation for semantic search
DIALECTIC_QUERY_GENERATION_PROVIDER=groq
DIALECTIC_QUERY_GENERATION_MODEL=llama-3.1-8b-instant
# Semantic search settings
DIALECTIC_SEMANTIC_SEARCH_TOP_K=10
DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE=0.85
DIALECTIC_SEMANTIC_SEARCH_TOP_K=10 # Number of results to retrieve
DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE=0.85 # Maximum distance for relevance
```
**Deriver:**
**Deriver (Theory of Mind):**
The Deriver is a background processing system that extracts facts from messages and builds theory-of-mind representations of peers.
```bash
# Deriver model (default: Google)
# LLM settings for deriver
DERIVER_PROVIDER=google
DERIVER_MODEL=gemini-2.0-flash-lite
DERIVER_MODEL=gemini-2.5-flash-lite
DERIVER_MAX_OUTPUT_TOKENS=10000
DERIVER_THINKING_BUDGET_TOKENS=1024 # Only used with Anthropic provider
DERIVER_MAX_INPUT_TOKENS=23000 # Maximum input tokens for deriver
# Worker settings
DERIVER_WORKERS=1
DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5
DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0
DERIVER_WORKERS=1 # Number of background worker processes
DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0 # Time between queue checks
DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 # Timeout for stale sessions
# Peer card settings
DERIVER_PEER_CARD_PROVIDER=openai
DERIVER_PEER_CARD_MODEL=gpt-5-nano-2025-08-07
DERIVER_PEER_CARD_MAX_OUTPUT_TOKENS=2000
# Queue management
DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000 # Keep errored items for 30 days
# Maximum number of observations to store in working representation
# This is applied to both explicit and deductive observations
DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100
# Working representation settings
DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=50 # Max observations stored
DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=4096 # Max tokens per batch
```
**Peer Card:**
Peer cards are short, structured summaries of peer identity and characteristics.
```bash
# Enable/disable peer card generation
PEER_CARD_ENABLED=true
# LLM settings for peer card generation
PEER_CARD_PROVIDER=openai
PEER_CARD_MODEL=gpt-5-nano-2025-08-07
PEER_CARD_MAX_OUTPUT_TOKENS=4000 # Includes thinking tokens for GPT-5 models
```
**Summary Generation:**
```bash
# Summary model (default: Google)
SUMMARY_PROVIDER=google
SUMMARY_MODEL=gemini-1.5-flash-latest
SUMMARY_MAX_TOKENS_SHORT=1000
SUMMARY_MAX_TOKENS_LONG=2000
SUMMARY_THINKING_BUDGET_TOKENS=512
# Summary frequency
SUMMARY_MESSAGES_PER_SHORT_SUMMARY=20
SUMMARY_MESSAGES_PER_LONG_SUMMARY=60
Session summaries provide compressed context for long conversations. Honcho creates two types: short summaries (frequent) and long summaries (comprehensive).
```bash
# Enable/disable summarization
SUMMARY_ENABLED=true
# LLM settings for summary generation
SUMMARY_PROVIDER=openai
SUMMARY_MODEL=gpt-4o-mini-2024-07-18
SUMMARY_MAX_TOKENS_SHORT=1000 # Max tokens for short summaries
SUMMARY_MAX_TOKENS_LONG=4000 # Max tokens for long summaries
SUMMARY_THINKING_BUDGET_TOKENS=512 # Only used with Anthropic provider
# Summary frequency thresholds
SUMMARY_MESSAGES_PER_SHORT_SUMMARY=20 # Create short summary every N messages
SUMMARY_MESSAGES_PER_LONG_SUMMARY=60 # Create long summary every N messages
```
### Default Provider Usage
By default, Honcho uses:
- **Anthropic** for dialectic API responses
- **Groq** for query generation
- **Google** for deriving theory of mind and summarization
- **Anthropic** (Claude) for dialectic API responses
- **Groq** for query generation (fast, cost-effective)
- **Google** (Gemini) for theory of mind derivation
- **OpenAI** (GPT) for peer cards and summarization
- **OpenAI** for embeddings (if `EMBED_MESSAGES=true`)
You only need to set the API keys for the providers you plan to use.
You only need to set the API keys for the providers you plan to use. All providers are configurable per feature.
## Additional Features Configuration
### Dream Processing
Dream processing consolidates and refines peer representations during idle periods, similar to how human memory consolidation works during sleep.
**Dream Settings:**
```bash
# Enable/disable dream processing
DREAM_ENABLED=true
# Trigger thresholds
DREAM_DOCUMENT_THRESHOLD=50 # Minimum documents to trigger a dream
DREAM_IDLE_TIMEOUT_MINUTES=60 # Minutes of inactivity before dream can start
DREAM_MIN_HOURS_BETWEEN_DREAMS=8 # Minimum hours between dreams for a peer
# Dream types to enable
DREAM_ENABLED_TYPES=["consolidate"] # Currently supported: consolidate
# LLM settings for dream processing
DREAM_PROVIDER=openai
DREAM_MODEL=gpt-4o-mini-2024-07-18
DREAM_MAX_OUTPUT_TOKENS=2000
```
### Webhook Configuration
Webhooks allow you to receive real-time notifications when events occur in Honcho (e.g., new messages, session updates).
**Webhook Settings:**
```bash
# Webhook secret for signing payloads (optional but recommended)
WEBHOOK_SECRET=your-webhook-signing-secret
# Limit on webhooks per workspace
WEBHOOK_MAX_WORKSPACE_LIMIT=10
```
### Metrics Collection
Enable metrics collection for monitoring Honcho performance and usage.
**Metrics Settings:**
```bash
# Enable/disable metrics collection
METRICS_ENABLED=false
# Namespace for metrics (used in metric names)
METRICS_NAMESPACE=honcho
```
## Monitoring Configuration
@ -296,13 +419,17 @@ You only need to set the API keys for the providers you plan to use.
**Sentry Settings:**
```bash
# Enable/disable Sentry
# Enable/disable Sentry error tracking
SENTRY_ENABLED=false
# Sentry configuration
SENTRY_DSN=https://your-sentry-dsn@sentry.io/project-id
SENTRY_TRACES_SAMPLE_RATE=0.1
SENTRY_PROFILES_SAMPLE_RATE=0.1
SENTRY_RELEASE=2.4.0 # Optional: track which version errors come from
SENTRY_ENVIRONMENT=production # Environment name (development, staging, production)
# Sampling rates (0.0 to 1.0)
SENTRY_TRACES_SAMPLE_RATE=0.1 # 10% of transactions tracked
SENTRY_PROFILES_SAMPLE_RATE=0.1 # 10% of transactions profiled
```
## Environment-Specific Examples
@ -313,7 +440,7 @@ SENTRY_PROFILES_SAMPLE_RATE=0.1
```toml
[app]
LOG_LEVEL = "DEBUG"
SESSION_PEERS_LIMIT = 10
SESSION_OBSERVERS_LIMIT = 10
EMBED_MESSAGES = false
[db]
@ -323,21 +450,40 @@ POOL_SIZE = 5
[auth]
USE_AUTH = false
[cache]
ENABLED = false
[dialectic]
PROVIDER = "anthropic"
MODEL = "claude-sonnet-4-20250514"
QUERY_GENERATION_PROVIDER = "groq"
QUERY_GENERATION_MODEL = "llama-3.1-8b-instant"
PERFORM_QUERY_GENERATION = false
MAX_OUTPUT_TOKENS = 2500
[summary]
PROVIDER = "google"
MODEL = "gemini-1.5-flash-latest"
MAX_TOKENS_SHORT = 1000
MAX_TOKENS_LONG = 2000
[deriver]
WORKERS = 1
PROVIDER = "google"
MODEL = "gemini-2.5-flash-lite"
[peer_card]
ENABLED = true
PROVIDER = "openai"
MODEL = "gpt-5-nano-2025-08-07"
[summary]
ENABLED = true
PROVIDER = "openai"
MODEL = "gpt-4o-mini-2024-07-18"
MAX_TOKENS_SHORT = 1000
MAX_TOKENS_LONG = 4000
[dream]
ENABLED = true
[webhook]
MAX_WORKSPACE_LIMIT = 10
[metrics]
ENABLED = false
[sentry]
ENABLED = false
@ -349,7 +495,12 @@ ENABLED = false
LOG_LEVEL=DEBUG
DB_CONNECTION_URI=postgresql+psycopg://postgres:postgres@localhost:5432/honcho_dev
AUTH_USE_AUTH=false
ANTHROPIC_API_KEY=your-dev-anthropic-key
CACHE_ENABLED=false
# LLM Provider API Keys
LLM_ANTHROPIC_API_KEY=your-dev-anthropic-key
LLM_OPENAI_API_KEY=your-dev-openai-key
LLM_GEMINI_API_KEY=your-dev-gemini-key
```
### Production Configuration
@ -358,7 +509,7 @@ ANTHROPIC_API_KEY=your-dev-anthropic-key
```toml
[app]
LOG_LEVEL = "WARNING"
SESSION_PEERS_LIMIT = 10
SESSION_OBSERVERS_LIMIT = 10
EMBED_MESSAGES = true
[db]
@ -369,27 +520,50 @@ MAX_OVERFLOW = 40
[auth]
USE_AUTH = true
[cache]
ENABLED = true
URL = "redis://redis:6379/0"
DEFAULT_TTL_SECONDS = 300
[dialectic]
PROVIDER = "anthropic"
MODEL = "claude-sonnet-4-20250514"
QUERY_GENERATION_PROVIDER = "groq"
QUERY_GENERATION_MODEL = "llama-3.1-8b-instant"
PERFORM_QUERY_GENERATION = false
MAX_OUTPUT_TOKENS = 2500
[summary]
PROVIDER = "google"
MODEL = "gemini-1.5-flash-latest"
MAX_TOKENS_SHORT = 1000
MAX_TOKENS_LONG = 2000
[deriver]
WORKERS = 4
PROVIDER = "google"
MODEL = "gemini-2.0-flash-lite"
MODEL = "gemini-2.5-flash-lite"
[peer_card]
ENABLED = true
PROVIDER = "openai"
MODEL = "gpt-5-nano-2025-08-07"
[summary]
ENABLED = true
PROVIDER = "openai"
MODEL = "gpt-4o-mini-2024-07-18"
MAX_TOKENS_SHORT = 1000
MAX_TOKENS_LONG = 4000
[dream]
ENABLED = true
PROVIDER = "openai"
MODEL = "gpt-4o-mini-2024-07-18"
[webhook]
MAX_WORKSPACE_LIMIT = 10
[metrics]
ENABLED = true
[sentry]
ENABLED = true
ENVIRONMENT = "production"
TRACES_SAMPLE_RATE = 0.1
PROFILES_SAMPLE_RATE = 0.1
```
**Environment variables for production:**
@ -397,11 +571,27 @@ TRACES_SAMPLE_RATE = 0.1
# .env.production
LOG_LEVEL=WARNING
DB_CONNECTION_URI=postgresql+psycopg://honcho_user:secure_password@prod-db:5432/honcho_prod
# Authentication
AUTH_USE_AUTH=true
AUTH_JWT_SECRET=your-super-secret-jwt-key
ANTHROPIC_API_KEY=your-prod-anthropic-key
GEMINI_API_KEY=your-prod-gemini-key
# Cache
CACHE_ENABLED=true
CACHE_URL=redis://redis:6379/0
# LLM Provider API Keys
LLM_ANTHROPIC_API_KEY=your-prod-anthropic-key
LLM_OPENAI_API_KEY=your-prod-openai-key
LLM_GEMINI_API_KEY=your-prod-gemini-key
LLM_GROQ_API_KEY=your-prod-groq-key
# Webhooks
WEBHOOK_SECRET=your-webhook-signing-secret
# Monitoring
SENTRY_DSN=https://your-sentry-dsn@sentry.io/project-id
SENTRY_ENVIRONMENT=production
```
## Migration Management

51
docs/v2/documentation/core-concepts/architecture.mdx
View File

@ -7,17 +7,15 @@ sidebarTitle: "Architecture"
<Note> The goal of this page is to build an intuition for the primitives in Honcho and how they fit together </Note>
Honcho has 3 main components that work together to manage agent identity and context.
Honcho has 2 main components that work together to manage agent identity and context.
- **The Storage API**: The Memory layer for storing interaction history for your agents
- **The Deriver**: The background processing layer that builds representations of users and agents
- **The Dialectic API**: The natural language API for chatting with representations
- **The Memory Layer**: The Memory layer for storing interaction history for your agents
- **The Reasoning Layer**: The background processing layer that builds representations of users and agents
Below we'll deep dive into these different areas, discussing the data
primitives, the flow of data through the system, artifacts Honcho produces, and
how to use them.
## Data Model
Honcho has a hierarchical data model centered around the entities below.
@ -37,9 +35,9 @@ Honcho has a hierarchical data model centered around the entities below.
style SM fill:#e8f5e9,stroke:#2e7d32,color:#000
```
There are `Workspaces` at the top that contain `Peers` and `Sessions`. A `Peer`
can be part of many `Sessions` and a `Session` can have many `Peers`. `Sessions`
hold messages that are sent by `Peers`.
- A `Workspaces` has `Peers` & `Sessions`
- A `Peer` can be in multiple `Sessions` and can send `Messages` in a `Session`.
- A `Session` can have many `Peers` and stores `Messages` sent by its `Peers`.
### <Icon icon="building" /> Workspaces
@ -126,19 +124,38 @@ with a single peer and structure the data as messages.
- File uploads (PDFs, text files, JSON documents)
## Deriver
## Reasoning Layer
At the core of developing representations of Peers, we have the Deriver. The
Deriver refers to a set of processes in Honcho that enqueue new messages sent
by peers and reasons over them to extract facts, insights, and context.
The raw data you store in Honcho is useful, but it's not in a format that's most
useful for an LLM to consume. There may be too many tokens that need to be
compacted, key facts about what happened may be hard to piece together because
they involve messages from across different sessions, etc.
To solve this problem, Honcho has a reasoning layer that continually processes
incoming data to form the most informationally dense and useful representations of `Peers`
that we can then expose to agents. Honcho does the following tasks in
the reasoning engine.
- **Fact Derivation**
- **Generate Summaries**
- **Generate Peer Cards**
- **Dreaming**
Honcho will reason about each `Message` it
ingests to generate new facts and insights that are spelled out and easy to
consume in an LLM prompt.
We refer to this module of Honcho as the `Deriver`, because it's constantly
deriving new insights from messages. The sum total of all these generated
insights are what we refer to as a `Representation`, all the data related to who
and what a `Peer` is.
Depending on the configuration of a `Peer` or `Session`, the deriver will behave
differently and update different representations.
Facts derived here are used in the Dialectic chat endpoint to generate
context-aware responses that can correctly reference both concrete facts
extracted from messages and social insights deduced from facts, tone, and
opinion.
Facts derived here are used in the Dialectic chat endpoint, get_context
endpoint,
<Info>
Deriver tasks are processed in parallel, but tasks affecting the same peer representation will always be processed serially in order of message creation, so as to properly understand their cumulative effect.
@ -149,7 +166,7 @@ There are two types of tasks that the deriver currently does:
- **Representation Tasks**: Generate/update peer representations
- **Summary Tasks**: Generate conversation summaries
### Peer Representations
### Local & Global Representations
Peer representations are more of an abstract concept, as they are made up of
various pieces of data stored throughout Honcho. There are however

2
docs/v2/documentation/core-concepts/configuration.mdx
View File

@ -1,5 +1,5 @@
---
title: 'Configuration'
title: 'Configure Reasoning'
description: 'Customizing how Honcho handles peers and sessions'
icon: 'wrench'
---

44
docs/v2/documentation/core-concepts/features.mdx
View File

@ -1,44 +0,0 @@
---
title: 'Features'
description: 'Key features and capabilities of Honcho'
icon: 'star'
---
This page is a quick overview of the features within Honcho. In-depth
guides are available for each feature in the [Spellbooks - Design Patterns](../../guides/overview#design-patterns) section.
### Local vs Global Representation
Peers in Honcho are abstract entities that can represent humans, agents, or NPCs. Honcho has a two-layer approach to forming representations of Peers.
- **Global Representation**: Representation owned by a Peer that is constructed from everything the Peer has sent within Honcho.
- **Local Representation**: The representation that a Peer forms of other Peers, based on the messages those other Peers have sent (as observed by the Peer forming the representation).
- At the Session level, you can configure which Peers are able to observe messages from other Peers in that Session. This determines which Peers form representations of others within the Session.
### Queue Status
To help developers understand when a Peer's representation is fully up to date, Honcho exposes the ability to poll the status of Peer-centric queues that construct representations.
- If no Session is specified, the queue status reflects pending work for the Peer's global representation.
- If a Session is specified, the queue status reflects pending work for the Peer's working representation in that Session.
### Search
Honcho implements a powerful search endpoint that allows you to search for messages across a workspace, session, or peer with complex [filters](/v2/guides/using-filters).
The search process combines full-text and semantic search using reciprocal rank fusion. By default, all messages ingested into Honcho have embeddings generated and stored in the database, enabling semantic search -- if this feature is disabled, the search process will only use full-text search.
Results are returned in the form of a list of Message objects, and you may choose how many results to return. The default is 10 results, with a maximum of 100.
In the SDK, search is available on `Workspace`, `Session`, and `Peer` objects, and an optional `filters` parameter may be used to apply a narrower search scope such as a time range or developer-defined metadata attached to messages.
Note that results are not ordered by recency, only relevance. Results can be sorted by timestamp or a filter on the `created_at` field can limit results to recent messages.
[Look here for examples of how to use search in the SDK](/v2/guides/search).
### Scoped API Keys
Builders can create scoped API keys to control access to different resources within Honcho.
- **Workspace-Level Keys**: Access to everything scoped to a Workspace.
- **Peer-Level Keys**: Access to everything scoped to a Peer.
- **Session-Level Keys**: Access to everything scoped to a Session.
### Get Context
Honcho provides a powerful context retrieval feature that delivers formatted conversation context from sessions, making it easy to integrate with LLMs like OpenAI, Anthropic, and others.
- By default, the context includes a blend of summary and messages which covers the entire history of the session.
- Summaries are generated automatically at intervals, and recent messages are included based on your specified token budget for the context.
- You can set any token limit, and if you prefer, you can disable summaries so that the context consists entirely of the most recent messages up to your chosen limit.

0
docs/v2/guides/dialectic-endpoint.mdx → docs/v2/documentation/core-concepts/features/dialectic-endpoint.mdx
View File

0
docs/v2/guides/file-uploads.mdx → docs/v2/documentation/core-concepts/features/file-uploads.mdx
View File

2
docs/v2/guides/get-context.mdx → docs/v2/documentation/core-concepts/features/get-context.mdx
View File

@ -1,5 +1,5 @@
---
title: 'Working with Session Context'
title: 'Get Context'
description: 'Learn how to use get_context() to retrieve and format conversation context for LLM integration'
icon: 'messages'
---

68
docs/v2/documentation/core-concepts/features/local-vs-global.mdx Normal file
View File

@ -0,0 +1,68 @@
---
title: Local vs Global Representations
description: Model directional relationships between Peers in Honcho
icon: location-pin
---
One of the unique affordances of Honcho is that it allows developers to model
directional relationships between Peers. What I mean by this is you can model
how one `Peer` thinks about another `Peer`.
There are many use cases where you don't want every agent or human to know
everything about another user such as games or multi-agent workflows. To
illustrate this, the following examples shows 2 conversations.
Conversation #1 (With Bob and Alice)
```
Alice: I had a great breakfast today.
Bob: What did you eat?
Alice: I had pancakes and eggs and bacon
```
Conversation #2 (With Alice and Charlie)
```
Alice: I actually didn't eat any breakfast today.
Charlie: Oh that's too bad.
Alice: But I lied to Bob and told him I did, so back me up if you see them.
```
Alice told Bob a lie in this conversation. If we stored both of these
conversations in Honcho with Alice, Bob, and Charlie as `Peers` and let them
use Honcho to get insights on each other then Bob would immediately know this
deception. For example:
<CodeGroup>
```python Python
# Bob could run
alice.chat("What did Alice eat today?")
# Response: Alice did not eat anything today
```
</CodeGroup>
This is a problem. Bob shouldn't be able to know everything about Alice in this
situation. So to support these situations we support what we call **Local
Representations**.
By default insights generated for a `Peer` are scoped globally. This means every
message sent by that `Peer` in any conversation updates the same representation
of that `Peer`. However, we can enable **Local Representations** so Bob can
form a representation Alice based only on what they observe Alice do.
This feature is illustrated in the graphic below:
<img src="/images/local-vs-global-reps.png" alt="Peer Representations" />
We can enable local representation for a `Peer` by setting `observe_others=True`.
This is shown in the [Configure
Reasoning](/v2/documentation/core-concepts/configuration) page.
Now if we used Bob's local representation of Alice then Bob would only get
insights on what they've seen Alice say to them.
```python
bob.chat(target="alice", query="What did Alice eat today?")
# Response: Alice ate pancakes, eggs, and bacon
```
<Note>
Local Representations are turned off by default
</Note>

132
docs/v2/documentation/core-concepts/features/queue-status.mdx Normal file
View File

@ -0,0 +1,132 @@
---
title: Queue Status
description: Learn how to check the status of the Deriver
icon: lines-leaning
---
Whenever `Messages` are stored in Honcho, a background process called the
[Deriver](/docs/v2/documentation/core-concepts/architecture#reasoning-layer) is
triggered to reason about the conversation and generate insights.
The Deriver is an asynchronous process and, depending on load may not immediately
generated insights for the latest message you've sent. To help with this, Honcho
provides several utilities to check the status of the Deriver.
<CodeGroup>
```python Python
from honcho import Honcho
honcho = Honcho()
status = honcho.get_deriver_status()
honcho.poll_deriver_status()
```
```typescript typescript
import { Honcho } from '@honcho-ai/sdk';
const honcho = new Honcho({});
const status = await honcho.getDeriverStatus();
await honcho.pollDeriverStatus();
```
</CodeGroup>
Output types
<CodeGroup>
```python Python
class DeriverStatus(BaseModel):
completed_work_units: int
"""Completed work units"""
in_progress_work_units: int
"""Work units currently being processed"""
pending_work_units: int
"""Work units waiting to be processed"""
total_work_units: int
"""Total work units"""
sessions: Optional[Dict[str, Sessions]] = None
"""Per-session status when not filtered by session"""
```
```typescript TypeScript
Promise<{
totalWorkUnits: number
completedWorkUnits: number
inProgressWorkUnits: number
pendingWorkUnits: number
sessions?: Record<string, DeriverStatus.Sessions>
}>
```
</CodeGroup>
Whenever a `Message` is sent it will generate several tasks. These could
be tasks such as generating insights, cleaning up a representation, summarizing
a conversation etc. These tasks are defined based on who is sending the
message, what `Session` the message is in, and potentially who is observing the
message. We call the combination of these parameters a `work_unit`
This has a few different implications.
- tasks within the same work_unit are processed sequentially, but multiple
work_units will be processed in parallel
- If local representations are turned in a Session then a `Message` will
generate an additional work unit for every `Peer` that has `observe_others=True`
The `get_deriver_status` and `poll_deriver_status` methods can take additional
parameters to scope the status to a specific work unit
<CodeGroup>
```python Python
def get_deriver_status(
self,
observer_id: str | None = None,
sender_id: str | None = None,
session_id: str | None = None,
) -> DeriverStatus:
```
```typescript TypeScript
export const DeriverStatusOptionsSchema = z.object({
observerId: z.string().optional(),
senderId: z.string().optional(),
sessionId: z.string().optional(),
timeoutMs: z
.number()
.positive('Timeout must be a positive number')
.optional(),
})
```
</CodeGroup>
Additionally, there are deriver status and polling deriver status methods
available on the `Session` objects in each of the SDKs.
Below are the function signatures for the session level deriver status method
<CodeGroup>
```python python
@validate_call
def get_deriver_status(
self,
observer_id: str | None = None,
sender_id: str | None = None,
) -> DeriverStatus:
```
```typescript TypeScript
async getDeriverStatus(
options?: Omit<DeriverStatusOptions, 'sessionId'>
): Promise<{
totalWorkUnits: number
completedWorkUnits: number
inProgressWorkUnits: number
pendingWorkUnits: number
sessions?: Record<string, DeriverStatus.Sessions>
}>
```
</CodeGroup>

0
docs/v2/guides/search.mdx → docs/v2/documentation/core-concepts/features/search.mdx
View File

61
docs/v2/documentation/core-concepts/features/storing-data.mdx Normal file
View File

@ -0,0 +1,61 @@
---
title: Storing Data
description: "Store Data in Honcho to Generate Memories and Insights"
icon: "memory"
---
The most basic building block of Honcho's data model is the `Message` object.
A `Message` is sent by a `Peer` and saved in a `Session`
<CodeGroup>
```python Python
from honcho import Honcho
honcho = Honcho()
peer = honcho.peer("sample-peer")
session = honcho.session("sample-session")
message = peer.message("Hello, world!", session_id=session.id)
session.add_messages([message])
```
```typescript TypeScript
import { Honcho } from '@honcho-ai/sdk';
const honcho = new Honcho({});
const peer = await honcho.peer('sample-peer');
const session = await honcho.session('sample-session');
const message = peer.message('Hello, world!');
await session.addMessages([message]);
```
</CodeGroup>
Once a `Message` is saved in Honcho, it will kick off a background task that
looks at the new data to generate insights about the `Peer` that sent the `Message`
This is the default behavior of Honcho and can be turned off by [configuring the
Peer or Session](/v2/documentation/core-concepts/configuration)
This pattern of having a Peer, Session, and Messages is highly flexible and
works for many different use cases and agent setups. Some use cases may only
need a single Peer, but many Sessions. Others will only use a single `Session`
for their entire app. These are flexible components that work in any situation.
## Chat Bots
A common use case for Honcho to is to build a chatbot like ChatGPT or Claude.
In this case you can simply
- Make a `Peer` for the User
- Make a `Peer` for the AI
Then you can make a `Session` for each thread of conversation and save
`Messages` from the user and assistant in each turn of conversation

0
docs/v2/guides/streaming-response.mdx → docs/v2/documentation/core-concepts/features/streaming-response.mdx
View File

0
docs/v2/guides/using-filters.mdx → docs/v2/documentation/core-concepts/features/using-filters.mdx
View File

0
docs/v2/guides/working-rep.mdx → docs/v2/documentation/core-concepts/features/working-rep.mdx
View File

134
docs/v2/documentation/introduction/overview.mdx
View File

@ -5,108 +5,102 @@ icon: "brain"
sidebarTitle: "Overview"
---
When building agents developers often run into the same walls:
Honcho is an AI-native memory library for building agents with
[state-of-the-art](https://blog.plasticlabs.ai/research/Introducing-Neuromancer-XR)
long-term memory.
> "My agent forgets everything between chats"
Agents using Honcho have perfect recall with a wide variety of tools to traverse
their history and get the exact context they need when they need it.
You need memory: session management, message storage, context handling. It's table stakes, but surprisingly complex to get right.
It then goes beyond basic memory by reasoning about the stored history
to expand the latent information available to your agent. Agents using Honcho
will understand who they are, who they are interacting with, what happened, and
when it happened — all without you having to think about it.
> "My agent treats everyone exactly the same"
Use it to build
You need personalization: user modeling, preference learning, behavioral adaptation. Now you're building a [social cognition](../core-concepts/glossary#social-cognition) engine.
- Highly personalized experiences
- Agents with social cognition
- Agents with rich identity that evolve over time
- Multi-agent systems with complex social dynamics
> "I'm writing infrastructure instead of features"
You need Honcho
<img src="/images/agent_hierarchy.png" alt="Honcho's Hiearchy of Agents" />
Honcho delivers production-ready memory infrastructure from day one. Store
conversations, manage sessions, get perfectly formatted context for any LLM.
But here's the magic: while your agents are chatting, Honcho is learning. It
builds Theory of Mind models automatically, transforming raw conversations into
rich psychological understanding.
```python
# Start simple - just add messages
# Start simple by just adding messages
session.add_messages([alice.message("I learn best with examples")])
# Get powerful - query user psychology
# Honcho will automatically reason about the message to generate insights about Alice
# Get insights by chatting with the agent
insight = peer.chat("How should I explain this concept?")
# > "This user learns best through concrete examples..."
```
Your agents evolve from goldfish to counselor, on the same infrastructure. That's Honcho.
Designed for developers and agents alike:
- **Natural Language Queries**: Chat with Honcho in natural language via the [Dialectic API](../core-concepts/architecture#dialectic-api) and let agents backchannel
- **Automatic Context Management**: Smart summarization that respects token limits
- **Native multi-agent support**: Break out of User/Assistant Paradigms and build complex multi-agent systems
- **Natural Language Queries**: Chat with Honcho in natural language via the [Dialectic API](../core-concepts/architecture#dialectic-api) to get insights about your users and agents
- **Automatic Context Management**: Smart conversation summaries to have infinite chats
- **Native multi-agent support**: Sessions can natively have as many participants as you need
- **Agent-first interfaces**: MCP connections and APIs designed for agents to consume and use as tools
- **Provider Agnostic**: Works with any LLM or Agent Framework
## How It Works
### Storage
<Accordion title="High Level Diagram" defaultOpen="true">
<Frame>
<img src="/images/overview/honcho-overview.svg" alt="High Level Honcho Diagram" />
</Frame>
</Accordion>
Developers use Honcho to store information about their users and application via
two integrated layers:
At a high level Honcho works very simply:
<img src="/images/basic_honcho_flowchart.png" alt="Basic Honcho Flowchart" />
1. Store messages sent by users and agents in Honcho
2. Honcho reasons about the messages to generate insights about each entity in
the system
3. At runtime your agents can leverage insights from Honcho to get the exact
context they need
**Memory Layer**: Captures all user interactions - messages, preferences, and
behavioral patterns - in a peer-centric data model that scales from individual
conversations to complex multi-agent scenarios. This also queues up messages for
the reasoning layer to process.
There are several API endpoints to leverage the memory & insights in Honcho.
**Reasoning Layer**: Continuously analyzes stored interactions to build
psychological profiles using [theory of mind](../core-concepts/glossary#theory-of-mind)
inference, extracting patterns about communication style, decision-making
preferences, and mental models.
### Get Context
### Retrieval
This is the easiest way to leverage Honcho. simply call get context and get the
most relevant information for your conversation. This endpoint is highly
customizable so you can specify parameters such as:
Once data is stored and generated within Honcho, the API exposes several
different ways to retrieve and use those insights.
- A number of tokens you want
- An option to include summaries of the conversation
- An option to get a profile of a specific user (Peer Card & Representation)
**[Dialectic API](/v2/guides/dialectic-endpoint)**: This is the
flagship endpoint that allows developers to send natural language queries to
Honcho to chat with the representation of each user in your system to get
dynamic, in-context actionable insights.
### Search
Example Queries
This endpoint lets you search across Honcho for relevant messages using a
hybrid search strategy that combines full-text and semantic search.
You can optionally scope the endpoint to a specific workspace, peer, or session.
### Working Representation
This endpoint gives you a snapshot of a user or what we call a
**Representation**. Essentially, a list of explicit and deductive facts about
the user that are relevant to the current conversation.
Plug this into your prompt to get a quick overview of the user.
### Dialectic API
This endpoint lets you chat with Honcho about any entity in your system. Honcho
will leverage what it has remembered and learned about the entity to provide in-context actionable insights.
This is especially helpful when you want your agent to back-channel with Honcho to
change its behavior at runtime.
Example Queries:
- "What's the best way to explain technical concepts to this user?"
- "Is this user more task-oriented or relationship-oriented?"
- "What time of day is this user most engaged?"
- "How does this user prefer to receive feedback?"
- "What are this user's core values based on our conversations?"
**[Get Context](/v2/guides/get-context)**: This endpoint abstracts context window
constraints and continuously retrieves the most relevant and recent data from a
conversation. Provide a token budget and Honcho will return a combination of
summaries and messages that provide session context. Use this for creating
long-running conversations. We crafted our summaries to provide the most
[coverage of a session possible](../core-concepts/summarizer).
**[Search](/v2/guides/search)**: This endpoint allows you to search across Honcho
for relevant messages either at the workspace, peer, or session level. This
endpoint uses a hybrid search strategy that combines text search and cosine
similarity.
**[Working Representations](/v2/guides/working-rep)**: Get a cached, snapshot
of a user in the context of a session. Instead of waiting for an LLM to
synthesize an in-context response via the Dialectic endpoint, use this to get
recent insights you can plug into your context window.
## Ideal For
**Personalized AI assistants** that need to understand individual psychology, not just remember conversations.
**Customer-facing agents** that must adapt their approach based on user communication preferences and emotional context.
**Multi-agent systems** where AI needs to understand human collaborators' working styles and decision-making patterns.
**NPCs** where you want autonomous agents with a rich and deep personality that isn't the average sycophantic llm
## Getting Started
@ -115,7 +109,7 @@ Ready to integrate Honcho into your application?
<CardGroup cols={2}> <Card title="Quickstart Guide" icon="rocket"
href="/v2/documentation/introduction/quickstart"> Get up and running with
Honcho in minutes </Card> <Card title="Core Concepts" icon="brain"
href="/v2/documentation/core-concepts/glossary"> Understand Honcho's
href="/v2/documentation/core-concepts/architecture"> Understand Honcho's
fundamental concepts </Card> </CardGroup>
## Community & Support

158
docs/v2/documentation/introduction/quickstart.mdx
View File

@ -62,7 +62,7 @@ The Honcho client is the main entry point for interacting with Honcho's API. By
from honcho import Honcho
# Initialize client (uses demo environment and default workspace)
client = Honcho()
honcho = Honcho()
```
@ -70,7 +70,7 @@ client = Honcho()
import { Honcho } from '@honcho-ai/sdk';
// Initialize client (uses demo environment and default workspace)
const client = new Honcho({});
const honcho = new Honcho({});
```
</CodeGroup>
@ -83,7 +83,7 @@ import os
from honcho import Honcho
# Production environment with API key
client = Honcho(
honcho = Honcho(
api_key=os.environ["HONCHO_API_KEY"],
environment="production",
# Create a workspace, otherwise set to "default"
@ -95,7 +95,7 @@ client = Honcho(
import { Honcho } from '@honcho-ai/sdk';
// Production environment with API key
const client = new Honcho({
const honcho = new Honcho({
apiKey: process.env.HONCHO_API_KEY!,
environment: "production",
// Create a workspace, otherwise set to "default"
@ -110,13 +110,13 @@ Peers represent individual users, AI agents, or any conversational entity in you
<CodeGroup>
```python Python
alice = client.peer("alice")
bob = client.peer("bob")
alice = honcho.peer("alice")
bob = honcho.peer("bob")
```
```typescript TypeScript
const alice = await client.peer("alice")
const bob = await client.peer("bob")
const alice = await honcho.peer("alice")
const bob = await honcho.peer("bob")
```
</CodeGroup>
@ -126,12 +126,12 @@ Sessions are independent conversations that can include multiple peers:
<CodeGroup>
```python Python
session = client.session("session_1")
session = honcho.session("session_1")
session.add_peers([alice, bob])
```
```typescript TypeScript
const session = await client.session("session_1")
const session = await honcho.session("session_1")
await session.addPeers([alice, bob])
```
</CodeGroup>
@ -171,7 +171,7 @@ Now ask Honcho what it's learned - this is where the magic happens:
<CodeGroup>
```python Python
# Ask what Bob is like
response = alice.chat("Tell me about Bob's interests and habits")
response = bob.chat("Tell me about Bob's interests and habits")
print(response)
# Returns rich context like:
@ -182,36 +182,128 @@ print(response)
```
```typescript TypeScript
(async () => {
// Ask what Bob is like
const response = await alice.chat("Tell me about Bob's interests and habits");
console.log(response);
bob.chat("Tell me about Bob's interests and habits").then((response) => {
console.log(response);
// Returns rich context like:
// "Bob is health-conscious and has been working on getting back in shape.
// He regularly goes to the gym, particularly in the evenings, and finds
// exercise helps him relax. He's encouraging about fitness and willing
// to share advice about workout routines."
})
```
</CodeGroup>
// Returns rich context like:
// "Bob is health-conscious and has been working on getting back in shape.
// He regularly goes to the gym, particularly in the evenings, and finds
// exercise helps him relax. He's encouraging about fitness and willing
// to share advice about workout routines."
})();
## 7. Putting it all together
<CodeGroup>
```python Python
import os
from honcho import Honcho
# Create your client
honcho = Honcho(
api_key=os.environ["HONCHO_API_KEY"],
environment="production",
# Create a workspace, otherwise set to "default"
# workspaceId="your-workspace-id"
)
# Get your Peers
alice = honcho.peer("alice")
bob = honcho.peer("bob")
# Make a Session and add your Peers
session = honcho.session("session_1")
session.add_peers([alice, bob])
# Add messages sent by your Peers
session.add_messages([
alice.message("Hi Bob, how are you?"),
bob.message("I'm good, thank you!"),
alice.message("What are you doing today after work?"),
bob.message("I'm going to the gym! I've been trying to get back in shape."),
alice.message("That's great! I should probably start exercising too."),
bob.message("You should! I find that evening workouts help me relax."),
])
# Get insights about your Peers
response = bob.chat("Tell me about Bob's interests and habits")
print(response)
# Returns rich context like:
# "Bob is health-conscious and has been working on getting back in shape.
# He regularly goes to the gym, particularly in the evenings, and finds
# exercise helps him relax. He's encouraging about fitness and willing
# to share advice about workout routines."
```
```typescript TypeScript
import { Honcho } from '@honcho-ai/sdk';
// Create your client
const honcho = new Honcho({
apiKey: process.env.HONCHO_API_KEY!,
environment: "production",
// Create a workspace, otherwise set to "default"
// workspace: "your-workspace-id"
});
// Get your Peers
const alice = await honcho.peer("alice")
const bob = await honcho.peer("bob")
// Make a Session and add your peers
const session = await honcho.session("session_1")
await session.addPeers([alice, bob])
// Add messages sent by your Peers
await session.addMessages([
alice.message("Hi Bob, how are you?"),
bob.message("I'm good, thank you!"),
alice.message("What are you doing today after work?"),
bob.message("I'm going to the gym! I've been trying to get back in shape."),
alice.message("That's great! I should probably start exercising too."),
bob.message("You should! I find that evening workouts help me relax."),
])
// Get insights about your peers
bob.chat("Tell me about Bob's interests and habits").then((response) => {
console.log(response);
// Returns rich context like:
// "Bob is health-conscious and has been working on getting back in shape.
// He regularly goes to the gym, particularly in the evenings, and finds
// exercise helps him relax. He's encouraging about fitness and willing
// to share advice about workout routines."
})
```
</CodeGroup>
## What Just Happened?
Honcho automatically built rich psychological profiles from just a few messages:
You just got through building a simple conversation between two people, Alice
and Bob. We:
- **Theory of Mind Processing**: Understanding personality, preferences, and patterns
- **Ambient Learning**: No surveys or explicit training - just natural conversation
- **Rich Context**: Far more detailed than simple conversation history
1. Set up our connection to Honcho.
2. Setup who the participants of our conversation are, these are called `Peers`.
3. Made a `Session` and added our `Peers` to it.
4. Sent messages from our `Peers`
5. Chat with Honcho to get insights about one of the `Peers` in the conversation
The response isn't just retrieving stored text - it's synthesizing insights about Bob's personality, habits, and communication style.
As soon as you save a message in Honcho, it will start to reason about it to
pull out insights and develop a profile of the user. This is the default
behavior and can be toggled off via [the configuration](/v2/documentation/core-concepts/configuration).
## Next Steps
This covers the core concepts: **peers**, **sessions**, **messages**, and **dialectic queries**.
- For production use, [sign up for the managed platform](https://app.honcho.dev) or get an [overview here](../reference/platform).
- For detailed API reference, check out our [SDK documentation](../reference/sdk).
- For more examples, explore our [guides](../guides/overview).
---
<CardGroup cols={3}>
<Card title="Architecture" icon="rocket"
href="/v2/documentation/core-concepts/architecture">
Learn about the data primitives in Honcho and how they work together
</Card>
<Card title="Start Building" icon="brain" href="https://app.honcho.dev">
Sign up for Managed Honcho and get started building agents now.
</Card>
<Card title="Guides" icon="book" href="/v2/guides/overview">
Check out spellbooks to see different examples apps built with Honcho
</Card>
</CardGroup>

12
docs/v2/documentation/introduction/vibecoding.mdx
View File

@ -5,7 +5,17 @@ description: "Universal starter prompt for building with Honcho"
sidebarTitle: 'Vibecoding Setup'
---
Copy this prompt into Cursor, Claude, or any AI coding assistant to start building with Honcho.
These docs are designed to be easily consumable for LLMs. Each page has a button
the lets you copy the page as Markdown or paste directly into ChatGPT or Claude.
Additionally, we follow the llms.txt standard. There are both an llms.txt and
llms-full.txt available.
- [llms.txt](/llms.txt)
- [llms-full.txt](/llms-full.txt)
Additionally, we provide a starter prompt to paste into a coding assistant to
quickly get started building with Honcho.
## 🚀 Universal Starter Prompt

4
docs/v2/documentation/reference/platform.mdx
View File

@ -1,8 +1,8 @@
---
title: "Managed Honcho Platform"
title: "The Honcho Dashboard"
icon: "rocket"
description: "Build socially intelligent agents without worrying about infrastructure"
sidebarTitle: "Platform Overview"
sidebarTitle: "Dashboard Overview"
---
<Card title="Sign up to start using Honcho!" icon="rocket" href="https://app.honcho.dev">

46
docs/v2/guides/overview.mdx
View File

@ -11,40 +11,20 @@ AI development often feels like magic - you craft the right prompt and get exact
Whether you're integrating Honcho into existing platforms, exploring advanced features, or getting up and running quickly, these guides provide concrete examples and implementation patterns.
## What You'll Find Here
Each spellbook focuses on a specific use case with working code you can adapt to your needs. The goal is to get you from idea to working prototype as quickly as possible, then provide the depth you need to scale and customize.
### Getting Started
**[Overview](/v2/guides/overview)** - You are here
**[MCP Integration](/v2/guides/mcp)** - Get Honcho running with a single prompt in Cursor or Claude Code
### Application Interfaces
## Application Interfaces
Ready-to-use integration patterns for popular platforms:
**[Discord Bot](/v2/guides/discord)** - Build a Discord bot that remembers users across conversations
**[Telegram Bot](/v2/guides/telegram)** - Create a Telegram bot with persistent user understanding
### Design Patterns
Implementation patterns for Honcho's core capabilities:
**[Dialectic Endpoint](/v2/guides/dialectic-endpoint)** - Query user psychology in natural language
**[Working with Session Context](/v2/guides/get-context)** - Manage conversation flow and context windows
**[Search](/v2/guides/search)** - Search your data using natural language
**[Working Representations](/v2/guides/working-rep)** - Understanding and customizing user models
**[Streaming Responses](/v2/guides/streaming-response)** - Handle real-time interactions efficiently
**[Using Filters](/v2/guides/using-filters)** - Control what data gets processed and how
**[File Uploads](/v2/guides/file-uploads)** - Upload PDF, text, or JSON files to create messages
## Philosophy
These aren't just API documentation - they're implementation patterns that solve real problems. Each spellbook focuses on a specific use case with working code you can adapt to your needs.
The goal is to get you from idea to working prototype as quickly as possible, then provide the depth you need to scale and customize.
<CardGroup cols={3}>
<Card title="Discord Bot" icon="discord" href="/v2/guides/discord">
Build a Discord bot that remembers users across conversations
</Card>
<Card title="Telegram Bot" icon="telegram" href="/v2/guides/telegram">
Create a Telegram bot with persistent user understanding
</Card>
<Card title="MCP" icon="link" href="/v2/guides/mcp">
Get Honcho running with a single prompt in Cursor or Claude Code
</Card>
</CardGroup>

13
migrations/env.py
View File

@ -1,4 +1,4 @@
import logging
import logging # noqa: I001
import sys
from logging.config import fileConfig
from pathlib import Path
@ -12,6 +12,10 @@ from src.config import settings
# Import your models
from src.db import Base
# Import all models so they register with Base.metadata
import src.models # noqa: F401
# Set up logging more verbosely
logging.basicConfig()
logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
@ -166,6 +170,13 @@ def run_migrations_online() -> None:
connection=connection,
target_metadata=target_metadata,
version_table_schema=target_metadata.schema,
include_schemas=True,
include_object=lambda obj, name, type_, reflected, compare_to: (
# Only include objects from our target schema
getattr(obj, "schema", None) == target_metadata.schema
if hasattr(obj, "schema")
else True
),
)
with context.begin_transaction():

10
migrations/script.py.mako
View File

@ -5,18 +5,20 @@ Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
from migrations.utils import get_schema
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
down_revision: str | None = ${repr(down_revision)}
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
schema = get_schema()
def upgrade() -> None:
${upgrades if upgrades else "pass"}

86
migrations/utils.py
View File

@ -71,3 +71,89 @@ def constraint_exists(
else:
raise ValueError(f"Invalid constraint type: {type}")
return any(constraint["name"] == constraint_name for constraint in constraints)
def make_column_non_nullable_safe(table_name: str, column_name: str) -> None:
"""
Make a column non-nullable using a non-blocking approach to minimize lock duration.
WARNING: Only use this if you can guarantee that:
1. No NULL values currently exist in the column
2. The application code is already writing non-NULL values to this column or
3. The column has never accepted NULLs in practice
This uses a 4-step process to avoid long exclusive locks:
1. Add CHECK constraint with NOT VALID (instant, no scan)
2. Validate the constraint (scans but allows concurrent read/writes to the table)
3. Set column NOT NULL (fast since we've validated the constraint)
4. Drop the redundant CHECK constraint
Args:
table_name: The name of the table
column_name: The name of the column to make non-nullable
"""
schema = get_schema()
conn = op.get_bind()
constraint_name = f"{table_name}_{column_name}_not_null"
# Step 1: Check if the column is already non-nullable
inspector = sa.inspect(op.get_bind())
columns = inspector.get_columns(table_name, schema=schema)
column_info = next((col for col in columns if col["name"] == column_name), None)
if column_info is None:
raise ValueError(f"Column {table_name}.{column_name} does not exist")
if not column_info["nullable"]:
print(f"Column {table_name}.{column_name} is already non-nullable, skipping...")
return
# Step 2: Add CHECK constraint without validation (instant)
# Note: op.create_check_constraint() doesn't support NOT VALID, so use raw SQL
# Get the identifier preparer for safe quoting
dialect = conn.dialect
preparer = dialect.identifier_preparer
quoted_schema = preparer.quote(schema)
quoted_table = preparer.quote(table_name)
quoted_constraint = preparer.quote(constraint_name)
quoted_column = preparer.quote(column_name)
# Step 2: Add CHECK constraint without validation (instant)
# Note: op.create_check_constraint() doesn't support NOT VALID, so use raw SQL
if not constraint_exists(table_name, constraint_name, "check"):
conn.execute(
sa.text(
f"""
ALTER TABLE {quoted_schema}.{quoted_table}
ADD CONSTRAINT {quoted_constraint}
CHECK ({quoted_column} IS NOT NULL)
NOT VALID
"""
)
)
# Step 3: Validate constraint (scans but allows concurrent operations)
conn.execute(
sa.text(
f"""
ALTER TABLE {quoted_schema}.{quoted_table}
VALIDATE CONSTRAINT {quoted_constraint}
"""
)
)
# Step 4: Set NOT NULL (fast with validated constraint)
op.alter_column(
table_name,
column_name,
nullable=False,
schema=schema,
)
# Step 5: Drop the redundant CHECK constraint
op.drop_constraint(
constraint_name,
table_name,
type_="check",
schema=schema,
)

341
migrations/versions/066e87ca5b07_align_schema_with_declarative_models.py Normal file
View File

@ -0,0 +1,341 @@
"""align_schema_with_declarative_models
Revision ID: 066e87ca5b07
Revises: bb6fb3a7a643
Create Date: 2025-10-27 12:36:51.614959
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from migrations.utils import (
column_exists,
constraint_exists,
fk_exists,
get_schema,
index_exists,
make_column_non_nullable_safe,
)
# revision identifiers, used by Alembic.
revision: str = "066e87ca5b07"
down_revision: str | None = "bb6fb3a7a643"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = get_schema()
def upgrade() -> None:
"""
The application code has been previously updated to ensure none of the following columns have NULL values but the actual DB schema is out of sync with our SQLAlchemy model definitions.
This migration fixes this by making the columns non-nullable using a non-blocking approach to minimize lock duration.
"""
conn = op.get_bind()
# Make peers.workspace_name non-nullable
if column_exists("peers", "workspace_name"):
make_column_non_nullable_safe("peers", "workspace_name")
# Make sessions.workspace_name non-nullable
if column_exists("sessions", "workspace_name"):
make_column_non_nullable_safe("sessions", "workspace_name")
# Make active_queue_sessions.work_unit_key non-nullable
if column_exists("active_queue_sessions", "work_unit_key"):
make_column_non_nullable_safe("active_queue_sessions", "work_unit_key")
# Make documents.embedding non-nullable
if column_exists("documents", "embedding"):
make_column_non_nullable_safe("documents", "embedding")
# Add primary key constraint to message_embeddings.id
if column_exists("message_embeddings", "id") and not constraint_exists(
"message_embeddings", "pk_message_embeddings", "primary"
):
conn.execute(
sa.text(
f"""
ALTER TABLE {schema}.message_embeddings
ADD CONSTRAINT pk_message_embeddings
PRIMARY KEY (id)
"""
)
)
# Rename indexes on peers table
inspector = sa.inspect(conn)
index_renames = [
("peers", "ix_users_created_at", "ix_peers_created_at"),
("peers", "ix_users_name", "ix_peers_name"),
("workspaces", "ix_apps_created_at", "ix_workspaces_created_at"),
("workspaces", "ix_apps_name", "ix_workspaces_name"),
]
for table_name, old_name, new_name in index_renames:
if index_exists(table_name, old_name, inspector):
conn.execute(
sa.text(f"ALTER INDEX {schema}.{old_name} RENAME TO {new_name}")
)
# Drop redundant indexes
if index_exists("workspaces", "ix_apps_public_id", inspector):
op.drop_index("ix_apps_public_id", table_name="workspaces", schema=schema)
if index_exists("peers", "ix_users_public_id", inspector):
op.drop_index("ix_users_public_id", table_name="peers", schema=schema)
if index_exists("sessions", "ix_sessions_public_id", inspector):
op.drop_index("ix_sessions_public_id", table_name="sessions", schema=schema)
if index_exists("documents", "ix_documents_public_id", inspector):
op.drop_index("ix_documents_public_id", table_name="documents", schema=schema)
if index_exists("collections", "ix_collections_public_id", inspector):
op.drop_index(
"ix_collections_public_id", table_name="collections", schema=schema
)
# Drop redundant unique constraints
if constraint_exists("workspaces", "uq_apps_public_id", "unique", inspector):
op.drop_constraint(
"uq_apps_public_id", "workspaces", type_="unique", schema=schema
)
if constraint_exists("peers", "uq_users_public_id", "unique", inspector):
op.drop_constraint("uq_users_public_id", "peers", type_="unique", schema=schema)
if constraint_exists("sessions", "uq_sessions_public_id", "unique", inspector):
op.drop_constraint(
"uq_sessions_public_id", "sessions", type_="unique", schema=schema
)
if constraint_exists(
"collections", "uq_collections_public_id", "unique", inspector
):
op.drop_constraint(
"uq_collections_public_id", "collections", type_="unique", schema=schema
)
if constraint_exists("documents", "uq_documents_public_id", "unique", inspector):
op.drop_constraint(
"uq_documents_public_id", "documents", type_="unique", schema=schema
)
# Drop unnecessary index on active queue
if index_exists(
"active_queue_sessions",
f"ix_{schema}_active_queue_sessions_work_unit_key",
inspector,
):
op.drop_index(
f"ix_{schema}_active_queue_sessions_work_unit_key",
table_name="active_queue_sessions",
schema=schema,
)
# Add FK constraint on queue.session_id to sessions.id
if not fk_exists("queue", "fk_queue_session_id"):
# Add constraint without validation (fast, doesn't scan)
conn.execute(
sa.text(
f"""
ALTER TABLE {schema}.queue
ADD CONSTRAINT fk_queue_session_id
FOREIGN KEY (session_id)
REFERENCES {schema}.sessions(id)
NOT VALID
"""
)
)
# Validate constraint (scans but allows concurrent reads)
conn.execute(
sa.text(
f"ALTER TABLE {schema}.queue VALIDATE CONSTRAINT fk_queue_session_id"
)
)
# Create missing indexes
if not index_exists("peers", "ix_peers_workspace_name", inspector):
op.create_index(
"ix_peers_workspace_name", "peers", ["workspace_name"], schema=schema
)
if not index_exists("collections", "ix_collections_workspace_name", inspector):
op.create_index(
"ix_collections_workspace_name",
"collections",
["workspace_name"],
schema=schema,
)
if not index_exists("documents", "ix_documents_workspace_name", inspector):
op.create_index(
"ix_documents_workspace_name",
"documents",
["workspace_name"],
schema=schema,
)
if not fk_exists("session_peers", "fk_session_peers_workspace_name", inspector):
# Add constraint without validation (fast, doesn't scan)
conn.execute(
sa.text(
f"""
ALTER TABLE {schema}.session_peers
ADD CONSTRAINT fk_session_peers_workspace_name
FOREIGN KEY (workspace_name)
REFERENCES {schema}.workspaces(name)
NOT VALID
"""
)
)
# Validate constraint (scans but allows concurrent reads)
conn.execute(
sa.text(
f"ALTER TABLE {schema}.session_peers VALIDATE CONSTRAINT fk_session_peers_workspace_name"
)
)
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
if fk_exists("session_peers", "fk_session_peers_workspace_name", inspector):
op.drop_constraint(
"fk_session_peers_workspace_name",
table_name="session_peers",
type_="foreignkey",
schema=schema,
)
if index_exists("documents", "ix_documents_workspace_name", inspector):
op.drop_index(
"ix_documents_workspace_name", table_name="documents", schema=schema
)
if index_exists("peers", "ix_peers_workspace_name", inspector):
op.drop_index("ix_peers_workspace_name", table_name="peers", schema=schema)
if index_exists("collections", "ix_collections_workspace_name", inspector):
op.drop_index(
"ix_collections_workspace_name", table_name="collections", schema=schema
)
# First, drop the FK constraint (we'll recreate it later if needed)
if fk_exists("queue", "fk_queue_session_id"):
op.drop_constraint(
"fk_queue_session_id",
"queue",
type_="foreignkey",
schema=schema,
)
if not index_exists(
"active_queue_sessions",
f"ix_{schema}_active_queue_sessions_work_unit_key",
inspector,
):
op.create_index(
f"ix_{schema}_active_queue_sessions_work_unit_key",
table_name="active_queue_sessions",
columns=["work_unit_key"],
schema=schema,
)
# Recreate the redundant unique constraints
if not constraint_exists("sessions", "uq_sessions_public_id", "unique", inspector):
op.create_unique_constraint(
"uq_sessions_public_id", "sessions", ["id"], schema=schema
)
if not constraint_exists("peers", "uq_users_public_id", "unique", inspector):
op.create_unique_constraint(
"uq_users_public_id", "peers", ["id"], schema=schema
)
if not constraint_exists("workspaces", "uq_apps_public_id", "unique", inspector):
op.create_unique_constraint(
"uq_apps_public_id", "workspaces", ["id"], schema=schema
)
if not constraint_exists(
"collections", "uq_collections_public_id", "unique", inspector
):
op.create_unique_constraint(
"uq_collections_public_id", "collections", ["id"], schema=schema
)
if not constraint_exists(
"documents", "uq_documents_public_id", "unique", inspector
):
op.create_unique_constraint(
"uq_documents_public_id", "documents", ["id"], schema=schema
)
# Recreate the redundant indexes
if not index_exists("sessions", "ix_sessions_public_id", inspector):
op.create_index("ix_sessions_public_id", "sessions", ["id"], schema=schema)
if not index_exists("peers", "ix_users_public_id", inspector):
op.create_index("ix_users_public_id", "peers", ["id"], schema=schema)
if not index_exists("workspaces", "ix_apps_public_id", inspector):
op.create_index("ix_apps_public_id", "workspaces", ["id"], schema=schema)
if not index_exists("documents", "ix_documents_public_id", inspector):
op.create_index("ix_documents_public_id", "documents", ["id"], schema=schema)
if not index_exists("collections", "ix_collections_public_id", inspector):
op.create_index(
"ix_collections_public_id",
"collections",
["id"],
schema=schema,
)
# Rename indexes on peers table back to original names
index_renames = [
("peers", "ix_peers_created_at", "ix_users_created_at"),
("peers", "ix_peers_name", "ix_users_name"),
("workspaces", "ix_workspaces_created_at", "ix_apps_created_at"),
("workspaces", "ix_workspaces_name", "ix_apps_name"),
]
for table_name, new_name, old_name in index_renames:
if index_exists(table_name, new_name, inspector):
conn.execute(
sa.text(f"ALTER INDEX {schema}.{new_name} RENAME TO {old_name}")
)
# Drop primary key constraint from message_embeddings.id
if constraint_exists("message_embeddings", "pk_message_embeddings", "primary"):
op.drop_constraint(
"pk_message_embeddings", "message_embeddings", "primary", schema=schema
)
# Make documents.embedding nullable
if column_exists("documents", "embedding"):
op.alter_column(
"documents",
"embedding",
nullable=True,
schema=schema,
)
# Make active_queue_sessions.work_unit_key nullable
if column_exists("active_queue_sessions", "work_unit_key"):
op.alter_column(
"active_queue_sessions",
"work_unit_key",
nullable=True,
schema=schema,
)
# Make sessions.workspace_name nullable
if column_exists("sessions", "workspace_name"):
op.alter_column(
"sessions",
"workspace_name",
nullable=True,
schema=schema,
)
# Make peers.workspace_name nullable
if column_exists("peers", "workspace_name"):
op.alter_column(
"peers",
"workspace_name",
nullable=True,
schema=schema,
)

7
migrations/versions/08894082221a_replace_collection_name_with_observer_.py
View File

@ -53,7 +53,7 @@ def upgrade() -> None:
connection.execute(
text(
f"""
INSERT INTO {schema}.sessions (id, name, workspace_name, is_active) VALUES (:session_id, '__global_observations__', :workspace_name, true) ON CONFLICT DO NOTHING
INSERT INTO {schema}.sessions (id, name, workspace_name, is_active, metadata, internal_metadata, configuration, created_at) VALUES (:session_id, '__global_observations__', :workspace_name, true, '{{}}', '{{}}', '{{}}', NOW()) ON CONFLICT DO NOTHING
"""
),
{"session_id": session_id, "workspace_name": workspace_name},
@ -446,7 +446,10 @@ def upgrade() -> None:
schema=schema,
)
# Step 17: Drop the name column from collections
# Step 17: Drop the name_length check constraint before dropping the name column from collections
if constraint_exists("collections", "name_length", "check", inspector):
op.drop_constraint("name_length", "collections", schema=schema)
if column_exists("collections", "name", inspector):
op.drop_column("collections", "name", schema=schema)

5
migrations/versions/20f89a421aff_rename_metamessage_type_to_label.py
View File

@ -11,6 +11,8 @@ from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from migrations.utils import constraint_exists
# revision identifiers, used by Alembic.
revision: str = "20f89a421aff"
down_revision: str | None = "556a16564f50"
@ -61,7 +63,8 @@ def upgrade() -> None:
)
# Rename check constraint
op.execute("ALTER TABLE metamessages DROP CONSTRAINT metamessage_type_length;")
if constraint_exists("metamessages", "metamessage_type_length", "check"):
op.execute("ALTER TABLE metamessages DROP CONSTRAINT metamessage_type_length;")
op.create_check_constraint("label_length", "metamessages", "length(label) <= 512")
# ### end Alembic commands ###

164
migrations/versions/b8183c5ffb48_codify_document_level_and_times_derived.py Normal file
View File

@ -0,0 +1,164 @@
"""codify_document_level_and_times_derived
Revision ID: b8183c5ffb48
Revises: ec8f94139b02
Create Date: 2025-10-31 12:48:54.597269
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy import text
from migrations.utils import column_exists, constraint_exists, get_schema
# revision identifiers, used by Alembic.
revision: str = "b8183c5ffb48"
down_revision: str | None = "ec8f94139b02"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = get_schema()
def upgrade() -> None:
"""Codify level and times_derived from internal_metadata into explicit columns."""
connection = op.get_bind()
inspector = sa.inspect(connection)
# Step 1: Add level column (nullable initially)
if not column_exists("documents", "level", inspector):
op.add_column(
"documents",
sa.Column(
"level",
sa.TEXT(),
nullable=True,
),
schema=schema,
)
# Step 2: Add times_derived column (nullable initially)
if not column_exists("documents", "times_derived", inspector):
op.add_column(
"documents",
sa.Column("times_derived", sa.Integer(), nullable=True),
schema=schema,
)
# Step 3: Populate level and times_derived from internal_metadata in batches
# Default to 'explicit' for level and 1 for times_derived if not present in metadata
batch_size = 5000
while True:
result = connection.execute(
text(
f"""
WITH batch AS (
SELECT id
FROM {schema}.documents
WHERE level IS NULL OR times_derived IS NULL
LIMIT :batch_size
)
UPDATE {schema}.documents d
SET
level = COALESCE(
d.internal_metadata->>'level',
'explicit'
),
times_derived = COALESCE(
(d.internal_metadata->>'times_derived')::integer,
1
)
FROM batch
WHERE d.id = batch.id
"""
),
{"batch_size": batch_size},
)
if result.rowcount == 0:
break
# Step 4: Make level NOT NULL with server default
op.alter_column(
"documents",
"level",
nullable=False,
server_default=text("'explicit'"),
schema=schema,
)
# Step 5: Make times_derived NOT NULL with server default
op.alter_column(
"documents",
"times_derived",
nullable=False,
server_default=text("1"),
schema=schema,
)
# Step 6: Add CHECK constraint for level
if not constraint_exists("documents", "level_valid", "check", inspector):
op.create_check_constraint(
"level_valid",
"documents",
"level IN ('explicit', 'deductive')",
schema=schema,
)
def downgrade() -> None:
"""Restore level and times_derived to internal_metadata."""
connection = op.get_bind()
inspector = sa.inspect(connection)
# Step 1: Drop CHECK constraint for level
if constraint_exists("documents", "level_valid", "check", inspector):
op.drop_constraint(
"level_valid",
"documents",
type_="check",
schema=schema,
)
# Step 2: Copy level and times_derived back to internal_metadata in batches (optional, for safety)
batch_size = 5000
while True:
result = connection.execute(
text(
f"""
WITH batch AS (
SELECT id
FROM {schema}.documents
WHERE internal_metadata IS NULL
OR NOT (internal_metadata ? 'level')
OR NOT (internal_metadata ? 'times_derived')
LIMIT :batch_size
)
UPDATE {schema}.documents d
SET internal_metadata = jsonb_set(
jsonb_set(
COALESCE(d.internal_metadata, '{{}}'::jsonb),
'{{level}}',
to_jsonb(d.level)
),
'{{times_derived}}',
to_jsonb(d.times_derived)
)
FROM batch
WHERE d.id = batch.id
"""
),
{"batch_size": batch_size},
)
if result.rowcount == 0:
break
# Step 3: Drop the level column
if column_exists("documents", "level", inspector):
op.drop_column("documents", "level", schema=schema)
# Step 4: Drop the times_derived column
if column_exists("documents", "times_derived", inspector):
op.drop_column("documents", "times_derived", schema=schema)

387
migrations/versions/e9b705f9adf9_add_server_defaults_to_timestamp_.py Normal file
View File

@ -0,0 +1,387 @@
"""add server defaults to timestamp boolean and jsonb columns
Revision ID: e9b705f9adf9
Revises: 066e87ca5b07
Create Date: 2025-10-29 12:08:36.803611
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from migrations.utils import get_schema
# revision identifiers, used by Alembic.
revision: str = "e9b705f9adf9"
down_revision: str | None = "066e87ca5b07"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = get_schema()
def upgrade() -> None:
# Add server defaults for timestamp columns
op.alter_column(
"workspaces",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"peers",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"sessions",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"messages",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"message_embeddings",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"collections",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"documents",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"queue",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"webhook_endpoints",
"created_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"session_peers",
"joined_at",
server_default=sa.func.now(),
schema=schema,
)
op.alter_column(
"active_queue_sessions",
"last_updated",
server_default=sa.func.now(),
schema=schema,
)
# Add server defaults for JSONB columns
op.alter_column(
"workspaces",
"metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"workspaces",
"internal_metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"workspaces",
"configuration",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"peers",
"metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"peers",
"internal_metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"peers",
"configuration",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"sessions",
"metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"sessions",
"internal_metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"sessions",
"configuration",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"messages",
"metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"messages",
"internal_metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"collections",
"metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"collections",
"internal_metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"documents",
"internal_metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"session_peers",
"configuration",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
op.alter_column(
"session_peers",
"internal_metadata",
server_default=sa.text("'{}'::jsonb"),
schema=schema,
)
# Add server defaults for boolean columns
op.alter_column(
"sessions",
"is_active",
server_default=sa.text("true"),
schema=schema,
)
op.alter_column(
"queue",
"processed",
server_default=sa.text("false"),
schema=schema,
)
def downgrade() -> None:
# Remove server defaults for timestamp columns
op.alter_column(
"workspaces",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"peers",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"sessions",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"messages",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"message_embeddings",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"collections",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"documents",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"queue",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"webhook_endpoints",
"created_at",
server_default=None,
schema=schema,
)
op.alter_column(
"session_peers",
"joined_at",
server_default=None,
schema=schema,
)
op.alter_column(
"active_queue_sessions",
"last_updated",
server_default=None,
schema=schema,
)
# Remove server defaults for JSONB columns
op.alter_column(
"workspaces",
"metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"workspaces",
"internal_metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"workspaces",
"configuration",
server_default=None,
schema=schema,
)
op.alter_column(
"peers",
"metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"peers",
"internal_metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"peers",
"configuration",
server_default=None,
schema=schema,
)
op.alter_column(
"sessions",
"metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"sessions",
"internal_metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"sessions",
"configuration",
server_default=None,
schema=schema,
)
op.alter_column(
"messages",
"metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"messages",
"internal_metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"collections",
"metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"collections",
"internal_metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"documents",
"internal_metadata",
server_default=None,
schema=schema,
)
op.alter_column(
"session_peers",
"configuration",
server_default=None,
schema=schema,
)
op.alter_column(
"session_peers",
"internal_metadata",
server_default=None,
schema=schema,
)
# Remove server defaults for boolean columns
op.alter_column(
"sessions",
"is_active",
server_default=None,
schema=schema,
)
op.alter_column(
"queue",
"processed",
server_default=None,
schema=schema,
)

252
migrations/versions/ec8f94139b02_codify_workspace_name_and_message_id_in_.py Normal file
View File

@ -0,0 +1,252 @@
"""codify workspace_name and message_id in queue table
Revision ID: ec8f94139b02
Revises: e9b705f9adf9
Create Date: 2025-10-28 17:39:51.778665
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from migrations.utils import column_exists, fk_exists, get_schema, index_exists
# revision identifiers, used by Alembic.
revision: str = "ec8f94139b02"
down_revision: str | None = "e9b705f9adf9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
schema = get_schema()
def upgrade() -> None:
# Step 1: Drop rows where workspace_name is NULL in payload
# This removes invalid/corrupted queue items
conn = op.get_bind()
batch_size = 10000
while True:
result = conn.execute(
sa.text(
f"""
DELETE FROM "{schema}".queue
WHERE id IN (
SELECT id FROM "{schema}".queue
WHERE payload->>'workspace_name' IS NULL
LIMIT :batch_size
)
"""
),
{"batch_size": batch_size},
)
if result.rowcount == 0:
break
# Step 2: Add workspace_name column (nullable initially for backfill)
op.add_column(
"queue",
sa.Column("workspace_name", sa.TEXT(), nullable=True),
schema=schema,
)
# Step 3: Add message_id column (nullable, as not all tasks have message_id)
op.add_column(
"queue",
sa.Column("message_id", sa.BigInteger(), nullable=True),
schema=schema,
)
# Step 4: Backfill workspace_name and message_id from payload in batches
batch_size = 5000
while True:
result = conn.execute(
sa.text(
f"""
WITH batch AS (
SELECT id
FROM "{schema}".queue
WHERE workspace_name IS NULL
OR (message_id IS NULL AND payload ? 'message_id' AND payload->>'message_id' IS NOT NULL)
LIMIT :batch_size
)
UPDATE "{schema}".queue q
SET
workspace_name = COALESCE(q.workspace_name, q.payload->>'workspace_name'),
message_id = COALESCE(
q.message_id,
CASE
WHEN q.payload ? 'message_id' AND q.payload->>'message_id' IS NOT NULL
THEN (q.payload->>'message_id')::bigint
END
)
FROM batch
WHERE q.id = batch.id
"""
),
{"batch_size": batch_size},
)
if result.rowcount == 0:
break
# Step 5: Remove workspace_name and message_id from JSONB payloads in batches
while True:
result = conn.execute(
sa.text(
f"""
WITH batch AS (
SELECT id
FROM "{schema}".queue
WHERE payload ? 'workspace_name' OR payload ? 'message_id'
LIMIT :batch_size
)
UPDATE "{schema}".queue q
SET payload = q.payload - 'workspace_name' - 'message_id'
FROM batch
WHERE q.id = batch.id
"""
),
{"batch_size": batch_size},
)
if result.rowcount == 0:
break
# Step 6: Make workspace_name non-nullable
op.alter_column("queue", "workspace_name", nullable=False, schema=schema)
# Step 7: Add foreign key constraint on workspace_name -> workspaces.name
op.create_foreign_key(
"fk_queue_workspace_name",
"queue",
"workspaces",
["workspace_name"],
["name"],
source_schema=schema,
referent_schema=schema,
)
# Step 7b: Add foreign key constraint on message_id -> messages.id
op.create_foreign_key(
"fk_queue_message_id",
"queue",
"messages",
["message_id"],
["id"],
source_schema=schema,
referent_schema=schema,
)
# Step 8: Add index on workspace_name (for FK performance and filtering)
op.create_index(
op.f("ix_queue_workspace_name"),
"queue",
["workspace_name"],
unique=False,
schema=schema,
)
# Step 9: Add partial index on message_id WHERE message_id IS NOT NULL
# This optimizes JOINs with the messages table
op.create_index(
"ix_queue_message_id_not_null",
"queue",
["message_id"],
unique=False,
schema=schema,
postgresql_where=sa.text("message_id IS NOT NULL"),
)
# Step 10: Add composite index on (workspace_name, processed)
# This optimizes queries that filter unprocessed items by workspace
op.create_index(
"ix_queue_workspace_name_processed",
"queue",
["workspace_name", "processed"],
unique=False,
schema=schema,
)
# Step 11: Add composite index on (work_unit_key, processed, id)
# This is critical for the hot path: "get next unprocessed item for this work unit"
# Covers: WHERE work_unit_key = ? AND NOT processed ORDER BY id
op.create_index(
"ix_queue_work_unit_key_processed_id",
"queue",
["work_unit_key", "processed", "id"],
unique=False,
schema=schema,
)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
# Drop indexes
if index_exists("queue", "ix_queue_work_unit_key_processed_id", inspector):
op.drop_index(
"ix_queue_work_unit_key_processed_id", table_name="queue", schema=schema
)
if index_exists("queue", "ix_queue_workspace_name_processed", inspector):
op.drop_index(
"ix_queue_workspace_name_processed", table_name="queue", schema=schema
)
if index_exists("queue", "ix_queue_message_id_not_null", inspector):
op.drop_index("ix_queue_message_id_not_null", table_name="queue", schema=schema)
if index_exists("queue", "ix_queue_workspace_name", inspector):
op.drop_index(
op.f("ix_queue_workspace_name"), table_name="queue", schema=schema
)
# Drop foreign key constraints
if fk_exists("queue", "fk_queue_message_id", inspector):
op.drop_constraint("fk_queue_message_id", "queue", schema=schema)
if fk_exists("queue", "fk_queue_workspace_name", inspector):
op.drop_constraint("fk_queue_workspace_name", "queue", schema=schema)
# Restore workspace_name and message_id to payload in batches
conn = op.get_bind()
batch_size = 5000
if column_exists("queue", "workspace_name", inspector) or column_exists(
"queue", "message_id", inspector
):
while True:
result = conn.execute(
sa.text(
f"""
WITH batch AS (
SELECT id
FROM "{schema}".queue
WHERE (workspace_name IS NOT NULL AND NOT (payload ? 'workspace_name'))
OR (message_id IS NOT NULL AND NOT (payload ? 'message_id'))
LIMIT :batch_size
)
UPDATE "{schema}".queue q
SET payload = q.payload
|| CASE WHEN q.workspace_name IS NOT NULL AND NOT (q.payload ? 'workspace_name')
THEN jsonb_build_object('workspace_name', q.workspace_name)
ELSE '{{}}'::jsonb END
|| CASE WHEN q.message_id IS NOT NULL AND NOT (q.payload ? 'message_id')
THEN jsonb_build_object('message_id', q.message_id)
ELSE '{{}}'::jsonb END
FROM batch
WHERE q.id = batch.id
"""
),
{"batch_size": batch_size},
)
if result.rowcount == 0:
break
# Drop columns
if column_exists("queue", "message_id", inspector):
op.drop_column("queue", "message_id", schema=schema)
if column_exists("queue", "workspace_name", inspector):
op.drop_column("queue", "workspace_name", schema=schema)

14
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "honcho"
version = "2.4.1"
version = "2.4.2"
description = "Honcho Server"
authors = [
{name = "Plastic Labs", email = "hello@plasticlabs.ai"},
@ -33,6 +33,8 @@ dependencies = [
"typing-extensions>=4.11.0",
"prometheus-client>=0.20.0",
"json-repair>=0.49.0",
"redis>=6.0.0",
"cashews[redis]==7.4.1",
]
[tool.uv]
dev-dependencies = [
@ -47,6 +49,7 @@ dev-dependencies = [
"pre-commit>=4.2.0",
"pytest-cov>=6.2.1",
"honcho-ai",
"fakeredis>=2.32.0",
]
[tool.uv.workspace]
@ -85,6 +88,15 @@ addopts = "--strict-markers --cov=src/ --cov=sdks/python/src/honcho --cov-report
testpaths = ["tests"]
pythonpath = ["src"]
[tool.coverage.report]
exclude_lines = [
"pragma: no cover",
"def __repr__",
"raise AssertionError",
"raise NotImplementedError",
"if __name__ == .__main__.:",
"if TYPE_CHECKING:",
]
[tool.basedpyright]
# BasedPyright currently seems like the best type checker option, much faster

141
scripts/run_alembic_tests.py Executable file
View File

@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""
Run alembic tests selectively based on changed files.
Note that this script is actively used within our precommit hooks and should not be removed.
If this script is moved, the corresponding precommit hook will need to be updated.
This script determines which specific alembic tests to run based on:
1. If a test file changed, run the test for that revision
2. If a migration file changed, run the corresponding test
The alembic test system uses a parameterized test in test_pipeline.py that runs
for each revision. We filter these tests using pytest's -k flag with the revision IDs.
Usage: python scripts/run_alembic_tests.py <file1> <file2> ...
"""
import re
import subprocess
import sys
from pathlib import Path
def extract_revision_id(filepath: Path) -> str | None:
"""Extract the revision ID from a migration or test filename.
Migration files: {revision_id}_{description}.py
Test files: test_{revision_id}_{description}.py
Returns the revision_id (e.g., "05486ce795d5") or None if not found.
"""
filename = filepath.name
# Remove .py extension
if not filename.endswith(".py"):
return None
filename = filename[:-3]
# Remove test_ prefix if present
if filename.startswith("test_"):
filename = filename[5:]
# Extract revision ID (first part before underscore)
# Revision IDs are typically 12 characters of hex
match = re.match(r"^([a-f0-9]{12})_", filename)
if match:
return match.group(1)
return None
def main():
if len(sys.argv) < 2:
print("No files to check, skipping alembic tests")
sys.exit(0)
changed_files = [Path(f) for f in sys.argv[1:]]
# Paths
repo_root = Path(__file__).parent.parent
migrations_dir = repo_root / "migrations" / "versions"
tests_dir = repo_root / "tests" / "alembic" / "revisions"
alembic_tests_dir = repo_root / "tests" / "alembic"
# Collect revision IDs to test
revision_ids: set[str] = set()
run_full_suite = False
for filepath in changed_files:
filepath = Path(filepath).resolve()
# Check if file is under tests/alembic (including subdirectories)
# If it's not a revision-specific test file, run full suite
if (
filepath.parent == alembic_tests_dir
or alembic_tests_dir in filepath.parents
) and not (filepath.parent == tests_dir and filepath.name.startswith("test_")):
run_full_suite = True
print(
f"Infrastructure file changed: {filepath.name} -> will run full test suite"
)
continue
# Case 1: Test file changed - extract its revision ID
if filepath.parent == tests_dir and filepath.name.startswith("test_"):
revision_id = extract_revision_id(filepath)
if revision_id:
revision_ids.add(revision_id)
print(
f"Test file changed: {filepath.name} -> testing revision {revision_id}"
)
# Case 2: Migration file changed - extract its revision ID
elif filepath.parent == migrations_dir:
revision_id = extract_revision_id(filepath)
if revision_id:
revision_ids.add(revision_id)
print(
f"Migration changed: {filepath.name} -> testing revision {revision_id}"
)
if run_full_suite:
# Run full test suite without -k filter
print("\nRunning full alembic test suite due to infrastructure file changes\n")
cmd = [
"uv",
"run",
"pytest",
"tests/alembic/test_pipeline.py",
]
elif revision_ids:
# Build a -k expression to filter tests by revision ID
# pytest -k "rev1 or rev2 or rev3"
k_expression = " or ".join(sorted(revision_ids))
print(
f"\nRunning tests for {len(revision_ids)} revision(s): {', '.join(sorted(revision_ids))}"
)
print()
# Run pytest on test_pipeline.py with -k filter
cmd = [
"uv",
"run",
"pytest",
"tests/alembic/test_pipeline.py",
"-k",
k_expression,
]
else:
print("No alembic tests to run")
sys.exit(0)
result = subprocess.run(cmd, cwd=repo_root)
sys.exit(result.returncode)
if __name__ == "__main__":
main()

11
src/cache/__init__.py vendored Normal file
View File

@ -0,0 +1,11 @@
"""Cache module for Honcho."""
from src.cache.client import (
close_cache,
init_cache,
)
__all__ = [
"close_cache",
"init_cache",
]

123
src/cache/client.py vendored Normal file
View File

@ -0,0 +1,123 @@
from __future__ import annotations
import asyncio
import logging
from typing import cast
import sentry_sdk
from cashews import cache
from cashews.picklers import PicklerType
from redis import exceptions as redis_exc
from tenacity import (
AsyncRetrying,
retry_if_exception_type,
stop_after_delay,
wait_exponential_jitter,
)
from src.config import settings
logger = logging.getLogger(__name__)
_cache_lock = asyncio.Lock()
def is_cache_enabled() -> bool:
return settings.CACHE.ENABLED
def get_cache_namespace() -> str:
# CACHE.NAMESPACE is guaranteed to be non-None by AppSettings.propagate_namespace validator
return cast(str, settings.CACHE.NAMESPACE)
async def init_cache() -> None:
"""Initialize and verify cache connection if enabled."""
async with _cache_lock:
# Close existing backends to force recreation with new ContextVars
await cache.close()
if not is_cache_enabled():
# Use in-memory cache when caching is disabled
logger.info("Cache disabled, using in-memory cache")
cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType]
return
# Setup cache with Redis backend
try:
cache.setup( # pyright: ignore[reportUnknownMemberType]
settings.CACHE.URL,
pickle_type=PicklerType.SQLALCHEMY,
)
except Exception as setup_err:
logger.warning(
"Cache setup failed for %s: %s. Falling back to in-memory cache",
settings.CACHE.URL,
setup_err,
)
if settings.SENTRY.ENABLED:
sentry_sdk.capture_exception(setup_err)
# Fallback to in-memory cache
cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType]
return
cache.enable()
# Retry Redis ping with exponential backoff
try:
async for attempt in AsyncRetrying(
wait=wait_exponential_jitter(initial=0.2, max=2.0),
stop=stop_after_delay(5), # give it a bit more headroom
retry=retry_if_exception_type(
(
redis_exc.TimeoutError,
redis_exc.ConnectionError,
asyncio.TimeoutError,
TimeoutError,
)
),
reraise=True,
):
with attempt:
async with asyncio.timeout(2):
await cache.ping()
logger.info("Connected to cache at %s", settings.CACHE.URL)
except (
redis_exc.TimeoutError,
redis_exc.ConnectionError,
asyncio.TimeoutError,
TimeoutError,
) as e:
logger.warning(
"Failed to connect to cache at %s: %s. Falling back to in-memory cache",
settings.CACHE.URL,
e,
)
if settings.SENTRY.ENABLED:
sentry_sdk.capture_exception(e)
# Fallback to in-memory cache
await cache.close()
cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType]
except Exception as e:
logger.warning(
"Unexpected cache error at %s: %s. Falling back to in-memory cache",
settings.CACHE.URL,
e,
)
if settings.SENTRY.ENABLED:
sentry_sdk.capture_exception(e)
# Fallback to in-memory cache
await cache.close()
cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType]
async def close_cache() -> None:
await cache.close()
__all__ = [
"init_cache",
"close_cache",
"cache",
]

37
src/config.py
View File

@ -59,6 +59,7 @@ class TomlConfigSettingsSource(PydanticBaseSettingsSource):
"DB": "db",
"AUTH": "auth",
"SENTRY": "sentry",
"CACHE": "cache",
"LLM": "llm",
"DERIVER": "deriver",
"PEER_CARD": "peer_card",
@ -228,6 +229,9 @@ class DeriverSettings(BackupLLMSettingsMixin, HonchoSettings):
PROVIDER: SupportedProviders = "google"
MODEL: str = "gemini-2.5-flash-lite"
# Whether to deduplicate documents when creating them
DEDUPLICATE: bool = True
MAX_OUTPUT_TOKENS: Annotated[int, Field(default=10_000, gt=0, le=100_000)] = 10_000
# Thinking budget tokens are only applied when using Anthropic as provider
THINKING_BUDGET_TOKENS: Annotated[int, Field(default=1024, gt=0, le=5000)] = 1024
@ -319,7 +323,22 @@ class MetricsSettings(HonchoSettings):
model_config = SettingsConfigDict(env_prefix="METRICS_", extra="ignore") # pyright: ignore
ENABLED: bool = False
NAMESPACE: str = "honcho"
NAMESPACE: str | None = None
class CacheSettings(HonchoSettings):
model_config = SettingsConfigDict(env_prefix="CACHE_", extra="ignore") # pyright: ignore
ENABLED: bool = False
URL: str = "redis://localhost:6379/0?suppress=false"
NAMESPACE: str | None = None
DEFAULT_TTL_SECONDS: Annotated[int, Field(default=300, ge=1, le=86_400)] = (
300 # how long to keep items in cache
)
DEFAULT_LOCK_TTL_SECONDS: Annotated[int, Field(default=5, ge=1, le=86_400)] = (
5 # how long to hold a lock on a resource when fetching DB after cache miss
)
class DreamSettings(BackupLLMSettingsMixin, HonchoSettings):
@ -363,6 +382,8 @@ class AppSettings(HonchoSettings):
COLLECT_METRICS_LOCAL: bool = False
LOCAL_METRICS_FILE: str = "metrics.jsonl"
NAMESPACE: str = "honcho" # Top-level namespace for all settings, can be overridden by nested-model settings
# Nested settings models
DB: DBSettings = Field(default_factory=DBSettings)
AUTH: AuthSettings = Field(default_factory=AuthSettings)
@ -374,6 +395,7 @@ class AppSettings(HonchoSettings):
SUMMARY: SummarySettings = Field(default_factory=SummarySettings)
WEBHOOK: WebhookSettings = Field(default_factory=WebhookSettings)
METRICS: MetricsSettings = Field(default_factory=MetricsSettings)
CACHE: CacheSettings = Field(default_factory=CacheSettings)
DREAM: DreamSettings = Field(default_factory=DreamSettings)
@field_validator("LOG_LEVEL")
@ -383,6 +405,19 @@ class AppSettings(HonchoSettings):
raise ValueError(f"Invalid log level: {v}")
return log_level
@model_validator(mode="after")
def propagate_namespace(self) -> "AppSettings":
"""Propagate top-level NAMESPACE to nested settings if not explicitly set.
After this validator runs, CACHE.NAMESPACE and METRICS.NAMESPACE are guaranteed
to exist.
"""
if self.CACHE.NAMESPACE is None:
self.CACHE.NAMESPACE = self.NAMESPACE
if self.METRICS.NAMESPACE is None:
self.METRICS.NAMESPACE = self.NAMESPACE
return self
# Create a single global instance of the settings
settings: AppSettings = AppSettings()

80
src/crud/document.py
View File

@ -103,9 +103,10 @@ async def create_documents(
*,
observer: str,
observed: str,
deduplicate: bool = False,
) -> int:
"""
Create multiple documents with NO duplicate detection.
Create multiple documents with optional duplicate detection.
Args:
db: Database session
@ -120,6 +121,16 @@ async def create_documents(
honcho_documents: list[models.Document] = []
for doc in documents:
try:
# for each document, if deduplicate is True, perform a process
# that checks against existing documents and either rejects this document
# as a duplicate OR deletes an existing document that is a duplicate.
if deduplicate:
is_duplicate = await is_rejected_duplicate(
db, doc, workspace_name, observer=observer, observed=observed
)
if is_duplicate:
continue
metadata_dict = doc.metadata.model_dump(exclude_none=True)
honcho_documents.append(
models.Document(
@ -127,6 +138,8 @@ async def create_documents(
observer=observer,
observed=observed,
content=doc.content,
level=doc.level,
times_derived=doc.times_derived,
internal_metadata=metadata_dict,
embedding=doc.embedding,
session_name=doc.session_name,
@ -147,3 +160,68 @@ async def create_documents(
) from e
return len(honcho_documents)
async def is_rejected_duplicate(
db: AsyncSession,
doc: schemas.DocumentCreate,
workspace_name: str,
*,
observer: str,
observed: str,
) -> bool:
"""
Check if a document is a duplicate of an existing document.
Uses: 1) Cosine similarity (>=0.95), 2) Token diff for retention.
Returns True if both:
- the document is deemed a duplicate of an existing document
- the existing document is deemed a superior duplicate
If the document is not a duplicate, returns False.
If the document is a duplicate AND the new document is superior,
deletes the existing document and returns False.
"""
# Step 1: Find potential duplicates using cosine similarity
similar_docs = await query_documents(
db=db,
workspace_name=workspace_name,
query=doc.content,
observer=observer,
observed=observed,
max_distance=0.05,
top_k=1,
embedding=doc.embedding,
)
if not similar_docs:
return False
existing_doc = similar_docs[0]
# Step 2: Determine which has more information using token set difference
tokens_new = set(embedding_client.encoding.encode(doc.content))
tokens_existing = set(embedding_client.encoding.encode(existing_doc.content))
unique_new = len(tokens_new - tokens_existing)
unique_existing = len(tokens_existing - tokens_new)
score_new = len(tokens_new) + (unique_new * 10)
score_existing = len(tokens_existing) + (unique_existing * 10)
# If new document has more or equal information, keep it and delete existing
if score_new >= score_existing:
logger.warning(
f"[DUPLICATE DETECTION] Deleting existing in favor of new. new='{doc.content}', existing='{existing_doc.content}'."
)
await db.delete(existing_doc)
await db.flush() # Flush to make deletion visible in this transaction
return False # Don't reject the new document
# Existing document has more information, reject the new one
logger.warning(
f"[DUPLICATE DETECTION] Rejecting new in favor of existing. new='{doc.content}', existing='{existing_doc.content}'."
)
return True

83
src/crud/peer.py
View File

@ -1,16 +1,35 @@
from logging import getLogger
from typing import Any
from cashews import NOT_NONE
from sqlalchemy import Select, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from src import models, schemas
from src.cache.client import cache, get_cache_namespace
from src.config import settings
from src.crud.workspace import get_or_create_workspace
from src.exceptions import ConflictException, ResourceNotFoundException
from src.utils.filter import apply_filter
logger = getLogger(__name__)
PEER_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}:peer:{peer_name}"
PEER_LOCK_PREFIX = f"{get_cache_namespace()}:lock"
def peer_cache_key(workspace_name: str, peer_name: str) -> str:
"""Generate cache key for peer."""
return (
get_cache_namespace()
+ ":"
+ PEER_CACHE_KEY_TEMPLATE.format(
workspace_name=workspace_name,
peer_name=peer_name,
)
)
async def get_or_create_peers(
db: AsyncSession,
@ -35,6 +54,8 @@ async def get_or_create_peers(
Raises:
ConflictException: If we fail to get or create the peers
"""
await get_or_create_workspace(db, schemas.WorkspaceCreate(name=workspace_name))
peer_names = [p.name for p in peers]
stmt = (
select(models.Peer)
@ -75,8 +96,6 @@ async def get_or_create_peers(
try:
db.add_all(new_peers)
await db.commit()
# Return combined list of existing and new peers
return existing_peers + new_peers
except IntegrityError:
await db.rollback()
if _retry:
@ -85,6 +104,37 @@ async def get_or_create_peers(
) from None
return await get_or_create_peers(db, workspace_name, peers, _retry=True)
# Invalidate cache for all updated/created peers - read-through pattern
for peer_obj in existing_peers + new_peers:
cache_key = peer_cache_key(workspace_name, peer_obj.name)
await cache.delete(cache_key)
# Return combined list of existing and new peers
return existing_peers + new_peers
@cache(
key=PEER_CACHE_KEY_TEMPLATE,
ttl=f"{settings.CACHE.DEFAULT_TTL_SECONDS}s",
prefix=get_cache_namespace(),
condition=NOT_NONE,
)
@cache.locked(
key=PEER_CACHE_KEY_TEMPLATE,
ttl=f"{settings.CACHE.DEFAULT_LOCK_TTL_SECONDS}s",
prefix=PEER_LOCK_PREFIX,
)
async def _fetch_peer(
db: AsyncSession,
workspace_name: str,
peer_name: str,
) -> models.Peer | None:
return await db.scalar(
select(models.Peer)
.where(models.Peer.workspace_name == workspace_name)
.where(models.Peer.name == peer_name)
)
async def get_peer(
db: AsyncSession,
@ -105,21 +155,16 @@ async def get_peer(
Raises:
ResourceNotFoundException: If the peer does not exist
"""
# Try to get the existing peer
stmt = (
select(models.Peer)
.where(models.Peer.workspace_name == workspace_name)
.where(models.Peer.name == peer.name)
)
result = await db.execute(stmt)
existing_peer = result.scalar_one_or_none()
existing_peer = await _fetch_peer(db, workspace_name, peer.name)
if existing_peer is None:
raise ResourceNotFoundException(
f"Peer {peer.name} not found in workspace {workspace_name}"
)
if existing_peer is not None:
return existing_peer
# Merge cached object into session (cached objects are detached)
existing_peer = await db.merge(existing_peer, load=False)
raise ResourceNotFoundException(
f"Peer {peer.name} not found in workspace {workspace_name}"
)
return existing_peer
async def get_peers(
@ -166,7 +211,13 @@ async def update_peer(
honcho_peer.configuration = peer.configuration
await db.commit()
logger.debug(f"Peer {peer_name} updated successfully")
await db.refresh(honcho_peer)
# Invalidate cache - read-through pattern
cache_key = peer_cache_key(workspace_name, honcho_peer.name)
await cache.delete(cache_key)
logger.debug("Peer %s updated successfully", peer_name)
return honcho_peer

11
src/crud/peer_card.py
View File

@ -7,7 +7,8 @@ from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncSession
from src import exceptions, models, schemas
from src.crud.peer import get_peer
from src.cache.client import cache
from src.crud.peer import get_peer, peer_cache_key
logger = logging.getLogger(__name__)
@ -81,14 +82,20 @@ async def set_peer_card(
}
)
)
.returning(models.Peer)
)
result = await db.execute(stmt)
if result.rowcount == 0:
updated_peer = result.scalar_one_or_none()
if updated_peer is None:
raise exceptions.ResourceNotFoundException(
f"Peer {observer} not found in workspace {workspace_name}"
)
await db.commit()
# Invalidate cache - read-through pattern
cache_key = peer_cache_key(workspace_name, observer)
await cache.delete(cache_key)
def construct_peer_card_label(*, observer: str, observed: str) -> str:
if observer == observed:

117
src/crud/representation.py
View File

@ -145,7 +145,6 @@ class RepresentationManager:
metadata: schemas.DocumentMetadata = schemas.DocumentMetadata(
message_ids=[message_id_range],
level=obs_level,
premises=obs_premises,
message_created_at=format_datetime_utc(message_created_at),
)
@ -154,18 +153,20 @@ class RepresentationManager:
schemas.DocumentCreate(
content=obs_content,
session_name=session_name,
level=obs_level,
metadata=metadata,
embedding=embedding,
)
)
# Use bulk creation with NO duplicate detection
# Use bulk creation with optional duplicate detection
new_documents = await crud.create_documents(
db,
documents_to_create,
self.workspace_name,
observer=self.observer,
observed=self.observed,
deduplicate=settings.DERIVER.DEDUPLICATE,
)
try:
@ -175,41 +176,6 @@ class RepresentationManager:
return new_documents
async def get_relevant_observations(
self,
query: str,
*,
top_k: int = 5,
max_distance: float = 0.3,
level: str | None = None,
conversation_context: str = "",
) -> Representation:
"""
Unified method to get relevant observations with flexible options.
Args:
query: The search query
top_k: Number of results to return
max_distance: Maximum distance for semantic similarity
level: Optional reasoning level to filter by
conversation_context: Additional conversation context
Returns:
Representation
"""
async with tracked_db("representation_manager.get_relevant_observations") as db:
documents = await self._get_observations_internal(
db,
query,
top_k,
max_distance,
level,
conversation_context,
)
# convert documents to representation
return Representation.from_documents(documents)
async def get_working_representation(
self,
*,
@ -330,7 +296,6 @@ class RepresentationManager:
top_k: int,
max_distance: float | None = None,
level: str | None = None,
conversation_context: str = "",
) -> list[models.Document]:
"""Query documents by semantic similarity."""
try:
@ -339,7 +304,6 @@ class RepresentationManager:
db,
query,
level,
conversation_context,
top_k,
max_distance,
)
@ -349,7 +313,7 @@ class RepresentationManager:
workspace_name=self.workspace_name,
observer=self.observer,
observed=self.observed,
query=self._build_truncated_query(query, conversation_context),
query=query,
max_distance=max_distance,
top_k=top_k,
)
@ -397,7 +361,7 @@ class RepresentationManager:
models.Document.observer == self.observer,
models.Document.observed == self.observed,
)
.order_by(models.Document.internal_metadata["times_derived"].desc())
.order_by(models.Document.times_derived.desc())
)
result = await db.execute(stmt)
@ -412,11 +376,10 @@ class RepresentationManager:
top_k: int,
max_distance: float,
level: str | None,
conversation_context: str,
) -> list[models.Document]:
"""Internal method that does the actual observation retrieval."""
return await self._query_documents_semantic(
db, query, top_k, max_distance, level, conversation_context
db, query, top_k, max_distance, level
)
async def _query_documents_for_level(
@ -424,7 +387,6 @@ class RepresentationManager:
db: AsyncSession,
query: str,
level: str,
conversation_context: str,
count: int,
max_distance: float | None = None,
) -> list[models.Document]:
@ -434,7 +396,7 @@ class RepresentationManager:
workspace_name=self.workspace_name,
observer=self.observer,
observed=self.observed,
query=self._build_truncated_query(query, conversation_context),
query=query,
max_distance=max_distance,
top_k=count * FILTER_OVERSAMPLING_FACTOR,
filters=self._build_filter_conditions(level),
@ -454,76 +416,13 @@ class RepresentationManager:
conditions: list[dict[str, Any]] = []
if level:
conditions.append({"internal_metadata": {"level": level}})
conditions.append({"level": level})
if not conditions:
return {}
return conditions[0] if len(conditions) == 1 else {"AND": conditions}
def _build_truncated_query(
self,
query: str,
conversation_context: str = "",
max_tokens: int | None = None,
) -> str:
"""Build a query that fits within token limits with clear priorities.
Args:
query: The search query
conversation_context: Optional conversation context to include
max_tokens: Maximum tokens allowed (defaults to setting with buffer)
Returns:
Truncated query string that fits within token limits
"""
max_tokens = max_tokens or (settings.MAX_EMBEDDING_TOKENS - 100)
encoding = embedding_client.encoding
# Pre-calculate all token counts once
query_prefix = "Current message: "
context_prefix = "\nContext: "
prefix_tokens = len(encoding.encode(query_prefix))
context_prefix_tokens = len(encoding.encode(context_prefix))
query_tokens = encoding.encode(query)
# Simple case: query alone fits
if prefix_tokens + len(query_tokens) <= max_tokens:
if not conversation_context:
return f"{query_prefix}{query}"
# Try to add context
context_tokens = encoding.encode(conversation_context)
total_without_context = (
prefix_tokens + len(query_tokens) + context_prefix_tokens
)
if total_without_context + len(context_tokens) <= max_tokens:
return f"{query_prefix}{query}{context_prefix}{conversation_context}"
# Truncate context to fit
available_context_tokens = max_tokens - total_without_context
if available_context_tokens > 0:
truncated_context = encoding.decode(
context_tokens[-available_context_tokens:]
)
return f"{query_prefix}{query}{context_prefix}{truncated_context}"
else:
# No room left for context; keep full query intact
return f"{query_prefix}{query}"
# Query itself is too long - truncate it
available_query_tokens = max_tokens - prefix_tokens
if available_query_tokens > 0:
# Keep the end (recency) of the query
truncated_query = encoding.decode(query_tokens[-available_query_tokens:])
return f"{query_prefix}{truncated_query}"
# Pathological case - just return what we can
logger.warning("Token limit too restrictive: %s", max_tokens)
return encoding.decode(query_tokens[:max_tokens])
# Module-level functions for backward compatibility and convenience

122
src/crud/session.py
View File

@ -1,6 +1,7 @@
from logging import getLogger
from typing import Any
from cashews import NOT_NONE
from nanoid import generate as generate_nanoid
from sqlalchemy import Select, case, cast, func, insert, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
@ -9,6 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.types import BigInteger, Boolean
from src import models, schemas
from src.cache.client import cache, get_cache_namespace
from src.config import settings
from src.exceptions import (
ConflictException,
@ -18,12 +20,48 @@ from src.exceptions import (
from src.utils.filter import apply_filter
from .peer import get_or_create_peers, get_peer
# Import workspace and peer functions that are needed
from .workspace import get_or_create_workspace
logger = getLogger(__name__)
SESSION_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}:session:{session_name}"
SESSION_LOCK_PREFIX = f"{get_cache_namespace()}:lock"
def session_cache_key(workspace_name: str, session_name: str) -> str:
"""Generate cache key for session."""
return (
get_cache_namespace()
+ ":"
+ SESSION_CACHE_KEY_TEMPLATE.format(
workspace_name=workspace_name,
session_name=session_name,
)
)
@cache(
key=SESSION_CACHE_KEY_TEMPLATE,
ttl=f"{settings.CACHE.DEFAULT_TTL_SECONDS}s",
prefix=get_cache_namespace(),
condition=NOT_NONE,
)
@cache.locked(
key=SESSION_CACHE_KEY_TEMPLATE,
ttl=f"{settings.CACHE.DEFAULT_LOCK_TTL_SECONDS}s",
prefix=SESSION_LOCK_PREFIX,
)
async def _fetch_session(
db: AsyncSession,
workspace_name: str,
session_name: str,
) -> models.Session | None:
return await db.scalar(
select(models.Session)
.where(models.Session.workspace_name == workspace_name)
.where(models.Session.name == session_name)
)
def count_observers_in_config(
peer_configs: dict[str, schemas.SessionPeerConfig],
@ -79,15 +117,14 @@ async def get_or_create_session(
ConflictException: If we fail to get or create the session
"""
stmt = (
select(models.Session)
.where(models.Session.workspace_name == workspace_name)
.where(models.Session.name == session.name)
)
if not session.name:
raise ValueError("Session name must be provided")
result = await db.execute(stmt)
honcho_session = await _fetch_session(db, workspace_name, session.name)
honcho_session = result.scalar_one_or_none()
# Merge cached object into session if it exists (cached objects are detached)
if honcho_session is not None:
honcho_session = await db.merge(honcho_session, load=False)
# Check if session already exists
if honcho_session is None:
@ -148,6 +185,12 @@ async def get_or_create_session(
)
await db.commit()
await db.refresh(honcho_session)
cache_key = session_cache_key(workspace_name, session.name)
await cache.set(
cache_key, honcho_session, expire=settings.CACHE.DEFAULT_TTL_SECONDS
)
return honcho_session
@ -170,22 +213,17 @@ async def get_session(
Raises:
ResourceNotFoundException: If the session does not exist
"""
stmt = (
select(models.Session)
.where(models.Session.workspace_name == workspace_name)
.where(models.Session.name == session_name)
)
session = await _fetch_session(db, workspace_name, session_name)
result = await db.execute(stmt)
honcho_session = result.scalar_one_or_none()
if honcho_session is None:
if session is None:
raise ResourceNotFoundException(
f"Session {session_name} not found in workspace {workspace_name}"
)
return honcho_session
# Merge cached object into session (cached objects are detached)
session = await db.merge(session, load=False)
return session
async def update_session(
@ -220,6 +258,12 @@ async def update_session(
honcho_session.configuration = session.configuration
await db.commit()
await db.refresh(honcho_session)
# Invalidate cache - read-through pattern
cache_key = session_cache_key(workspace_name, session_name)
await cache.delete(cache_key)
logger.debug("Session %s updated successfully", session_name)
return honcho_session
@ -241,22 +285,16 @@ async def delete_session(
Raises:
ResourceNotFoundException: If the session does not exist
"""
stmt = (
select(models.Session)
.where(models.Session.workspace_name == workspace_name)
.where(models.Session.name == session_name)
)
result = await db.execute(stmt)
honcho_session = result.scalar_one_or_none()
if honcho_session is None:
logger.warning(
f"Session {session_name} not found in workspace {workspace_name}"
)
raise ResourceNotFoundException("Session not found")
honcho_session = await get_session(db, session_name, workspace_name)
honcho_session.is_active = False
await db.commit()
await db.refresh(honcho_session)
# Invalidate cache - read-through pattern
cache_key = session_cache_key(workspace_name, session_name)
await cache.delete(cache_key)
logger.debug("Session %s marked as inactive", session_name)
return True
@ -359,7 +397,10 @@ async def clone_session(
db.add(new_session_peer)
await db.commit()
await db.refresh(new_session)
logger.debug("Session %s cloned successfully", original_session_name)
# Cache will be populated on next read - read-through pattern
return new_session
@ -385,18 +426,7 @@ async def remove_peers_from_session(
ResourceNotFoundException: If the session does not exist
"""
# Verify session exists
stmt = (
select(models.Session)
.where(models.Session.workspace_name == workspace_name)
.where(models.Session.name == session_name)
)
result = await db.execute(stmt)
session = result.scalar_one_or_none()
if session is None:
raise ResourceNotFoundException(
f"Session {session_name} not found in workspace {workspace_name}"
)
await get_session(db, session_name, workspace_name)
# Soft delete specified session peers by setting left_at timestamp
update_stmt = (
@ -409,7 +439,7 @@ async def remove_peers_from_session(
)
.values(left_at=func.now())
)
result = await db.execute(update_stmt)
await db.execute(update_stmt)
await db.commit()
return True

5
src/crud/webhook.py
View File

@ -63,7 +63,7 @@ async def get_or_create_webhook_endpoint(
async def list_webhook_endpoints(
db: AsyncSession, workspace_name: str
workspace_name: str,
) -> Select[tuple[models.WebhookEndpoint]]:
"""
List all webhook endpoints, optionally filtered by workspace.
@ -75,9 +75,6 @@ async def list_webhook_endpoints(
Returns:
List of webhook endpoints
"""
# Verify workspace exists
await get_workspace(db, workspace_name)
return select(models.WebhookEndpoint).where(
models.WebhookEndpoint.workspace_name == workspace_name
)

87
src/crud/workspace.py
View File

@ -1,16 +1,51 @@
from logging import getLogger
from typing import Any
from cashews import NOT_NONE
from sqlalchemy import Select, delete, func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from src import models, schemas
from src.cache.client import cache, get_cache_namespace
from src.config import settings
from src.exceptions import ConflictException, ResourceNotFoundException
from src.utils.filter import apply_filter
logger = getLogger(__name__)
WORKSPACE_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}"
WORKSPACE_LOCK_PREFIX = f"{get_cache_namespace()}:lock"
def workspace_cache_key(workspace_name: str) -> str:
"""Generate cache key for workspace."""
return (
get_cache_namespace()
+ ":"
+ WORKSPACE_CACHE_KEY_TEMPLATE.format(workspace_name=workspace_name)
)
@cache(
key=WORKSPACE_CACHE_KEY_TEMPLATE,
ttl=f"{settings.CACHE.DEFAULT_TTL_SECONDS}s",
prefix=get_cache_namespace(),
condition=NOT_NONE,
)
@cache.locked(
key=WORKSPACE_CACHE_KEY_TEMPLATE,
ttl=f"{settings.CACHE.DEFAULT_LOCK_TTL_SECONDS}s",
prefix=WORKSPACE_LOCK_PREFIX,
)
async def _fetch_workspace(
db: AsyncSession, workspace_name: str
) -> models.Workspace | None:
"""Fetch a workspace from the database."""
return await db.scalar(
select(models.Workspace).where(models.Workspace.name == workspace_name)
)
async def get_or_create_workspace(
db: AsyncSession,
@ -31,14 +66,17 @@ async def get_or_create_workspace(
Raises:
ConflictException: If we fail to get or create the workspace
"""
# Try to get the existing workspace
stmt = select(models.Workspace).where(models.Workspace.name == workspace.name)
result = await db.execute(stmt)
existing_workspace = result.scalar_one_or_none()
if not workspace.name:
raise ValueError("Workspace name must be provided")
# Check if workspace already exists
existing_workspace = await _fetch_workspace(db, workspace.name)
if existing_workspace is not None:
# Workspace already exists
logger.debug("Found existing workspace: %s", workspace.name)
# Merge cached object into session (cached objects are detached)
existing_workspace = await db.merge(existing_workspace, load=False)
return existing_workspace
# Workspace doesn't exist, create a new one
@ -50,7 +88,14 @@ async def get_or_create_workspace(
try:
db.add(honcho_workspace)
await db.commit()
await db.refresh(honcho_workspace)
logger.debug("Workspace created successfully: %s", workspace.name)
cache_key = workspace_cache_key(workspace.name)
await cache.set(
cache_key, honcho_workspace, expire=settings.CACHE.DEFAULT_TTL_SECONDS
)
return honcho_workspace
except IntegrityError:
await db.rollback()
@ -94,15 +139,15 @@ async def get_workspace(
Raises:
ResourceNotFoundException: If the workspace does not exist
"""
# Try to get the existing peer
stmt = select(models.Workspace).where(models.Workspace.name == workspace_name)
result = await db.execute(stmt)
existing_workspace = result.scalar_one_or_none()
existing_workspace = await _fetch_workspace(db, workspace_name)
if existing_workspace is not None:
return existing_workspace
if existing_workspace is None:
raise ResourceNotFoundException(f"Workspace {workspace_name} not found")
raise ResourceNotFoundException(f"Workspace {workspace_name} not found")
# Merge cached object into session (cached objects are detached)
existing_workspace = await db.merge(existing_workspace, load=False)
return existing_workspace
async def update_workspace(
@ -134,6 +179,12 @@ async def update_workspace(
honcho_workspace.configuration = workspace.configuration
await db.commit()
await db.refresh(honcho_workspace)
# Invalidate cache
cache_key = workspace_cache_key(workspace_name)
await cache.delete(cache_key)
logger.debug("Workspace with id %s updated successfully", honcho_workspace.id)
return honcho_workspace
@ -241,10 +292,18 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor
)
await db.delete(honcho_workspace)
await db.commit()
cache_key = workspace_cache_key(workspace_name)
workspace_pattern = f"{cache_key}*"
await cache.delete_match(workspace_pattern)
logger.debug("Workspace %s deleted", workspace_name)
except Exception as e:
logger.error("Failed to delete workspace %s: %s", workspace_name, e)
except Exception:
logger.exception(
"Failed to delete workspace %s",
workspace_name,
)
await db.rollback()
raise e
raise
return workspace_snapshot

2
src/db.py
View File

@ -46,6 +46,8 @@ SessionLocal = async_sessionmaker(
)
table_schema = settings.DB.SCHEMA
# Note: column_0_N_name expands to include all columns in multi-column constraints
# e.g., "workspace_id_tenant_id" for a composite constraint on both columns
meta = MetaData()
meta.schema = table_schema
Base = declarative_base(metadata=meta)

55
src/deriver/consumer.py
View File

@ -1,5 +1,4 @@
import logging
from typing import Any
import sentry_sdk
from pydantic import ValidationError
@ -26,8 +25,12 @@ logging.getLogger("sqlalchemy.engine.Engine").disabled = True
console = Console(markup=True)
async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None:
async def process_item(queue_item: models.QueueItem) -> None:
"""Process a single item from the queue."""
task_type = queue_item.task_type
queue_payload = queue_item.payload
workspace_name = queue_item.workspace_name
if task_type == "webhook":
try:
validated = WebhookPayload(**queue_payload)
@ -39,7 +42,7 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None:
)
raise ValueError(f"Invalid payload structure: {str(e)}") from e
async with tracked_db() as db:
await webhook_delivery.deliver_webhook(db, validated)
await webhook_delivery.deliver_webhook(db, validated, workspace_name)
elif task_type == "summary":
try:
@ -52,17 +55,21 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None:
)
raise ValueError(f"Invalid payload structure: {str(e)}") from e
# Use workspace_name and message_id from QueueItem columns
message_id = queue_item.message_id
if message_id is None:
raise ValueError("Summary tasks require a message_id")
message_public_id = validated.message_public_id
if not message_public_id:
logger.debug(
"Fetching message public ID for message %s", validated.message_id
)
logger.debug("Fetching message public ID for message %s", message_id)
async with tracked_db(operation_name="summary_fallback") as db:
stmt = (
select(models.Message)
.where(models.Message.workspace_name == validated.workspace_name)
.where(models.Message.workspace_name == workspace_name)
.where(models.Message.session_name == validated.session_name)
.where(models.Message.id == validated.message_id)
.where(models.Message.id == message_id)
)
result = await db.execute(stmt)
@ -70,22 +77,20 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None:
if message is None:
logger.error(
"Failed to fetch message with ID %s for process_summary_task",
validated.message_id,
message_id,
)
return
message_public_id = message.public_id
with sentry_sdk.start_transaction(name="process_summary_task", op="deriver"):
await summarizer.summarize_if_needed(
validated.workspace_name,
workspace_name,
validated.session_name,
validated.message_id,
message_id,
validated.message_seq_in_session,
message_public_id,
)
log_performance_metrics(
"summary", f"{validated.workspace_name}_{validated.message_id}"
)
log_performance_metrics("summary", f"{workspace_name}_{message_id}")
elif task_type == "dream":
with sentry_sdk.start_transaction(name="process_dream_task", op="deriver"):
@ -98,7 +103,7 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None:
queue_payload,
)
raise ValueError(f"Invalid payload structure: {str(e)}") from e
await process_dream(validated)
await process_dream(validated, workspace_name)
else:
raise ValueError(f"Invalid task type: {task_type}")
@ -109,29 +114,21 @@ async def process_representation_batch(
observer: str | None,
observed: str | None,
) -> None:
"""Validate incoming queue payloads and dispatch to the appropriate handler.
This function centralizes payload validation using a simple mapping from
task type to Pydantic model. After validation, routes the request to
the correct processor without repeating type checks elsewhere.
"""Prepares and processes a batch of messages for representation tasks.
Args:
task_type: The type of task to process
queue_payloads: List of payload dictionaries to process
observed (optional): For representation tasks, the observed from work_unit_key
to identify which messages should be focused on
observer (optional): For representation tasks, the observer from work_unit_key
to identify which messages should be focused on
messages: List of messages to process
observer: The observer of the messages
observed: The observed of the messages
"""
if not messages or not messages[0]:
logger.debug("process_representation_batch received no payloads")
logger.debug("process_representation_batch received no messages")
return
if observed is None or observer is None:
raise ValueError("observed and observer are required for representation tasks")
logger.debug(
"process_representation_batch received %s payloads",
"process_representation_batch received %s messages",
len(messages),
)

33
src/deriver/enqueue.py
View File

@ -11,7 +11,7 @@ from src.dreamer.dream_scheduler import get_affected_dream_keys, get_dream_sched
from src.exceptions import ValidationException
from src.models import QueueItem
from src.utils.queue_payload import create_payload
from src.utils.work_unit import get_work_unit_key
from src.utils.work_unit import construct_work_unit_key
logger = logging.getLogger(__name__)
@ -159,8 +159,16 @@ def create_representation_record(
session_id: Optional session ID
Returns:
Queue record dictionary
Queue record dictionary with workspace_name and message_id as separate fields
"""
workspace_name = message.get("workspace_name")
message_id = message.get("message_id")
if not isinstance(workspace_name, str):
raise TypeError("workspace_name is required and must be a string")
if not isinstance(message_id, int):
raise TypeError("message_id is required and must be an integer")
processed_payload = create_payload(
message=message,
task_type="representation",
@ -168,10 +176,12 @@ def create_representation_record(
observed=observed,
)
return {
"work_unit_key": get_work_unit_key(processed_payload),
"work_unit_key": construct_work_unit_key(workspace_name, processed_payload),
"payload": processed_payload,
"session_id": session_id,
"task_type": "representation",
"workspace_name": workspace_name,
"message_id": message_id,
}
@ -185,23 +195,32 @@ def create_summary_record(
Args:
message: The message payload
observed: Name of the sender
observer: Name of the target
session_id: Session ID
message_seq_in_session: The sequence number of the message in the session
Returns:
Queue record dictionary
Queue record dictionary with workspace_name and message_id as separate fields
"""
workspace_name = message.get("workspace_name")
message_id = message.get("message_id")
if not isinstance(workspace_name, str):
raise ValueError("workspace_name is required and must be a string")
if not isinstance(message_id, int):
raise ValueError("message_id is required and must be an integer")
processed_payload = create_payload(
message=message,
task_type="summary",
message_seq_in_session=message_seq_in_session,
)
return {
"work_unit_key": get_work_unit_key(processed_payload),
"work_unit_key": construct_work_unit_key(workspace_name, processed_payload),
"payload": processed_payload,
"session_id": session_id,
"task_type": "summary",
"workspace_name": workspace_name,
"message_id": message_id,
}

6
src/deriver/prompts.py
View File

@ -65,11 +65,9 @@ Current understanding of {peer_id}:
You are an agent who critically analyzes messages from {peer_id} through rigorous logical reasoning to produce only conclusions about them that are CERTAIN.
TARGET USER TO ANALYZE
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
You are analyzing: {peer_id}
The conversation may include messages from multiple participants, but you MUST focus ONLY on deriving conclusions about {peer_id}. Only use other participants' messages as context for understanding {peer_id}.
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
IMPORTANT NAMING RULES
• When you write a conclusion about {peer_id}, always start the sentence with their name (e.g. "Anthony is 25 years old").
@ -87,7 +85,7 @@ Here are strict definitions for the reasoning modes you are to employ:
- Current date and time (which is: {message_created_at})
- Timestamps from conversation history
- Follow strict literal necessity--if stated directly in message, extract a conclusion
- Latest message MUST be a premise, previous messages and timestamps may be used to contextualize
- New turn MUST be a premise, previous messages and timestamps may be used to contextualize
- Transforms a single message (premise) into ONE OR MULTIPLE conclusions
- Derive EVERYTHING that can be explicitly concluded
- Make sure EVERY conclusion is sufficiently contextualized, i.e. ensure each conclusion contains enough specific information about subjects and objects to make it self-contained and useful (e.g. instead of "Ann is nervous about the interview", use "Ann is nervous about the job interview at the pharmacy")
@ -128,7 +126,7 @@ Based on our definitions and examples, here's a summary of the logical reasoning
- Explicit/Deductive/Temporal/General → Further Deductive: Can use certain conclusions and known facts to deduce additional certain conclusions
- Probabilistic Deductive ↛ Further Deductive: If a deductive conclusion includes probabilistic qualifiers (likely, potentially, typically, might, etc) it may NOT be used as a premise for further deductions
**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to the latest message, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions.
**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to all new turns, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions. You must extract observations from all new turns.
{peer_card_section}

29
src/deriver/queue_manager.py
View File

@ -10,12 +10,13 @@ import sentry_sdk
from dotenv import load_dotenv
from nanoid import generate as generate_nanoid
from sentry_sdk.integrations.asyncio import AsyncioIntegration
from sqlalchemy import BigInteger, and_, delete, select, update
from sqlalchemy import and_, delete, select, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql import func
from src import models, prometheus
from src.cache.client import close_cache, init_cache
from src.config import settings
from src.dependencies import tracked_db
from src.deriver.consumer import (
@ -463,9 +464,7 @@ class QueueManager:
break
try:
await process_item(
work_unit.task_type, queue_item.payload
)
await process_item(queue_item)
await self.mark_queue_items_as_processed(
[queue_item], work_unit_key
)
@ -614,8 +613,7 @@ class QueueManager:
.select_from(models.QueueItem)
.join(
models.Message,
func.cast(models.QueueItem.payload["message_id"].astext, BigInteger)
== models.Message.id,
models.QueueItem.message_id == models.Message.id,
)
.where(~models.QueueItem.processed)
.where(models.Message.session_name == parsed_key.session_name)
@ -661,10 +659,7 @@ class QueueManager:
and_(
models.QueueItem.work_unit_key == work_unit_key,
~models.QueueItem.processed,
func.cast(
models.QueueItem.payload["message_id"].astext, BigInteger
)
== models.Message.id,
models.QueueItem.message_id == models.Message.id,
),
)
.where(allowed_condition)
@ -689,7 +684,11 @@ class QueueManager:
if items_to_process:
max_queue_item_message_id = max(
[qi.payload["message_id"] for qi in items_to_process]
[
qi.message_id
for qi in items_to_process
if qi.message_id is not None
]
)
messages_context = [ # remove any messages that are after the last message_id from queue items
m for m in messages_context if m.id <= max_queue_item_message_id
@ -766,6 +765,13 @@ class QueueManager:
async def main():
logger.debug("Starting queue manager")
try:
await init_cache()
except Exception as e:
logger.warning(
"Error initializing cache in queue manager; proceeding without cache: %s", e
)
manager = QueueManager()
try:
await manager.initialize()
@ -773,4 +779,5 @@ async def main():
logger.error(f"Error in main: {str(e)}")
sentry_sdk.capture_exception(e)
finally:
await close_cache()
logger.debug("Main function exiting")

17
src/dreamer/dream_scheduler.py
View File

@ -12,7 +12,7 @@ from src import models
from src.config import settings
from src.dependencies import tracked_db
from src.utils.queue_payload import create_dream_payload
from src.utils.work_unit import get_work_unit_key, parse_work_unit_key
from src.utils.work_unit import construct_work_unit_key, parse_work_unit_key
logger = getLogger(__name__)
@ -48,13 +48,13 @@ def get_affected_dream_keys(message: dict[str, Any]) -> list[str]:
return []
# Generate dream work unit key for this peer's collection
dream_key = get_work_unit_key(
dream_key = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"workspace_name": workspace_name,
"observer": peer_name,
"observed": peer_name,
}
},
)
return [dream_key]
@ -193,7 +193,6 @@ class DreamScheduler:
) -> None:
"""Execute the dream by enqueueing it and updating collection metadata."""
dream_payload = create_dream_payload(
workspace_name=workspace_name,
dream_type="consolidate",
observer=observer,
observed=observed,
@ -205,6 +204,8 @@ class DreamScheduler:
"payload": dream_payload,
"session_id": None,
"task_type": "dream",
"workspace_name": workspace_name,
"message_id": None, # Dreams don't have a message_id
}
await db.execute(insert(models.QueueItem), [dream_record])
@ -325,13 +326,13 @@ async def check_and_schedule_dream(
dream_scheduler = get_dream_scheduler()
if dream_scheduler:
collection_work_unit_key = get_work_unit_key(
collection_work_unit_key = construct_work_unit_key(
collection.workspace_name,
{
"task_type": "dream",
"workspace_name": collection.workspace_name,
"observer": collection.observer,
"observed": collection.observed,
}
},
)
await dream_scheduler.schedule_dream(

23
src/dreamer/dreamer.py
View File

@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
@sentry_sdk.trace
async def process_dream(
payload: DreamPayload,
workspace_name: str,
) -> None:
"""
Process a dream task by performing collection maintenance operations.
@ -32,12 +33,12 @@ async def process_dream(
payload: The dream task payload containing workspace, peer, and dream type information
"""
logger.info(
f"Processing dream task: {payload.dream_type} for {payload.workspace_name}/{payload.observer}/{payload.observed}"
f"Processing dream task: {payload.dream_type} for {workspace_name}/{payload.observer}/{payload.observed}"
)
try:
if payload.dream_type == "consolidate":
await _process_consolidate_dream(payload)
await _process_consolidate_dream(payload, workspace_name)
## TODO other dream types
except Exception as e:
@ -50,7 +51,9 @@ async def process_dream(
# Don't re-raise - we want to mark the dream task as processed even if it fails
async def _process_consolidate_dream(payload: DreamPayload) -> None:
async def _process_consolidate_dream(
payload: DreamPayload, workspace_name: str
) -> None:
"""
Process a consolidation dream task.
@ -63,7 +66,7 @@ async def _process_consolidate_dream(payload: DreamPayload) -> None:
logger.info(
f"""
(っ- ‸ - ς)ᶻ z 𐰁 ᶻ z 𐰁 ᶻ z 𐰁\n
DREAM: consolidating documents for {payload.workspace_name}/{payload.observer}/{payload.observed}\n
DREAM: consolidating documents for {workspace_name}/{payload.observer}/{payload.observed}\n
𐰁 z ᶻ 𐰁 z ᶻ 𐰁 z ᶻ(っ- ‸ - ς)"""
)
@ -71,7 +74,7 @@ DREAM: consolidating documents for {payload.workspace_name}/{payload.observer}/{
async with tracked_db("dream_consolidate") as db:
documents = await crud.get_all_documents(
db,
payload.workspace_name,
workspace_name,
observer=payload.observer,
observed=payload.observed,
)
@ -88,7 +91,7 @@ DREAM: consolidating documents for {payload.workspace_name}/{payload.observer}/{
for cluster in clusters:
await _consolidate_cluster(
cluster,
payload.workspace_name,
workspace_name,
db,
observer=payload.observer,
observed=payload.observed,
@ -118,9 +121,7 @@ async def _consolidate_cluster(
logger.info("consolidated representation:\n%s", consolidated_representation)
# TODO: less hacky preservation of times_derived
total_times_derived = sum(
doc.internal_metadata.get("times_derived", 1) for doc in cluster
)
total_times_derived = sum(doc.times_derived for doc in cluster)
new_documents = [
*consolidated_representation.explicit,
@ -141,10 +142,8 @@ async def _consolidate_cluster(
# NOTE: other kinds of observations here in the future
metadata = schemas.DocumentMetadata(
times_derived=total_times_derived,
message_ids=obs.message_ids,
message_created_at=format_datetime_utc(obs.created_at),
level=level,
premises=premises,
)
@ -154,6 +153,8 @@ async def _consolidate_cluster(
schemas.DocumentCreate(
content=content,
session_name=obs.session_name,
level=level,
times_derived=total_times_derived,
metadata=metadata,
embedding=embedding,
)

17
src/main.py
View File

@ -16,6 +16,7 @@ from sentry_sdk.integrations.fastapi import FastApiIntegration
from sentry_sdk.integrations.starlette import StarletteIntegration
from src import prometheus
from src.cache.client import close_cache, init_cache
from src.config import settings
from src.db import engine, request_context
from src.exceptions import HonchoException
@ -109,8 +110,18 @@ if SENTRY_ENABLED:
@asynccontextmanager
async def lifespan(_: FastAPI):
yield
await engine.dispose()
try:
await init_cache()
except Exception as e:
logger.warning(
"Error initializing cache in api process; proceeding without cache: %s", e
)
try:
yield
finally:
await close_cache()
await engine.dispose()
app = FastAPI(
@ -123,7 +134,7 @@ app = FastAPI(
title="Honcho API",
summary="The Identity Layer for the Agentic World",
description="""Honcho is a platform for giving agents user-centric memory and social cognition""",
version="2.4.1",
version="2.4.2",
contact={
"name": "Plastic Labs",
"url": "https://honcho.dev",

208
src/models.py
View File

@ -1,6 +1,6 @@
import datetime
from logging import getLogger
from typing import Any, Literal, final
from typing import Any, final
from dotenv import load_dotenv
from nanoid import generate as generate_nanoid
@ -26,6 +26,8 @@ from sqlalchemy.orm.properties import MappedColumn
from sqlalchemy.sql import func
from typing_extensions import override
from src.utils.types import DocumentLevel, TaskType
from .db import Base
load_dotenv(override=True)
@ -51,13 +53,25 @@ session_peers_table = Table(
nullable=False,
),
Column("peer_name", TEXT, primary_key=True, nullable=False),
Column("configuration", JSONB, default=dict),
Column("internal_metadata", JSONB, default=dict),
Column(
"configuration",
JSONB,
default=dict,
nullable=False,
server_default=text("'{}'::jsonb"),
),
Column(
"internal_metadata",
JSONB,
default=dict,
nullable=False,
server_default=text("'{}'::jsonb"),
),
Column(
"joined_at",
DateTime(timezone=True),
nullable=False,
default=func.now(),
server_default=func.now(),
),
Column(
"left_at",
@ -81,22 +95,28 @@ session_peers_table = Table(
class Workspace(Base):
__tablename__: str = "workspaces"
id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True)
name: Mapped[str] = mapped_column(TEXT, index=True, unique=True)
name: Mapped[str] = mapped_column(TEXT, unique=True)
peers = relationship("Peer", back_populates="workspace")
webhook_endpoints = relationship("WebhookEndpoint", back_populates="workspace")
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
h_metadata: Mapped[dict[str, Any]] = mapped_column(
"metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict)
internal_metadata: Mapped[dict[str, Any]] = mapped_column(
"internal_metadata", JSONB, default=dict
)
configuration: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
configuration: Mapped[dict[str, Any]] = mapped_column(
JSONB, default=dict, server_default=text("'{}'::jsonb")
)
__table_args__ = (
CheckConstraint("length(id) = 21", name="id_length"),
CheckConstraint("length(name) <= 512", name="name_length"),
CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"),
Index("ix_workspaces_created_at", "created_at"),
Index("ix_workspaces_name", "name"),
)
@ -104,18 +124,22 @@ class Workspace(Base):
class Peer(Base):
__tablename__: str = "peers"
id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True)
name: Mapped[str] = mapped_column(TEXT, index=True)
h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict)
name: Mapped[str] = mapped_column(TEXT, nullable=False)
h_metadata: Mapped[dict[str, Any]] = mapped_column(
"metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
internal_metadata: Mapped[dict[str, Any]] = mapped_column(
"internal_metadata", JSONB, default=dict
"internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), index=True, nullable=False
ForeignKey("workspaces.name"), nullable=False
)
configuration: Mapped[dict[str, Any]] = mapped_column(
JSONB, default=dict, server_default=text("'{}'::jsonb")
)
configuration: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
workspace = relationship("Workspace", back_populates="peers")
sessions = relationship(
@ -128,6 +152,9 @@ class Peer(Base):
CheckConstraint("length(name) <= 512", name="name_length"),
CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"),
Index("idx_peers_workspace_lookup", "workspace_name", "name"),
Index("ix_peers_created_at", "created_at"),
Index("ix_peers_name", "name"),
Index("ix_peers_workspace_name", "workspace_name"),
)
def __repr__(self) -> str:
@ -138,20 +165,24 @@ class Peer(Base):
class Session(Base):
__tablename__: str = "sessions"
id: Mapped[str] = mapped_column(TEXT, primary_key=True, default=generate_nanoid)
name: Mapped[str] = mapped_column(TEXT, index=True)
is_active: Mapped[bool] = mapped_column(default=True)
h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict)
name: Mapped[str] = mapped_column(TEXT)
is_active: Mapped[bool] = mapped_column(default=True, server_default=text("true"))
h_metadata: Mapped[dict[str, Any]] = mapped_column(
"metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
internal_metadata: Mapped[dict[str, Any]] = mapped_column(
"internal_metadata", JSONB, default=dict
"internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
messages = relationship("Message", back_populates="session")
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), index=True, nullable=False
ForeignKey("workspaces.name"), nullable=False
)
configuration: Mapped[dict[str, Any]] = mapped_column(
JSONB, default=dict, server_default=text("'{}'::jsonb")
)
configuration: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
peers = relationship(
"Peer", secondary=session_peers_table, back_populates="sessions"
@ -162,6 +193,7 @@ class Session(Base):
CheckConstraint("length(name) <= 512", name="name_length"),
CheckConstraint("length(id) = 21", name="id_length"),
CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"),
Index("ix_sessions_created_at", "created_at"),
)
def __repr__(self) -> str:
@ -175,26 +207,30 @@ class Message(Base):
BigInteger, Identity(), primary_key=True, autoincrement=True
)
public_id: Mapped[str] = mapped_column(
TEXT, index=True, unique=True, default=generate_nanoid
TEXT,
unique=True,
default=generate_nanoid,
)
# NOTE: Messages in Honcho 2.0 could historically be stored outside of a session.
# We have since assigned all of these messages to a default session.
session_name: Mapped[str] = mapped_column(index=True, nullable=False)
session_name: Mapped[str] = mapped_column(TEXT, nullable=False)
content: Mapped[str] = mapped_column(TEXT)
h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict)
h_metadata: Mapped[dict[str, Any]] = mapped_column(
"metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
internal_metadata: Mapped[dict[str, Any]] = mapped_column(
"internal_metadata", JSONB, default=dict
"internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
token_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
seq_in_session: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
seq_in_session: Mapped[int] = mapped_column(BigInteger, nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
session = relationship("Session", back_populates="messages")
peer_name: Mapped[str] = mapped_column(index=True)
peer_name: Mapped[str] = mapped_column(TEXT)
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), index=True
ForeignKey("workspaces.name"),
)
__table_args__ = (
@ -229,6 +265,11 @@ class Message(Base):
text("to_tsvector('english', content)"),
postgresql_using="gin",
),
Index("ix_messages_created_at", "created_at"),
Index("ix_messages_id", "id"),
Index("ix_messages_peer_name", "peer_name"),
Index("ix_messages_public_id", "public_id"),
Index("ix_messages_workspace_name", "workspace_name"),
)
@override
@ -246,15 +287,15 @@ class MessageEmbedding(Base):
content: Mapped[str] = mapped_column(TEXT)
embedding: MappedColumn[Any] = mapped_column(Vector(1536))
message_id: Mapped[str] = mapped_column(
ForeignKey("messages.public_id"), index=True
ForeignKey("messages.public_id"),
)
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), index=True
ForeignKey("workspaces.name"),
)
session_name: Mapped[str] = mapped_column(TEXT, index=True, nullable=False)
peer_name: Mapped[str | None] = mapped_column(TEXT, index=True)
session_name: Mapped[str] = mapped_column(TEXT, nullable=False)
peer_name: Mapped[str] = mapped_column(TEXT)
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
# Relationship to Message
@ -278,6 +319,11 @@ class MessageEmbedding(Base):
postgresql_with={"m": 16, "ef_construction": 64},
postgresql_ops={"embedding": "vector_cosine_ops"},
),
Index("idx_message_embeddings_created_at", "created_at"),
Index("idx_message_embeddings_message_id", "message_id"),
Index("idx_message_embeddings_peer_name", "peer_name"),
Index("idx_message_embeddings_session_name", "session_name"),
Index("idx_message_embeddings_workspace_name", "workspace_name"),
)
@ -286,20 +332,22 @@ class Collection(Base):
__tablename__: str = "collections"
id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True)
observer: Mapped[str] = mapped_column(TEXT, index=True)
observed: Mapped[str] = mapped_column(TEXT, index=True)
observer: Mapped[str] = mapped_column(TEXT)
observed: Mapped[str] = mapped_column(TEXT)
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
h_metadata: Mapped[dict[str, Any]] = mapped_column(
"metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict)
internal_metadata: Mapped[dict[str, Any]] = mapped_column(
"internal_metadata", JSONB, default=dict
"internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
documents = relationship(
"Document", back_populates="collection", cascade="all, delete, delete-orphan"
)
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), index=True
ForeignKey("workspaces.name"),
)
__table_args__ = (
@ -321,6 +369,10 @@ class Collection(Base):
["observed", "workspace_name"],
["peers.name", "peers.workspace_name"],
),
Index("idx_collections_observer", "observer"),
Index("idx_collections_observed", "observed"),
Index("ix_collections_created_at", "created_at"),
Index("ix_collections_workspace_name", "workspace_name"),
)
@ -329,26 +381,31 @@ class Document(Base):
__tablename__: str = "documents"
id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True)
internal_metadata: Mapped[dict[str, Any]] = mapped_column(
"internal_metadata", JSONB, default=dict
"internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb")
)
content: Mapped[str] = mapped_column(TEXT)
level: Mapped[DocumentLevel] = mapped_column(
TEXT, nullable=False, server_default="explicit"
)
times_derived: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("1")
)
embedding: MappedColumn[Any] = mapped_column(Vector(1536))
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
observer: Mapped[str] = mapped_column(TEXT, index=True)
observed: Mapped[str] = mapped_column(TEXT, index=True)
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), index=True
)
session_name: Mapped[str] = mapped_column(TEXT, index=True)
observer: Mapped[str] = mapped_column(TEXT)
observed: Mapped[str] = mapped_column(TEXT)
workspace_name: Mapped[str] = mapped_column(ForeignKey("workspaces.name"))
session_name: Mapped[str] = mapped_column(TEXT)
collection = relationship("Collection", back_populates="documents")
__table_args__ = (
CheckConstraint("length(id) = 21", name="id_length"),
CheckConstraint("length(content) <= 65535", name="content_length"),
CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"),
CheckConstraint("level IN ('explicit', 'deductive')", name="level_valid"),
# Composite foreign key constraint for collections
ForeignKeyConstraint(
["observer", "observed", "workspace_name"],
@ -383,33 +440,62 @@ class Document(Base):
"embedding": "vector_cosine_ops"
}, # Cosine distance operator
),
Index("idx_documents_observer", "observer"),
Index("idx_documents_observed", "observed"),
Index("idx_documents_session_name", "session_name"),
Index("ix_documents_created_at", "created_at"),
Index("ix_documents_workspace_name", "workspace_name"),
)
TaskType = Literal["webhook", "summary", "representation", "dream"]
@final
class QueueItem(Base):
__tablename__: str = "queue"
id: Mapped[int] = mapped_column(
BigInteger, Identity(), primary_key=True, autoincrement=True
)
session_id: Mapped[str] = mapped_column(
ForeignKey("sessions.id"), index=True, nullable=True
)
session_id: Mapped[str] = mapped_column(ForeignKey("sessions.id"), nullable=True)
work_unit_key: Mapped[str] = mapped_column(TEXT, nullable=False)
task_type: Mapped[TaskType] = mapped_column(TEXT, nullable=False)
payload: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
processed: Mapped[bool] = mapped_column(Boolean, default=False)
processed: Mapped[bool] = mapped_column(
Boolean, default=False, server_default=text("false")
)
error: Mapped[str | None] = mapped_column(TEXT, nullable=True)
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), index=True, default=func.now()
DateTime(timezone=True), server_default=func.now()
)
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), nullable=False
)
message_id: Mapped[int | None] = mapped_column(
BigInteger, ForeignKey("messages.id"), nullable=True
)
__table_args__ = (
Index("ix_queue_created_at", "created_at"),
Index("ix_queue_session_id", "session_id"),
Index(
"ix_queue_workspace_name",
"workspace_name",
),
Index(
"ix_queue_message_id_not_null",
"message_id",
postgresql_where=text("message_id IS NOT NULL"),
),
Index("ix_queue_workspace_name_processed", "workspace_name", "processed"),
Index(
"ix_queue_work_unit_key_processed_id",
"work_unit_key",
"processed",
"id",
),
)
def __repr__(self) -> str:
return f"QueueItem(id={self.id}, session_id={self.session_id}, work_unit_key={self.work_unit_key}, task_type={self.task_type}, payload={self.payload}, processed={self.processed})"
return f"QueueItem(id={self.id}, session_id={self.session_id}, work_unit_key={self.work_unit_key}, task_type={self.task_type}, payload={self.payload}, processed={self.processed}, workspace_name={self.workspace_name}, message_id={self.message_id})"
@final
@ -418,10 +504,10 @@ class ActiveQueueSession(Base):
id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True)
work_unit_key: Mapped[str] = mapped_column(TEXT, unique=True, index=True)
work_unit_key: Mapped[str] = mapped_column(TEXT, unique=True)
last_updated: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), default=func.now(), onupdate=func.now()
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
@ -430,11 +516,11 @@ class WebhookEndpoint(Base):
__tablename__: str = "webhook_endpoints"
id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True)
workspace_name: Mapped[str] = mapped_column(
ForeignKey("workspaces.name"), index=True, nullable=False
ForeignKey("workspaces.name"), nullable=False
)
url: Mapped[str] = mapped_column(TEXT, nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(
DateTime(timezone=True), default=func.now()
DateTime(timezone=True), server_default=func.now()
)
workspace = relationship("Workspace", back_populates="webhook_endpoints")

4
src/prometheus.py
View File

@ -5,6 +5,7 @@ This module defines all Prometheus metrics for all Honcho processes and exposes
"""
import logging
from typing import cast
from prometheus_client import (
CONTENT_TYPE_LATEST,
@ -28,7 +29,8 @@ class NamespacedCounter(Counter):
def labels(self, **kwargs: str) -> "NamespacedCounter":
"""Override labels to automatically appends namespace label"""
kwargs["namespace"] = settings.METRICS.NAMESPACE
# METRICS.NAMESPACE is guaranteed to be non-None by AppSettings.propagate_namespace validator
kwargs["namespace"] = cast(str, settings.METRICS.NAMESPACE)
return super().labels(**kwargs)

4
src/routers/messages.py
View File

@ -71,7 +71,7 @@ async def create_messages_for_session(
"peer_name": message.peer_name,
"created_at": message.created_at,
"message_public_id": message.public_id,
"message_seq_in_session": message.seq_in_session,
"seq_in_session": message.seq_in_session,
}
for message in created_messages
]
@ -135,7 +135,7 @@ async def create_messages_with_file(
"peer_name": message.peer_name,
"created_at": message.created_at,
"message_public_id": message.public_id,
"message_seq_in_session": message.seq_in_session,
"seq_in_session": message.seq_in_session,
}
for message in created_messages
]

2
src/routers/webhooks.py
View File

@ -61,7 +61,7 @@ async def list_webhook_endpoints(
if not jwt_params.ad and jwt_params.w is not None and jwt_params.w != workspace_id:
raise AuthenticationException("Unauthorized access to resource")
stmt = await crud.list_webhook_endpoints(db, workspace_id)
stmt = await crud.list_webhook_endpoints(workspace_id)
return await apaginate(db, stmt)

17
src/schemas.py
View File

@ -288,20 +288,12 @@ class DocumentBase(BaseModel):
class DocumentMetadata(BaseModel):
times_derived: int | None = Field(
default=None,
ge=1,
description="The number of times that a semantic duplicate document to this one has been derived",
)
message_ids: list[tuple[int, int]] = Field(
description="The ID range(s) of the messages that this document was derived from. Acts as a link to the primary source of the document. Note that as a document gets deduplicated, additional ranges will be added, because the same document could be derived from completely separate message ranges."
)
message_created_at: str = Field(
description="The timestamp of the message that this document was derived from. Note that this is not the same as the created_at timestamp of the document. This timestamp is usually only saved with second-level precision."
)
level: Literal["explicit", "deductive"] = Field(
description="The level of the document (explicit or deductive)"
)
premises: list[str] | None = Field(
default=None,
description="The premises of the deduction -- only applicable for deductive observations",
@ -313,6 +305,15 @@ class DocumentCreate(DocumentBase):
session_name: str = Field(
description="The session from which the document was derived"
)
level: Literal["explicit", "deductive"] = Field(
default="explicit",
description="The level of the document (explicit or deductive)",
)
times_derived: int = Field(
default=1,
ge=1,
description="The number of times that a semantic duplicate document to this one has been derived",
)
metadata: DocumentMetadata = Field()
embedding: list[float] = Field()

31
src/utils/queue_payload.py
View File

@ -14,9 +14,7 @@ class RepresentationPayload(BasePayload):
"""Payload for representation tasks."""
task_type: Literal["representation"] = "representation"
workspace_name: str
session_name: str
message_id: int
content: str
observer: str
observed: str
@ -33,9 +31,7 @@ class SummaryPayload(BasePayload):
"""Payload for summary tasks."""
task_type: Literal["summary"] = "summary"
workspace_name: str
session_name: str
message_id: int
message_seq_in_session: int
# Optional for backward compatibility with older queue items
message_public_id: str | None = None
@ -45,7 +41,6 @@ class WebhookPayload(BasePayload):
"""Payload for webhook delivery tasks."""
task_type: Literal["webhook"] = "webhook"
workspace_name: str
event_type: str
data: dict[str, Any]
@ -54,35 +49,33 @@ class DreamPayload(BasePayload):
"""Payload for dream tasks."""
task_type: Literal["dream"] = "dream"
workspace_name: str
dream_type: Literal["consolidate"] = "consolidate"
observer: str
observed: str
def create_webhook_payload(
workspace_name: str,
event_type: str,
data: dict[str, Any],
) -> dict[str, Any]:
return WebhookPayload(
workspace_name=workspace_name, event_type=event_type, data=data
).model_dump(mode="json")
"""Create a webhook payload."""
return WebhookPayload(event_type=event_type, data=data).model_dump(
mode="json", exclude_none=True
)
def create_dream_payload(
workspace_name: str,
dream_type: Literal["consolidate"] = "consolidate",
*,
observer: str,
observed: str,
) -> dict[str, Any]:
"""Create a dream payload."""
return DreamPayload(
workspace_name=workspace_name,
dream_type=dream_type,
observer=observer,
observed=observed,
).model_dump(mode="json")
).model_dump(mode="json", exclude_none=True)
def create_payload(
@ -96,6 +89,10 @@ def create_payload(
"""
Create a processed payload from a message for queue processing.
Note: workspace_name and message_id are no longer included in the returned payload
as they are now stored in dedicated columns on the queue table. The caller is
responsible for extracting and passing these values separately.
Args:
message: The original message dictionary
task_type: Type of task ('representation' or 'summary')
@ -104,7 +101,7 @@ def create_payload(
message_seq_in_session: Required for summary tasks, must be None for representation
Returns:
Processed payload dictionary ready for queue processing
Processed payload dictionary ready for queue processing (without workspace_name and message_id)
Raises:
ValueError: If the payload doesn't match the expected schema
@ -142,9 +139,7 @@ def create_payload(
validated_payload = RepresentationPayload(
content=content,
workspace_name=workspace_name,
session_name=session_name,
message_id=message_id,
created_at=created_at,
observer=observer,
observed=observed,
@ -161,16 +156,14 @@ def create_payload(
)
validated_payload = SummaryPayload(
workspace_name=workspace_name,
session_name=session_name,
message_id=message_id,
message_seq_in_session=message_seq_in_session,
message_public_id=message_public_id,
)
# Convert back to dict for compatibility with JSON serialization
# mode='json' ensures datetime is converted to ISO string
payload = validated_payload.model_dump(mode="json")
payload = validated_payload.model_dump(mode="json", exclude_none=True)
except Exception as e:
raise ValueError(f"Failed to create valid payload: {str(e)}") from e

4
src/utils/representation.py
View File

@ -271,7 +271,7 @@ class Representation(BaseModel):
session_name=doc.session_name,
)
for doc in documents
if doc.internal_metadata.get("level") == "explicit"
if doc.level == "explicit"
],
deductive=[
DeductiveObservation(
@ -284,7 +284,7 @@ class Representation(BaseModel):
premises=doc.internal_metadata.get("premises", []),
)
for doc in documents
if doc.internal_metadata.get("level") == "deductive"
if doc.level == "deductive"
],
)

2
src/utils/types.py
View File

@ -1,3 +1,5 @@
from typing import Literal
SupportedProviders = Literal["anthropic", "openai", "google", "groq", "custom", "vllm"]
TaskType = Literal["webhook", "summary", "representation", "dream"]
DocumentLevel = Literal["explicit", "deductive"]

6
src/utils/work_unit.py
View File

@ -15,11 +15,14 @@ class ParsedWorkUnit(BaseModel):
observed: str | None
def get_work_unit_key(payload: dict[str, Any] | ParsedWorkUnit) -> str:
def construct_work_unit_key(
workspace_name: str, payload: dict[str, Any] | ParsedWorkUnit
) -> str:
"""
Generate a work unit key for a given task type, workspace name, and event type.
Args:
workspace_name: The name of the workspace the work unit belongs to
payload: Dictionary containing work unit information
Returns:
@ -31,7 +34,6 @@ def get_work_unit_key(payload: dict[str, Any] | ParsedWorkUnit) -> str:
if isinstance(payload, ParsedWorkUnit):
payload = payload.model_dump()
workspace_name: str | None = payload.get("workspace_name")
task_type: str | None = payload.get("task_type")
if not workspace_name or not task_type:
raise ValueError(

13
src/webhooks/events.py
View File

@ -7,7 +7,7 @@ from pydantic import BaseModel
from src.dependencies import tracked_db
from src.models import QueueItem
from src.utils.queue_payload import create_webhook_payload
from src.utils.work_unit import get_work_unit_key
from src.utils.work_unit import construct_work_unit_key
logger = logging.getLogger(__name__)
@ -51,23 +51,26 @@ async def publish_webhook_event(event: WebhookEvent) -> None:
event: The webhook event to publish.
"""
try:
# Note: workspace_name is no longer included in the payload
# It's stored directly on the queue item
payload = create_webhook_payload(
workspace_name=event.workspace_id,
event_type=event.type.value,
data=event.model_dump(mode="json", exclude={"type"}),
)
async with tracked_db("publish_webhook_event") as db:
queue_item = QueueItem(
work_unit_key=get_work_unit_key(
work_unit_key=construct_work_unit_key(
event.workspace_id,
{
"task_type": "webhook",
"workspace_name": event.workspace_id,
}
},
),
payload=payload,
session_id=None,
task_type="webhook",
workspace_name=event.workspace_id,
message_id=None, # Webhooks don't have a message_id
)
db.add(queue_item)
await db.commit()

12
src/webhooks/webhook_delivery.py
View File

@ -15,16 +15,18 @@ from src.utils.queue_payload import WebhookPayload
logger = logging.getLogger(__name__)
async def deliver_webhook(db: AsyncSession, payload: WebhookPayload) -> None:
async def deliver_webhook(
db: AsyncSession, payload: WebhookPayload, workspace_name: str
) -> None:
"""
Deliver a single webhook event to its configured endpoints.
"""
async with httpx.AsyncClient(timeout=30.0) as client:
try:
webhook_urls = await _get_webhook_urls(db, payload.workspace_name)
webhook_urls = await _get_webhook_urls(db, workspace_name)
if not webhook_urls:
logger.debug(
f"No webhook endpoints for workspace {payload.workspace_name}, skipping."
f"No webhook endpoints for workspace {workspace_name}, skipping."
)
return
@ -72,7 +74,7 @@ async def deliver_webhook(db: AsyncSession, payload: WebhookPayload) -> None:
)
except httpx.RequestError:
logger.exception(f"Error sending webhook for {payload.workspace_name}.")
logger.exception(f"Error sending webhook for {workspace_name}.")
except Exception:
logger.exception("Unexpected error delivering webhook.")
@ -82,7 +84,7 @@ async def _get_webhook_urls(db: AsyncSession, workspace_name: str) -> list[str]:
Get all webhook endpoint URLs for a workspace.
"""
try:
endpoints = await list_webhook_endpoints(db, workspace_name)
endpoints = await list_webhook_endpoints(workspace_name)
result = await db.execute(endpoints)
return [endpoint.url for endpoint in result.scalars().all()]
except Exception:

6
tests/alembic/revisions/__init__.py
View File

@ -2,6 +2,7 @@
from . import (
test_05486ce795d5_make_session_name_required_on_messages,
test_066e87ca5b07_align_schema_with_declarative_models,
test_08894082221a_replace_collection_name_with_observer_,
test_20f89a421aff_rename_metamessage_type_to_label,
test_66e63cf2cf77_add_indexes_to_documents_table,
@ -12,13 +13,16 @@ from . import (
test_917195d9b5e9_add_messageembedding_table,
test_a1b2c3d4e5f6_initial_schema,
test_b765d82110bd_change_metamessages_to_user_level_with_,
test_b8183c5ffb48_codify_document_level_and_times_derived,
test_bb6fb3a7a643_add_message_seq_in_session_column,
test_c3828084f472_add_indexes_for_messages_and_,
test_d429de0e5338_adopt_peer_paradigm,
test_ec8f94139b02_codify_workspace_name_and_message_id_in_,
)
__all__ = [
"test_05486ce795d5_make_session_name_required_on_messages",
"test_066e87ca5b07_align_schema_with_declarative_models",
"test_08894082221a_replace_collection_name_with_observer_",
"test_20f89a421aff_rename_metamessage_type_to_label",
"test_556a16564f50_add_user_id_and_app_id_to_tables",
@ -29,7 +33,9 @@ __all__ = [
"test_917195d9b5e9_add_messageembedding_table",
"test_a1b2c3d4e5f6_initial_schema",
"test_b765d82110bd_change_metamessages_to_user_level_with_",
"test_b8183c5ffb48_codify_document_level_and_times_derived",
"test_bb6fb3a7a643_add_message_seq_in_session_column",
"test_c3828084f472_add_indexes_for_messages_and_",
"test_d429de0e5338_adopt_peer_paradigm",
"test_ec8f94139b02_codify_workspace_name_and_message_id_in_",
]

42
tests/alembic/revisions/test_066e87ca5b07_align_schema_with_declarative_models.py Normal file
View File

@ -0,0 +1,42 @@
"""Hooks for revision 066e87ca5b07 (align_schema_with_declarative_models)."""
from __future__ import annotations
from tests.alembic.registry import register_after_upgrade, register_before_upgrade
from tests.alembic.verifier import MigrationVerifier
@register_before_upgrade("066e87ca5b07")
def prepare_align_schema_with_declarative_models(verifier: MigrationVerifier) -> None:
"""Seed state and assertions before upgrading to 066e87ca5b07."""
# Assert columns exist but are nullable before migration
verifier.assert_column_exists(
"active_queue_sessions", "work_unit_key", exists=True, nullable=True
)
verifier.assert_column_exists("documents", "embedding", exists=True, nullable=True)
# Assert FK constraint does not exist yet
verifier.assert_constraint_exists(
"queue", "fk_queue_session_id", "foreign_key", exists=False
)
@register_after_upgrade("066e87ca5b07")
def verify_align_schema_with_declarative_models(verifier: MigrationVerifier) -> None:
"""Add assertions validating the effects of 066e87ca5b07."""
# Assert columns are now non-nullable
verifier.assert_column_exists(
"peers", "workspace_name", exists=True, nullable=False
)
verifier.assert_column_exists(
"sessions", "workspace_name", exists=True, nullable=False
)
verifier.assert_column_exists(
"active_queue_sessions", "work_unit_key", exists=True, nullable=False
)
verifier.assert_column_exists("documents", "embedding", exists=True, nullable=False)
# Assert FK constraint now exists
verifier.assert_constraint_exists(
"queue", "fk_queue_session_id", "foreign_key", exists=True
)

315
tests/alembic/revisions/test_b8183c5ffb48_codify_document_level_and_times_derived.py Normal file
View File

@ -0,0 +1,315 @@
"""Hooks for revision b8183c5ffb48 (codify_document_level_and_times_derived)."""
from __future__ import annotations
import json
from nanoid import generate as generate_nanoid
from sqlalchemy import text
from tests.alembic.registry import register_after_upgrade, register_before_upgrade
from tests.alembic.verifier import MigrationVerifier
# Test data constants
WORKSPACE_NAME = "test-workspace"
OBSERVER_NAME = "test-observer"
OBSERVED_NAME = "test-observed"
SESSION_NAME = "test-session"
@register_before_upgrade("b8183c5ffb48")
def prepare_codify_document_level_and_times_derived(
verifier: MigrationVerifier,
) -> None:
"""Seed state and assertions before upgrading to b8183c5ffb48."""
# Verify columns don't exist yet
verifier.assert_column_exists("documents", "level", exists=False)
verifier.assert_column_exists("documents", "times_derived", exists=False)
# Verify CHECK constraint doesn't exist yet
verifier.assert_constraint_exists("documents", "level_valid", "check", exists=False)
conn = verifier.conn
schema = verifier.schema
# Create workspace
conn.execute(
text(
f'INSERT INTO "{schema}"."workspaces" ("id", "name") VALUES (:ws_id, :ws_name)'
),
{"ws_id": generate_nanoid(), "ws_name": WORKSPACE_NAME},
)
# Create peers (observer and observed)
conn.execute(
text(
f'INSERT INTO "{schema}"."peers" ("id", "name", "workspace_name") '
+ "VALUES (:observer_id, :observer_name, :ws_name), "
+ "(:observed_id, :observed_name, :ws_name)"
),
{
"observer_id": generate_nanoid(),
"observer_name": OBSERVER_NAME,
"ws_name": WORKSPACE_NAME,
"observed_id": generate_nanoid(),
"observed_name": OBSERVED_NAME,
},
)
# Create session
conn.execute(
text(
f'INSERT INTO "{schema}"."sessions" ("id", "name", "workspace_name") '
+ "VALUES (:session_id, :session_name, :ws_name)"
),
{
"session_id": generate_nanoid(),
"session_name": SESSION_NAME,
"ws_name": WORKSPACE_NAME,
},
)
# Create collection (required for documents)
conn.execute(
text(
f'INSERT INTO "{schema}"."collections" ("id", "observer", "observed", "workspace_name") '
+ "VALUES (:collection_id, :observer, :observed, :ws_name)"
),
{
"collection_id": generate_nanoid(),
"observer": OBSERVER_NAME,
"observed": OBSERVED_NAME,
"ws_name": WORKSPACE_NAME,
},
)
# Create documents with different scenarios
# Scenario 1: Document with both level and times_derived in internal_metadata
conn.execute(
text(
f'INSERT INTO "{schema}"."documents" '
+ '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") '
+ "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)"
),
{
"id": generate_nanoid(),
"content": "Document with explicit level and times_derived=3",
"metadata": json.dumps({"level": "explicit", "times_derived": 3}),
"embedding": [0.1] * 1536,
"observer": OBSERVER_NAME,
"observed": OBSERVED_NAME,
"ws_name": WORKSPACE_NAME,
"session_name": SESSION_NAME,
},
)
# Scenario 2: Document with deductive level in internal_metadata
conn.execute(
text(
f'INSERT INTO "{schema}"."documents" '
+ '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") '
+ "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)"
),
{
"id": generate_nanoid(),
"content": "Document with deductive level and times_derived=5",
"metadata": json.dumps({"level": "deductive", "times_derived": 5}),
"embedding": [0.2] * 1536,
"observer": OBSERVER_NAME,
"observed": OBSERVED_NAME,
"ws_name": WORKSPACE_NAME,
"session_name": SESSION_NAME,
},
)
# Scenario 3: Document without level or times_derived (should get defaults)
conn.execute(
text(
f'INSERT INTO "{schema}"."documents" '
+ '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") '
+ "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)"
),
{
"id": generate_nanoid(),
"content": "Document without level or times_derived fields",
"metadata": json.dumps({"other_field": "value"}),
"embedding": [0.3] * 1536,
"observer": OBSERVER_NAME,
"observed": OBSERVED_NAME,
"ws_name": WORKSPACE_NAME,
"session_name": SESSION_NAME,
},
)
# Scenario 4: Document with only level in metadata
conn.execute(
text(
f'INSERT INTO "{schema}"."documents" '
+ '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") '
+ "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)"
),
{
"id": generate_nanoid(),
"content": "Document with only level field",
"metadata": json.dumps({"level": "explicit"}),
"embedding": [0.4] * 1536,
"observer": OBSERVER_NAME,
"observed": OBSERVED_NAME,
"ws_name": WORKSPACE_NAME,
"session_name": SESSION_NAME,
},
)
# Scenario 5: Document with only times_derived in metadata
conn.execute(
text(
f'INSERT INTO "{schema}"."documents" '
+ '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") '
+ "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)"
),
{
"id": generate_nanoid(),
"content": "Document with only times_derived field",
"metadata": json.dumps({"times_derived": 7}),
"embedding": [0.5] * 1536,
"observer": OBSERVER_NAME,
"observed": OBSERVED_NAME,
"ws_name": WORKSPACE_NAME,
"session_name": SESSION_NAME,
},
)
# Verify we have exactly 5 documents
count = conn.execute(text(f'SELECT COUNT(*) FROM "{schema}"."documents"')).scalar()
assert count == 5, f"Expected 5 documents but found {count}"
@register_after_upgrade("b8183c5ffb48")
def verify_codify_document_level_and_times_derived(verifier: MigrationVerifier) -> None:
"""Add assertions validating the effects of b8183c5ffb48."""
# Verify columns were added with correct nullability
verifier.assert_column_exists("documents", "level", nullable=False)
verifier.assert_column_exists("documents", "times_derived", nullable=False)
# Verify CHECK constraint exists
verifier.assert_constraint_exists("documents", "level_valid", "check")
conn = verifier.conn
schema = verifier.schema
# Verify all rows have non-null values after migration
verifier.assert_no_nulls("documents", "level")
verifier.assert_no_nulls("documents", "times_derived")
# Verify data transformation: level extracted from internal_metadata
explicit_count = conn.execute(
text(f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE "level" = :level'),
{"level": "explicit"},
).scalar()
deductive_count = conn.execute(
text(f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE "level" = :level'),
{"level": "deductive"},
).scalar()
# 3 documents should have explicit (scenarios 1, 3, 4)
# 1 document should have deductive (scenario 2)
# 1 document should have default explicit (scenario 5)
assert explicit_count == 4, f"Expected 4 explicit documents, got {explicit_count}"
assert deductive_count == 1, f"Expected 1 deductive document, got {deductive_count}"
# Verify specific times_derived values were migrated correctly
times_derived_3 = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."documents" '
+ 'WHERE "times_derived" = 3 AND "content" LIKE :pattern'
),
{"pattern": "%explicit level and times_derived=3%"},
).scalar()
assert (
times_derived_3 == 1
), f"Expected 1 document with times_derived=3, got {times_derived_3}"
times_derived_5 = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."documents" '
+ 'WHERE "times_derived" = 5 AND "content" LIKE :pattern'
),
{"pattern": "%deductive level and times_derived=5%"},
).scalar()
assert (
times_derived_5 == 1
), f"Expected 1 document with times_derived=5, got {times_derived_5}"
times_derived_7 = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."documents" '
+ 'WHERE "times_derived" = 7 AND "content" LIKE :pattern'
),
{"pattern": "%only times_derived field%"},
).scalar()
assert (
times_derived_7 == 1
), f"Expected 1 document with times_derived=7, got {times_derived_7}"
# Verify default times_derived=1 was applied to documents without it
times_derived_1 = conn.execute(
text(f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE "times_derived" = 1')
).scalar()
assert (
times_derived_1 == 2
), f"Expected 2 documents with times_derived=1, got {times_derived_1}"
# Verify internal_metadata still contains the original data (NOT removed by migration)
level_in_metadata = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE internal_metadata ? \'level\''
)
).scalar()
# 3 documents had level in metadata (scenarios 1, 2, 4)
assert (
level_in_metadata == 3
), f"Expected 3 documents with level in metadata, got {level_in_metadata}"
times_derived_in_metadata = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE internal_metadata ? \'times_derived\''
)
).scalar()
# 3 documents had times_derived in metadata (scenarios 1, 2, 5)
assert (
times_derived_in_metadata == 3
), f"Expected 3 documents with times_derived in metadata, got {times_derived_in_metadata}"
# Verify server defaults work for new documents
new_doc_id = generate_nanoid()
conn.execute(
text(
f'INSERT INTO "{schema}"."documents" '
+ '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") '
+ "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)"
),
{
"id": new_doc_id,
"content": "New document after migration",
"metadata": json.dumps({}),
"embedding": [0.6] * 1536,
"observer": OBSERVER_NAME,
"observed": OBSERVED_NAME,
"ws_name": WORKSPACE_NAME,
"session_name": SESSION_NAME,
},
)
# Verify the new document got default values
new_doc = conn.execute(
text(
f'SELECT "level", "times_derived" FROM "{schema}"."documents" WHERE "id" = :id'
),
{"id": new_doc_id},
).one()
assert (
new_doc.level == "explicit"
), f"Expected new document to have level='explicit', got {new_doc.level}"
assert (
new_doc.times_derived == 1
), f"Expected new document to have times_derived=1, got {new_doc.times_derived}"

284
tests/alembic/revisions/test_e9b705f9adf9_add_server_defaults_to_timestamp_.py Normal file
View File

@ -0,0 +1,284 @@
"""Hooks for revision e9b705f9adf9 (add server defaults to timestamp, boolean, and jsonb columns)."""
from __future__ import annotations
from nanoid import generate as generate_nanoid
from sqlalchemy import text
from tests.alembic.registry import register_after_upgrade, register_before_upgrade
from tests.alembic.verifier import MigrationVerifier
# Test data IDs
WORKSPACE_ID = generate_nanoid()
PEER_ID = generate_nanoid()
SESSION_ID = generate_nanoid()
MESSAGE_ID = generate_nanoid()
COLLECTION_ID = generate_nanoid()
DOCUMENT_ID = generate_nanoid()
@register_before_upgrade("e9b705f9adf9")
def prepare_add_server_defaults(verifier: MigrationVerifier) -> None:
"""Seed state before upgrading to e9b705f9adf9.
This migration adds server defaults to timestamp, JSONB, and boolean columns.
We verify that columns exist but don't have server defaults before the migration.
"""
conn = verifier.conn
schema = verifier.schema
inspector = verifier.get_inspector()
# Sample timestamp columns to check - they should exist but without server defaults
for table, column in [
("workspaces", "created_at"),
("peers", "created_at"),
("sessions", "created_at"),
("messages", "created_at"),
("collections", "created_at"),
("documents", "created_at"),
("queue", "created_at"),
]:
columns = inspector.get_columns(table, schema=schema)
col_info = next((c for c in columns if c["name"] == column), None)
assert (
col_info is not None
), f"Column {table}.{column} should exist before migration"
# Create test data to ensure existing rows work after migration
conn.execute(
text(
f'INSERT INTO "{schema}"."workspaces" '
+ '("id", "name", "created_at", "metadata", "internal_metadata", "configuration") '
+ "VALUES (:id, :name, NOW(), :metadata, :internal_metadata, :configuration)"
),
{
"id": WORKSPACE_ID,
"name": "test-workspace",
"metadata": "{}",
"internal_metadata": "{}",
"configuration": "{}",
},
)
conn.execute(
text(
f'INSERT INTO "{schema}"."peers" '
+ '("id", "name", "workspace_name", "created_at", "metadata", "internal_metadata", "configuration") '
+ "VALUES (:id, :name, :workspace_name, NOW(), :metadata, :internal_metadata, :configuration)"
),
{
"id": PEER_ID,
"name": "test-peer",
"workspace_name": "test-workspace",
"metadata": "{}",
"internal_metadata": "{}",
"configuration": "{}",
},
)
conn.execute(
text(
f'INSERT INTO "{schema}"."sessions" '
+ '("id", "name", "workspace_name", "created_at", "is_active", "metadata", "internal_metadata", "configuration") '
+ "VALUES (:id, :name, :workspace_name, NOW(), true, :metadata, :internal_metadata, :configuration)"
),
{
"id": SESSION_ID,
"name": "test-session",
"workspace_name": "test-workspace",
"metadata": "{}",
"internal_metadata": "{}",
"configuration": "{}",
},
)
@register_after_upgrade("e9b705f9adf9")
def verify_add_server_defaults(verifier: MigrationVerifier) -> None:
"""Validate server defaults were added correctly to all columns."""
conn = verifier.conn
schema = verifier.schema
inspector = verifier.get_inspector()
# Verify timestamp columns have server defaults (now() function)
timestamp_columns = [
("workspaces", "created_at"),
("peers", "created_at"),
("sessions", "created_at"),
("messages", "created_at"),
("message_embeddings", "created_at"),
("collections", "created_at"),
("documents", "created_at"),
("queue", "created_at"),
("webhook_endpoints", "created_at"),
("session_peers", "joined_at"),
("active_queue_sessions", "last_updated"),
]
for table, column in timestamp_columns:
columns = inspector.get_columns(table, schema=schema)
col_info = next((c for c in columns if c["name"] == column), None)
assert (
col_info is not None
), f"Column {table}.{column} not found after migration"
# Check that a server default exists
default = col_info.get("default")
assert default is not None, (
f"Column {table}.{column} should have a server default after migration, "
f"but default is None"
)
# Verify JSONB columns have server defaults (empty object '{}')
jsonb_columns = [
("workspaces", "metadata"),
("workspaces", "internal_metadata"),
("workspaces", "configuration"),
("peers", "metadata"),
("peers", "internal_metadata"),
("peers", "configuration"),
("sessions", "metadata"),
("sessions", "internal_metadata"),
("sessions", "configuration"),
("messages", "metadata"),
("messages", "internal_metadata"),
("collections", "metadata"),
("collections", "internal_metadata"),
("documents", "internal_metadata"),
("session_peers", "configuration"),
("session_peers", "internal_metadata"),
]
for table, column in jsonb_columns:
columns = inspector.get_columns(table, schema=schema)
col_info = next((c for c in columns if c["name"] == column), None)
assert (
col_info is not None
), f"Column {table}.{column} not found after migration"
# Check that a server default exists
default = col_info.get("default")
assert default is not None, (
f"Column {table}.{column} should have a server default after migration, "
f"but default is None"
)
# Verify boolean columns have server defaults
boolean_columns = [
("sessions", "is_active", "true"),
("queue", "processed", "false"),
]
for table, column, expected_default in boolean_columns:
columns = inspector.get_columns(table, schema=schema)
col_info = next((c for c in columns if c["name"] == column), None)
assert (
col_info is not None
), f"Column {table}.{column} not found after migration"
# Check that a server default exists
default = col_info.get("default")
assert default is not None, (
f"Column {table}.{column} should have a server default after migration, "
f"but default is None"
)
assert (
default == expected_default
), f"Column {table}.{column} should have a server default of {expected_default} after migration, but default is {default}"
# Test that defaults actually work by inserting rows without explicit values
test_workspace_id = generate_nanoid()
conn.execute(
text(
f'INSERT INTO "{schema}"."workspaces" ("id", "name") '
+ "VALUES (:id, :name)"
),
{"id": test_workspace_id, "name": "test-defaults-workspace"},
)
# Verify the inserted workspace has default values
workspace = conn.execute(
text(
'SELECT "created_at", "metadata", "internal_metadata", "configuration" '
+ f'FROM "{schema}"."workspaces" WHERE "id" = :id'
),
{"id": test_workspace_id},
).one()
assert workspace.created_at is not None, "created_at should be auto-populated"
assert workspace.metadata == {}, "metadata should default to empty object"
assert (
workspace.internal_metadata == {}
), "internal_metadata should default to empty object"
assert workspace.configuration == {}, "configuration should default to empty object"
# Test peer defaults
test_peer_id = generate_nanoid()
conn.execute(
text(
f'INSERT INTO "{schema}"."peers" ("id", "name", "workspace_name") '
+ "VALUES (:id, :name, :workspace_name)"
),
{
"id": test_peer_id,
"name": "test-defaults-peer",
"workspace_name": "test-defaults-workspace",
},
)
peer = conn.execute(
text(
'SELECT "created_at", "metadata", "internal_metadata", "configuration" '
+ f'FROM "{schema}"."peers" WHERE "id" = :id'
),
{"id": test_peer_id},
).one()
assert peer.created_at is not None, "peer created_at should be auto-populated"
assert peer.metadata == {}, "peer metadata should default to empty object"
assert (
peer.internal_metadata == {}
), "peer internal_metadata should default to empty object"
assert peer.configuration == {}, "peer configuration should default to empty object"
# Test session defaults (including boolean is_active)
test_session_id = generate_nanoid()
conn.execute(
text(
f'INSERT INTO "{schema}"."sessions" ("id", "name", "workspace_name") '
+ "VALUES (:id, :name, :workspace_name)"
),
{
"id": test_session_id,
"name": "test-defaults-session",
"workspace_name": "test-defaults-workspace",
},
)
session = conn.execute(
text(
'SELECT "created_at", "is_active", "metadata", "internal_metadata", "configuration" '
+ f'FROM "{schema}"."sessions" WHERE "id" = :id'
),
{"id": test_session_id},
).one()
assert session.created_at is not None, "session created_at should be auto-populated"
assert session.is_active is True, "session is_active should default to true"
assert session.metadata == {}, "session metadata should default to empty object"
assert (
session.internal_metadata == {}
), "session internal_metadata should default to empty object"
assert (
session.configuration == {}
), "session configuration should default to empty object"
# Verify pre-existing data still exists
existing_workspace = conn.execute(
text(f'SELECT "id" FROM "{schema}"."workspaces" WHERE "id" = :id'),
{"id": WORKSPACE_ID},
).one_or_none()
assert (
existing_workspace is not None
), "Pre-existing workspace should still exist after migration"

336
tests/alembic/revisions/test_ec8f94139b02_codify_workspace_name_and_message_id_in_.py Normal file
View File

@ -0,0 +1,336 @@
"""Hooks for revision ec8f94139b02 (codify_workspace_name_and_message_id_in_)."""
from __future__ import annotations
import json
from nanoid import generate as generate_nanoid
from sqlalchemy import text
from tests.alembic.registry import register_after_upgrade, register_before_upgrade
from tests.alembic.verifier import MigrationVerifier
# Test data constants
WORKSPACE_NAME_1 = "test-workspace-1"
WORKSPACE_NAME_2 = "test-workspace-2"
PEER_NAME_1 = "test-peer-1"
PEER_NAME_2 = "test-peer-2"
SESSION_NAME_1 = "test-session-1"
SESSION_NAME_2 = "test-session-2"
# Indexes that should be created by the migration
_INDEXES = (
("queue", "ix_queue_workspace_name"),
("queue", "ix_queue_message_id_not_null"),
("queue", "ix_queue_workspace_name_processed"),
("queue", "ix_queue_work_unit_key_processed_id"),
)
@register_before_upgrade("ec8f94139b02")
def prepare_codify_workspace_name_and_message_id_in(
verifier: MigrationVerifier,
) -> None:
"""Seed state and assertions before upgrading to ec8f94139b02."""
# Verify columns don't exist yet
verifier.assert_column_exists("queue", "workspace_name", exists=False)
verifier.assert_column_exists("queue", "message_id", exists=False)
# Verify foreign key constraints don't exist yet
verifier.assert_constraint_exists(
"queue", "fk_queue_workspace_name", "foreign_key", exists=False
)
verifier.assert_constraint_exists(
"queue", "fk_queue_message_id", "foreign_key", exists=False
)
# Verify indexes don't exist yet
verifier.assert_indexes_not_exist(_INDEXES)
conn = verifier.conn
schema = verifier.schema
# Create workspaces
conn.execute(
text(
f'INSERT INTO "{schema}"."workspaces" ("id", "name") '
+ "VALUES (:ws_id_1, :ws_name_1), (:ws_id_2, :ws_name_2)"
),
{
"ws_id_1": generate_nanoid(),
"ws_name_1": WORKSPACE_NAME_1,
"ws_id_2": generate_nanoid(),
"ws_name_2": WORKSPACE_NAME_2,
},
)
# Create peers
conn.execute(
text(
f'INSERT INTO "{schema}"."peers" ("id", "name", "workspace_name") '
+ "VALUES (:peer_id_1, :peer_name_1, :ws_name_1), "
+ "(:peer_id_2, :peer_name_2, :ws_name_2)"
),
{
"peer_id_1": generate_nanoid(),
"peer_name_1": PEER_NAME_1,
"ws_name_1": WORKSPACE_NAME_1,
"peer_id_2": generate_nanoid(),
"peer_name_2": PEER_NAME_2,
"ws_name_2": WORKSPACE_NAME_2,
},
)
# Create sessions
conn.execute(
text(
f'INSERT INTO "{schema}"."sessions" ("id", "name", "workspace_name") '
+ "VALUES (:session_id_1, :session_name_1, :ws_name_1), "
+ "(:session_id_2, :session_name_2, :ws_name_2)"
),
{
"session_id_1": generate_nanoid(),
"session_name_1": SESSION_NAME_1,
"ws_name_1": WORKSPACE_NAME_1,
"session_id_2": generate_nanoid(),
"session_name_2": SESSION_NAME_2,
"ws_name_2": WORKSPACE_NAME_2,
},
)
# Create some messages to reference in queue items
message_ids: list[str] = []
for i in range(10):
message_id: str = generate_nanoid()
message_ids.append(message_id)
workspace = WORKSPACE_NAME_1 if i % 2 == 0 else WORKSPACE_NAME_2
session = SESSION_NAME_1 if i % 2 == 0 else SESSION_NAME_2
peer = PEER_NAME_1 if i % 2 == 0 else PEER_NAME_2
conn.execute(
text(
f'INSERT INTO "{schema}"."messages" '
+ '("public_id", "workspace_name", "session_name", "peer_name", "content", "seq_in_session") '
+ "VALUES (:msg_id, :ws_name, :session_name, :peer_name, :content, :seq)"
),
{
"msg_id": message_id,
"ws_name": workspace,
"session_name": session,
"peer_name": peer,
"content": f"test message {i}",
"seq": i,
},
)
# Get internal message IDs for queue references
message_db_ids: list[int] = []
for msg_id in message_ids:
result = conn.execute(
text(f'SELECT "id" FROM "{schema}"."messages" WHERE "public_id" = :msg_id'),
{"msg_id": msg_id},
).one()
message_db_ids.append(result.id)
# Bulk insert 100k queue items with workspace_name and message_id in payload
# Use efficient batch insert with generate_series
conn.execute(text("SET LOCAL synchronous_commit = OFF"))
# Insert queue items in three categories:
# 1. Items with both workspace_name and message_id (60k)
# 2. Items with workspace_name but NO message_id (30k)
# 3. Items with workspace_name and NULL message_id value (10k)
# Category 1: Both workspace_name and message_id (60k items)
conn.execute(
text(
f'INSERT INTO "{schema}"."queue" '
+ '("work_unit_key", "task_type", "payload", "processed") '
+ "SELECT "
+ " 'work-unit-' || gs::text, "
+ " 'representation', "
+ " jsonb_build_object("
+ " 'workspace_name', CASE WHEN gs % 2 = 0 THEN :ws_name_1 ELSE :ws_name_2 END, "
+ " 'message_id', :msg_db_id_0 + (gs % 10), "
+ " 'other_field', 'value-' || gs::text"
+ " ), "
+ " false "
+ "FROM generate_series(1, :n) AS gs"
),
{
"ws_name_1": WORKSPACE_NAME_1,
"ws_name_2": WORKSPACE_NAME_2,
"msg_db_id_0": message_db_ids[0],
"n": 60_000,
},
)
# Category 2: Only workspace_name, no message_id key (30k items)
conn.execute(
text(
f'INSERT INTO "{schema}"."queue" '
+ '("work_unit_key", "task_type", "payload", "processed") '
+ "SELECT "
+ " 'work-unit-no-msg-' || gs::text, "
+ " 'summary', "
+ " jsonb_build_object("
+ " 'workspace_name', CASE WHEN gs % 2 = 0 THEN :ws_name_1 ELSE :ws_name_2 END, "
+ " 'other_field', 'value-' || gs::text"
+ " ), "
+ " false "
+ "FROM generate_series(60001, :n) AS gs"
),
{
"ws_name_1": WORKSPACE_NAME_1,
"ws_name_2": WORKSPACE_NAME_2,
"n": 90_000,
},
)
# Category 3: workspace_name with explicit NULL message_id (10k items)
conn.execute(
text(
f'INSERT INTO "{schema}"."queue" '
+ '("work_unit_key", "task_type", "payload", "processed") '
+ "SELECT "
+ " 'work-unit-null-msg-' || gs::text, "
+ " 'representation', "
+ " jsonb_build_object("
+ " 'workspace_name', CASE WHEN gs % 2 = 0 THEN :ws_name_1 ELSE :ws_name_2 END, "
+ " 'message_id', NULL::bigint, "
+ " 'other_field', 'value-' || gs::text"
+ " ), "
+ " false "
+ "FROM generate_series(90001, :n) AS gs"
),
{
"ws_name_1": WORKSPACE_NAME_1,
"ws_name_2": WORKSPACE_NAME_2,
"n": 100_000,
},
)
# Verify we have exactly 100k queue items
count = conn.execute(text(f'SELECT COUNT(*) FROM "{schema}"."queue"')).scalar()
assert count == 100_000, f"Expected 100k queue items but found {count}"
@register_after_upgrade("ec8f94139b02")
def verify_codify_workspace_name_and_message_id_in(verifier: MigrationVerifier) -> None:
"""Add assertions validating the effects of ec8f94139b02."""
# Verify columns were added with correct nullability
verifier.assert_column_exists("queue", "workspace_name", nullable=False)
verifier.assert_column_exists("queue", "message_id", nullable=True)
# Verify all indexes were created
verifier.assert_indexes_exist(_INDEXES)
# Verify foreign key constraints exist
verifier.assert_constraint_exists("queue", "fk_queue_workspace_name", "foreign_key")
verifier.assert_constraint_exists("queue", "fk_queue_message_id", "foreign_key")
conn = verifier.conn
schema = verifier.schema
# Verify all rows have non-null workspace_name after migration
verifier.assert_no_nulls("queue", "workspace_name")
# Verify data transformation: workspace_name extracted from payload
ws1_count = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."queue" '
+ 'WHERE "workspace_name" = :ws_name'
),
{"ws_name": WORKSPACE_NAME_1},
).scalar()
ws2_count = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."queue" '
+ 'WHERE "workspace_name" = :ws_name'
),
{"ws_name": WORKSPACE_NAME_2},
).scalar()
# Should be roughly 50/50 split (we alternate in the insert)
assert (
ws1_count == 50_000
), f"Expected 50k items with workspace_name_1, got {ws1_count}"
assert (
ws2_count == 50_000
), f"Expected 50k items with workspace_name_2, got {ws2_count}"
# Verify data transformation: message_id extracted from payload where it exists
msg_id_count = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."queue" '
+ 'WHERE "message_id" IS NOT NULL'
)
).scalar()
# Should be 60k items with message_id (category 1 only)
assert (
msg_id_count == 60_000
), f"Expected 60k items with message_id, got {msg_id_count}"
# Verify items without message_id in payload have NULL in column
null_msg_id_count = conn.execute(
text(f'SELECT COUNT(*) FROM "{schema}"."queue" ' + 'WHERE "message_id" IS NULL')
).scalar()
# Should be 40k items (30k without key + 10k with NULL value)
assert (
null_msg_id_count == 40_000
), f"Expected 40k items with NULL message_id, got {null_msg_id_count}"
# Verify workspace_name was removed from payload
ws_in_payload_count = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."queue" '
+ "WHERE payload ? 'workspace_name'"
)
).scalar()
assert (
ws_in_payload_count == 0
), f"Found {ws_in_payload_count} items still with workspace_name in payload"
# Verify message_id was removed from payload
msg_in_payload_count = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."queue" ' + "WHERE payload ? 'message_id'"
)
).scalar()
assert (
msg_in_payload_count == 0
), f"Found {msg_in_payload_count} items still with message_id in payload"
# Verify other_field remains in payload (data preservation)
other_field_count = conn.execute(
text(
f'SELECT COUNT(*) FROM "{schema}"."queue" '
+ "WHERE payload ? 'other_field'"
)
).scalar()
assert (
other_field_count == 100_000
), f"Expected all 100k items to retain other_field in payload, got {other_field_count}"
# Spot check: verify a specific queue item was transformed correctly
sample_item = conn.execute(
text(
'SELECT "workspace_name", "message_id", "payload" '
+ f'FROM "{schema}"."queue" '
+ "WHERE work_unit_key = 'work-unit-1' "
+ "LIMIT 1"
)
).one()
assert sample_item.workspace_name == WORKSPACE_NAME_2 # gs=1 is odd, so workspace 2
assert sample_item.message_id is not None # Category 1 item
payload = (
json.loads(sample_item.payload)
if isinstance(sample_item.payload, str)
else sample_item.payload
)
assert "workspace_name" not in payload
assert "message_id" not in payload
assert payload.get("other_field") == "value-1"

143
tests/bench/harness.py
View File

@ -11,6 +11,7 @@ This script:
"""
import argparse
import asyncio
import os
import shutil
import subprocess
@ -22,6 +23,8 @@ from pathlib import Path
import yaml
from src.cache.client import close_cache, init_cache
class HonchoHarness:
"""
@ -65,6 +68,9 @@ class HonchoHarness:
# Update the database port
compose_data["services"]["database"]["ports"] = [f"{self.db_port}:5432"]
# Update the Redis port
compose_data["services"]["redis"]["ports"] = ["6379:6379"]
# Add a unique project name to avoid conflicts
compose_data["name"] = f"honcho_harness_{self.db_port}"
@ -111,13 +117,15 @@ class HonchoHarness:
def get_database_env_vars(self) -> dict[str, str]:
"""
Get environment variables for database configuration and required API keys.
Get environment variables for database configuration, cache configuration, and required API keys.
Returns:
Dictionary of environment variables for database connection and API keys
Dictionary of environment variables for database connection, cache, and API keys
"""
return {
"DB_CONNECTION_URI": f"postgresql+psycopg://testuser:testpwd@localhost:{self.db_port}/honcho",
"CACHE_ENABLED": "true",
"CACHE_URL": "redis://localhost:6379/0",
}
def start_database(self) -> None:
@ -149,6 +157,99 @@ class HonchoHarness:
print("Database started successfully")
def start_redis(self) -> None:
"""
Start the Redis cache server using Docker Compose.
"""
print("Starting Redis cache server on port 6379...")
# Change to the temp directory and start the redis service
result = subprocess.run(
[
"docker-compose",
"-f",
str(self.docker_compose_file),
"-p",
f"honcho_harness_{self.db_port}",
"up",
"-d",
"redis",
],
cwd=self.temp_dir,
capture_output=True,
text=True,
)
if result.returncode != 0:
print(f"Failed to start Redis: {result.stderr}")
sys.exit(1)
print("Redis started successfully")
def wait_for_redis(self, timeout: int = 30) -> bool:
"""
Wait for Redis to be ready.
Args:
timeout: Maximum time to wait in seconds
Returns:
True if Redis is ready, False otherwise
"""
print("Waiting for Redis to be ready...")
start_time = time.time()
redis_port = 6379
while time.time() - start_time < timeout:
try:
import redis
# Test Redis connection
r = redis.Redis(
host="localhost", port=redis_port, decode_responses=True
)
r.ping() # pyright: ignore[reportUnknownMemberType]
print("Redis is ready!")
return True
except Exception:
pass
time.sleep(1) # Check every second
print("Redis failed to become ready within timeout")
return False
async def init_cache(self) -> None:
"""
Initialize the Redis cache connection.
"""
try:
# Add the project root to the path so we can import Honcho modules
sys.path.insert(0, str(self.project_root))
# Set environment variables for cache configuration
env = self.get_database_env_vars()
for key, value in env.items():
os.environ[key] = value
await init_cache()
print(f"[Instance {self.instance_id}] Cache initialized successfully")
except Exception as e:
print(f"[Instance {self.instance_id}] Failed to initialize cache: {e}")
async def close_cache(self) -> None:
"""
Close the Redis cache connection.
"""
try:
# Add the project root to the path so we can import Honcho modules
sys.path.insert(0, str(self.project_root))
await close_cache()
print(f"[Instance {self.instance_id}] Cache closed successfully")
except Exception as e:
print(f"[Instance {self.instance_id}] Failed to close cache: {e}")
def wait_for_database(self, timeout: int = 60) -> bool:
"""
Wait for the database to be ready.
@ -576,10 +677,16 @@ except Exception as e:
except Exception as e:
print(f"Error removing temp directory: {e}")
# Close cache
try:
asyncio.run(self.close_cache())
except Exception as e:
print(f"Error closing cache: {e}")
# Restore .env file
self.restore_env_file()
def run(self) -> None:
async def run(self) -> None:
"""
Run the complete Honcho harness.
"""
@ -606,11 +713,22 @@ except Exception as e:
# Start database
self.start_database()
# Start Redis
self.start_redis()
# Wait for database to be ready
if not self.wait_for_database():
print("Database failed to start. Exiting.")
sys.exit(1)
# Wait for Redis to be ready
if not self.wait_for_redis():
print("Redis failed to start. Exiting.")
sys.exit(1)
# Initialize cache
await self.init_cache()
# Provision database
self.provision_database()
@ -700,7 +818,7 @@ class HonchoHarnessPool:
)
self.harnesses.append(harness)
def run(self) -> None:
async def run(self) -> None:
"""
Run all Honcho harnesses in the pool.
"""
@ -748,6 +866,9 @@ class HonchoHarnessPool:
# Start database
harness.start_database()
# Start Redis
harness.start_redis()
# Wait for database to be ready
if not harness.wait_for_database():
print(
@ -755,6 +876,16 @@ class HonchoHarnessPool:
)
sys.exit(1)
# Wait for Redis to be ready
if not harness.wait_for_redis():
print(
f"Redis failed to start for instance {harness.instance_id}. Exiting."
)
sys.exit(1)
# Initialize cache
await harness.init_cache()
# Provision database
harness.provision_database()
@ -903,7 +1034,7 @@ Examples:
base_api_port=args.api_port,
project_root=args.project_root,
)
pool.run()
asyncio.run(pool.run())
else:
harness = HonchoHarness(
db_port=args.port,
@ -911,7 +1042,7 @@ Examples:
project_root=args.project_root,
instance_id=0,
)
harness.run()
asyncio.run(harness.run())
if __name__ == "__main__":

24
tests/bench/longmem.py
View File

@ -870,7 +870,7 @@ Evaluate whether the actual response correctly answers the question based on the
return results
async def run_all_questions(
self, test_file: Path, batch_size: int = 10
self, test_file: Path, batch_size: int = 10, test_count: int | None = None
) -> tuple[list[TestResult], float]:
"""
Run all questions in a longmemeval test file.
@ -878,11 +878,20 @@ Evaluate whether the actual response correctly answers the question based on the
Args:
test_file: Path to the longmemeval JSON file
batch_size: Number of questions to run concurrently in each batch
test_count: Optional number of tests to run (runs first N tests)
Returns:
Tuple of (list of test results, total duration)
"""
questions = self.load_test_file(test_file)
# Limit to first N questions if test_count is specified
if test_count is not None and test_count > 0:
questions = questions[:test_count]
print(
f"limiting to first {len(questions)} {'question' if len(questions) == 1 else 'questions'} from {test_file}"
)
print(
f"found {len(questions)} {'question' if len(questions) == 1 else 'questions'} in {test_file}"
)
@ -1140,6 +1149,7 @@ Examples:
%(prog)s --test-file tests/bench/longmemeval_data/longmemeval_s.json # Run longmemeval tests
%(prog)s --test-file test.json --pool-size 4 # Use 4 Honcho instances
%(prog)s --test-file test.json --base-api-port 8000 --pool-size 4 # Custom base port with pool
%(prog)s --test-file test.json --test-count 50 # Run only first 50 tests
""",
)
@ -1208,6 +1218,12 @@ Examples:
help="Use get_context + judge LLM instead of dialectic .chat endpoint (default: False)",
)
parser.add_argument(
"--test-count",
type=int,
help="Number of tests to run from the test file (default: all tests)",
)
args = parser.parse_args()
# Validate arguments
@ -1223,6 +1239,10 @@ Examples:
print(f"Error: Pool size must be positive, got {args.pool_size}")
return 1
if args.test_count is not None and args.test_count <= 0:
print(f"Error: Test count must be positive, got {args.test_count}")
return 1
# Create test runner
runner = LongMemEvalRunner(
base_api_port=args.base_api_port,
@ -1237,7 +1257,7 @@ Examples:
try:
# Run all questions
results, total_elapsed = await runner.run_all_questions(
args.test_file, args.batch_size
args.test_file, args.batch_size, args.test_count
)
runner.print_summary(results, total_elapsed_seconds=total_elapsed)

78
tests/conftest.py
View File

@ -6,6 +6,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
import jwt
import pytest
import pytest_asyncio
from cashews.backends.interface import ControlMixin
from cashews.picklers import PicklerType
from fakeredis import FakeAsyncRedis
from fastapi import Request
from fastapi.responses import JSONResponse
from fastapi.testclient import TestClient
@ -26,6 +29,7 @@ from sqlalchemy_utils import (
)
from src import models
from src.cache.client import cache
from src.config import settings
from src.db import Base
from src.dependencies import get_db
@ -164,8 +168,79 @@ async def db_session(db_engine: AsyncEngine):
await session.rollback()
@pytest_asyncio.fixture(scope="session")
async def fake_cache_session():
"""Set up fakeredis for caching once per test session."""
# Store original settings
original_enabled = settings.CACHE.ENABLED
original_url = settings.CACHE.URL
# Create a fake redis instance that persists for the session
fake_redis = FakeAsyncRedis(decode_responses=True)
# Patch redis creation to use fakeredis
# Cashews uses redis.asyncio.from_url to create connections
def fake_redis_from_url(*_args: Any, **_kwargs: Any):
return fake_redis
# Patch the cashews backend's _disable property to avoid ContextVar issues
# This works around cashews' ContextVar not being properly initialized in TestClient context
original_disable_property = ControlMixin._disable # pyright: ignore[reportPrivateUsage]
@property # type: ignore
def patched_disable_property(self): # pyright: ignore
try:
return original_disable_property.fget(self) # pyright: ignore[reportOptionalCall]
except LookupError:
# Return empty set as default if ContextVar not set in current context
return set() # pyright: ignore
# Start patching
redis_patch = patch("redis.asyncio.from_url", fake_redis_from_url)
redis_patch.start()
ControlMixin._disable = patched_disable_property # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue]
try:
# Enable caching and set URL for tests
settings.CACHE.ENABLED = True
settings.CACHE.URL = "redis://fake-redis:6379/0"
# Setup cache for tests that don't use TestClient (direct CRUD tests)
# For TestClient tests, the app's lifespan handler will also call cache.setup()
# The ContextVar patch above handles any context issues
cache.setup( # pyright: ignore[reportUnknownMemberType]
"redis://fake-redis:6379/0", pickle_type=PicklerType.SQLALCHEMY, enable=True
)
yield fake_redis
finally:
# Stop the patches
redis_patch.stop()
ControlMixin._disable = original_disable_property # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue]
# Restore original settings
settings.CACHE.ENABLED = original_enabled
settings.CACHE.URL = original_url
@pytest_asyncio.fixture(scope="function", autouse=True)
async def fake_cache(fake_cache_session: FakeAsyncRedis):
"""Clear cache between tests."""
# Clear cache before each test
await fake_cache_session.flushall() # pyright: ignore[reportUnknownMemberType]
yield cache
# Clear cache after each test
await fake_cache_session.flushall() # pyright: ignore[reportUnknownMemberType]
@pytest.fixture(scope="function")
async def client(db_session: AsyncSession):
async def client(
db_session: AsyncSession,
fake_cache_session: FakeAsyncRedis, # pyright: ignore[reportUnusedParameter]
) -> AsyncGenerator[TestClient, Any]:
"""Create a FastAPI TestClient for the scope of a single test function"""
# Register exception handlers for tests
@ -491,6 +566,7 @@ def mock_tracked_db(db_session: AsyncSession):
patch("src.dependencies.tracked_db", mock_tracked_db_context),
patch("src.deriver.queue_manager.tracked_db", mock_tracked_db_context),
patch("src.routers.sessions.tracked_db", mock_tracked_db_context),
patch("src.routers.peers.tracked_db", mock_tracked_db_context),
patch("src.crud.representation.tracked_db", mock_tracked_db_context),
patch("src.routers.peers.tracked_db", mock_tracked_db_context),
patch("src.dreamer.dreamer.tracked_db", mock_tracked_db_context),

1
tests/crud/test_workspace.py
View File

@ -318,6 +318,7 @@ class TestWorkspaceCRUD:
work_unit_key=f"representation:{test_workspace.name}:{session.name}:{test_peer.name}:{test_peer.name}",
task_type="representation",
payload={"test": "data"},
workspace_name=test_workspace.name,
)
db_session.add(queue_item)
await db_session.flush()

43
tests/deriver/conftest.py
View File

@ -1,7 +1,7 @@
import asyncio
from collections.abc import Callable, Generator
from collections.abc import Awaitable, Callable, Generator, Sequence
from datetime import datetime, timezone
from typing import Any, Literal
from typing import Any, Literal, TypeAlias, cast
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -11,7 +11,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
from src import crud, models, schemas
from src.utils.queue_payload import create_payload
from src.utils.work_unit import get_work_unit_key
from src.utils.work_unit import construct_work_unit_key
QueuePayload: TypeAlias = dict[str, Any]
QueuePayloadEntry: TypeAlias = QueuePayload | tuple[QueuePayload, int | None]
@pytest.fixture
@ -155,18 +158,29 @@ def create_queue_payload() -> Callable[..., Any]:
@pytest.fixture
async def add_queue_items(
db_session: AsyncSession,
) -> Callable[[list[dict[str, Any]], str], Any]:
) -> Callable[
[Sequence[QueuePayloadEntry], str, str], Awaitable[list[models.QueueItem]]
]:
"""Helper function to add queue items to the database"""
async def _add_items(
payloads: list[dict[str, Any]], session_id: str
payloads: Sequence[QueuePayloadEntry],
session_id: str,
workspace_name: str,
) -> list[models.QueueItem]:
"""Add queue items to the database and return them"""
queue_items: list[models.QueueItem] = []
for payload in payloads:
for payload_entry in payloads:
payload: QueuePayload
message_id: int | None
if isinstance(payload_entry, tuple):
payload, message_id = payload_entry
else:
payload = payload_entry
message_id = cast(int | None, payload.get("message_id"))
# Generate work_unit_key from the payload
task_type = payload.get("task_type", "unknown")
work_unit_key = get_work_unit_key(payload)
task_type = cast(str, payload.get("task_type", "unknown"))
work_unit_key = construct_work_unit_key(workspace_name, payload)
queue_item = models.QueueItem(
session_id=session_id,
@ -174,6 +188,8 @@ async def add_queue_items(
work_unit_key=work_unit_key,
payload=payload,
processed=False,
workspace_name=workspace_name,
message_id=message_id,
)
db_session.add(queue_item)
queue_items.append(queue_item)
@ -203,7 +219,7 @@ async def sample_queue_items(
messages = sample_messages
# Create various types of queue payloads
payloads: list[dict[str, Any]] = []
payloads: list[tuple[dict[str, Any], int]] = []
# Create representation payloads for each message
for message in messages:
@ -214,7 +230,7 @@ async def sample_queue_items(
observer=message.peer_name,
observed=message.peer_name,
)
payloads.append(payload1)
payloads.append((payload1, message.id))
# Representation for observer peer
payload2 = create_queue_payload(
@ -223,7 +239,7 @@ async def sample_queue_items(
observer=peer2.name, # peer2 observes others
observed=message.peer_name,
)
payloads.append(payload2)
payloads.append((payload2, message.id))
# Create summary payloads for session
for i, message in enumerate(messages):
@ -232,10 +248,10 @@ async def sample_queue_items(
task_type="summary",
message_seq_in_session=i + 1,
)
payloads.append(payload)
payloads.append((payload, message.id))
# Add all payloads as queue items
queue_items = await add_queue_items(payloads, session.id)
queue_items = await add_queue_items(payloads, session.id, session.workspace_name)
return queue_items
@ -294,6 +310,5 @@ def mock_representation_manager(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
mock_manager = AsyncMock(spec=RepresentationManager)
mock_manager.save_representation.return_value = 0
mock_manager.get_relevant_observations = AsyncMock(return_value=MagicMock())
return mock_manager

19
tests/deriver/test_deriver_processing.py
View File

@ -9,6 +9,7 @@ import pytest
from src import models
from src.deriver.deriver import process_representation_tasks_batch
from src.utils.representation import Representation
from src.utils.work_unit import construct_work_unit_key
@pytest.mark.asyncio
@ -32,14 +33,12 @@ class TestDeriverProcessing:
sample_session_with_peers: tuple[models.Session, list[models.Peer]],
):
"""Test that work unit keys are generated correctly"""
from src.utils.work_unit import get_work_unit_key
session, peers = sample_session_with_peers
peer1, peer2, _ = peers
# Create a payload for representation task
representation_payload = {
"workspace_name": "workspace1",
"session_name": session.name,
"observer": peer2.name,
"observed": peer1.name,
@ -47,22 +46,25 @@ class TestDeriverProcessing:
}
# Generate work unit key for representation
work_unit_key = get_work_unit_key(representation_payload)
expected_key = (
f"representation:workspace1:{session.name}:{peer2.name}:{peer1.name}"
work_unit_key = construct_work_unit_key(
session.workspace_name, representation_payload
)
expected_key = f"representation:{session.workspace_name}:{session.name}:{peer2.name}:{peer1.name}"
assert work_unit_key == expected_key
# Create a payload for summary task
summary_payload = {
"workspace_name": "workspace1",
"session_name": session.name,
"task_type": "summary",
}
# Generate work unit key for summary
summary_work_unit_key = get_work_unit_key(summary_payload)
expected_summary_key = f"summary:workspace1:{session.name}:None:None"
summary_work_unit_key = construct_work_unit_key(
session.workspace_name, summary_payload
)
expected_summary_key = (
f"summary:{session.workspace_name}:{session.name}:None:None"
)
assert summary_work_unit_key == expected_summary_key
async def test_mock_queue_manager(
@ -100,7 +102,6 @@ class TestDeriverProcessing:
await mock_representation_manager.save_representation(
Representation(explicit=[], deductive=[])
)
mock_representation_manager.get_relevant_observations.return_value = [] # type: ignore[attr-defined]
# Verify the methods were called
assert mock_representation_manager.save_representation.called # type: ignore[attr-defined]

101
tests/deriver/test_queue_processing.py
View File

@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from src import models
from src.config import settings
from src.deriver.queue_manager import QueueManager, WorkerOwnership
from src.utils.work_unit import get_work_unit_key
from src.utils.work_unit import construct_work_unit_key
@pytest.mark.asyncio
@ -139,7 +139,7 @@ class TestQueueProcessing:
for message in messages:
await db_session.refresh(message)
payloads: list[Any] = []
payloads: list[tuple[dict[str, Any], int]] = []
for message in messages:
payload = create_queue_payload( # type: ignore[reportUnknownArgumentType]
message=message,
@ -147,9 +147,9 @@ class TestQueueProcessing:
observed=peer.name,
observer=peer.name,
)
payloads.append(payload)
payloads.append((payload, message.id))
items = await add_queue_items(payloads, session.id)
items = await add_queue_items(payloads, session.id, session.workspace_name)
# Determine ascending order by DB id
ordered = (
(
@ -304,20 +304,23 @@ class TestQueueProcessing:
await db_session.refresh(message)
# Create queue items with token counts
payloads = [
create_queue_payload( # type: ignore[reportUnknownArgumentType]
message=msg,
task_type="representation",
observed=peer.name,
observer=peer.name,
payload_entries = [
(
create_queue_payload( # type: ignore[reportUnknownArgumentType]
message=msg,
task_type="representation",
observed=peer.name,
observer=peer.name,
),
msg,
)
for msg in messages
]
queue_items: list[models.QueueItem] = []
for payload in payloads:
for payload, message in payload_entries:
task_type = payload.get("task_type", "unknown")
work_unit_key = get_work_unit_key(payload)
work_unit_key = construct_work_unit_key(session.workspace_name, payload)
queue_item = models.QueueItem(
session_id=session.id,
@ -325,6 +328,8 @@ class TestQueueProcessing:
work_unit_key=work_unit_key,
payload=payload,
processed=False,
workspace_name=session.workspace_name,
message_id=message.id,
)
db_session.add(queue_item)
queue_items.append(queue_item)
@ -430,7 +435,7 @@ class TestQueueProcessing:
observed=peer.name,
observer=target.name,
)
work_unit_key = get_work_unit_key(payload)
work_unit_key = construct_work_unit_key(session.workspace_name, payload)
queue_item = models.QueueItem(
session_id=session.id,
@ -438,6 +443,8 @@ class TestQueueProcessing:
work_unit_key=work_unit_key,
payload=payload,
processed=False,
workspace_name=session.workspace_name,
message_id=message.id,
)
db_session.add(queue_item)
@ -596,7 +603,7 @@ class TestQueueProcessing:
observed=peer.name,
observer=target.name,
)
work_unit_key = get_work_unit_key(payload)
work_unit_key = construct_work_unit_key(session.workspace_name, payload)
queue_item = models.QueueItem(
session_id=session.id,
@ -604,6 +611,8 @@ class TestQueueProcessing:
work_unit_key=work_unit_key,
payload=payload,
processed=False,
workspace_name=session.workspace_name,
message_id=message.id,
)
db_session.add(queue_item)
@ -702,7 +711,6 @@ class TestQueueProcessing:
token_counts = [500, 600]
messages = [
models.Message(
id=999,
session_name=session.name,
workspace_name=session.workspace_name,
peer_name=peer.name,
@ -711,7 +719,6 @@ class TestQueueProcessing:
seq_in_session=1,
),
models.Message(
id=1000,
session_name=session.name,
workspace_name=session.workspace_name,
peer_name=peer.name,
@ -721,6 +728,15 @@ class TestQueueProcessing:
),
]
# Save messages to database first
for message in messages:
db_session.add(message)
await db_session.commit()
# Refresh to get the actual IDs
for message in messages:
await db_session.refresh(message)
# Create payloads and queue items
queue_items: list[models.QueueItem] = []
for i, message in enumerate(messages):
@ -729,7 +745,7 @@ class TestQueueProcessing:
)
payload["token_count"] = token_counts[i]
work_unit_key = get_work_unit_key(payload)
work_unit_key = construct_work_unit_key(session.workspace_name, payload)
queue_item = models.QueueItem(
session_id=session.id,
@ -737,6 +753,8 @@ class TestQueueProcessing:
work_unit_key=work_unit_key,
payload=payload,
processed=False,
workspace_name=session.workspace_name,
message_id=message.id,
)
db_session.add(queue_item)
queue_items.append(queue_item)
@ -747,10 +765,11 @@ class TestQueueProcessing:
processed_batches: list[dict[str, Any]] = []
async def mock_process_item(
task_type: str,
queue_payload: dict[str, Any], # pyright: ignore[reportUnusedParameter]
queue_item: models.QueueItem,
) -> None:
processed_batches.append({"task_type": task_type, "payload_count": 1})
processed_batches.append(
{"task_type": queue_item.task_type, "payload_count": 1}
)
qm = QueueManager()
work_unit_key = queue_items[0].work_unit_key
@ -838,21 +857,24 @@ class TestQueueProcessing:
await db_session.refresh(message)
# Create queue items
payloads = [
create_queue_payload( # type: ignore[reportUnknownArgumentType]
message=msg,
task_type="representation",
observed=peer.name,
observer=peer.name,
payload_entries = [
(
create_queue_payload( # type: ignore[reportUnknownArgumentType]
message=msg,
task_type="representation",
observed=peer.name,
observer=peer.name,
),
msg,
)
for msg in messages
]
# Add items to queue
queue_items: list[models.QueueItem] = []
for payload in payloads:
for payload, message in payload_entries:
task_type = payload.get("task_type", "unknown")
work_unit_key = get_work_unit_key(payload)
work_unit_key = construct_work_unit_key(session.workspace_name, payload)
queue_item = models.QueueItem(
session_id=session.id,
@ -860,6 +882,8 @@ class TestQueueProcessing:
work_unit_key=work_unit_key,
payload=payload,
processed=False,
workspace_name=session.workspace_name,
message_id=message.id,
)
db_session.add(queue_item)
queue_items.append(queue_item)
@ -949,21 +973,24 @@ class TestQueueProcessing:
await db_session.refresh(message)
# Create queue items
payloads = [
create_queue_payload( # type: ignore[reportUnknownArgumentType]
message=msg,
task_type="representation",
observed=peer.name,
observer=peer.name,
payload_entries = [
(
create_queue_payload( # type: ignore[reportUnknownArgumentType]
message=msg,
task_type="representation",
observed=peer.name,
observer=peer.name,
),
msg,
)
for msg in messages
]
# Add items to queue
queue_items: list[models.QueueItem] = []
for payload in payloads:
for payload, message in payload_entries:
task_type = payload.get("task_type", "unknown")
work_unit_key = get_work_unit_key(payload)
work_unit_key = construct_work_unit_key(session.workspace_name, payload)
queue_item = models.QueueItem(
session_id=session.id,
@ -971,6 +998,8 @@ class TestQueueProcessing:
work_unit_key=work_unit_key,
payload=payload,
processed=False,
workspace_name=session.workspace_name,
message_id=message.id,
)
db_session.add(queue_item)
queue_items.append(queue_item)

144
tests/integration/test_enqueue.py
View File

@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from src import crud, models, schemas
from src.deriver import enqueue
from src.deriver.enqueue import generate_queue_records
from src.models import Peer, QueueItem, Workspace
@ -1379,3 +1380,146 @@ class TestAdvancedEnqueueEdgeCases:
assert len(actual_payloads) == len(expected_payloads)
for expected in expected_payloads:
assert expected in actual_payloads
@pytest.mark.asyncio
class TestGenerateQueueRecordsSeqInSession:
"""Unit tests for generate_queue_records function focusing on seq_in_session handling"""
async def test_generate_queue_records_uses_seq_from_payload_not_crud(
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
):
"""
Test that generate_queue_records uses seq_in_session from payload
instead of making a CRUD call to get_message_seq_in_session.
"""
test_workspace, test_peer = sample_data
# Create a test session
test_session = models.Session(
workspace_name=test_workspace.name, name=str(generate_nanoid())
)
db_session.add(test_session)
await db_session.commit()
# Create a message payload with seq_in_session included
message_payload = {
"message_id": 12345,
"peer_name": test_peer.name,
"workspace_name": test_workspace.name,
"session_name": test_session.name,
"content": "Test message",
"seq_in_session": 20, # Multiple of MESSAGES_PER_SHORT_SUMMARY to trigger summary creation
"created_at": datetime.now(timezone.utc), # Required by create_payload
}
# Mock the CRUD function to track if it's called
# Also enable summary generation in settings
with (
patch("src.deriver.enqueue.crud.get_message_seq_in_session") as mock_crud,
patch("src.deriver.enqueue.settings.SUMMARY.ENABLED", new=True),
):
mock_crud.return_value = 200
mock_db_session = AsyncMock()
peers_config: dict[str, list[Any]] = {
test_peer.name: [
{"observe_me": True},
{"observe_others": True},
]
}
records = await generate_queue_records(
db_session=mock_db_session,
message=message_payload,
peers_with_configuration=peers_config,
session_id=test_session.id,
deriver_disabled=False,
)
mock_crud.assert_not_called()
assert len(records) > 0
summary_records = [r for r in records if r["task_type"] == "summary"]
assert len(summary_records) > 0, "Expected summary records to be created"
for record in summary_records:
assert (
record["payload"]["message_seq_in_session"]
!= mock_crud.return_value
)
assert record["payload"]["message_seq_in_session"] == 20
async def test_generate_queue_records_falls_back_to_crud_when_seq_missing(
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
):
"""
Test that generate_queue_records falls back to CRUD call
when seq_in_session is missing from payload.
This is the fallback behavior for backward compatibility.
"""
test_workspace, test_peer = sample_data
# Create a test session
test_session = models.Session(
workspace_name=test_workspace.name, name=str(generate_nanoid())
)
db_session.add(test_session)
await db_session.commit()
# Create a message payload WITHOUT seq_in_session
message_payload = {
"message_id": 12345,
"peer_name": test_peer.name,
"workspace_name": test_workspace.name,
"session_name": test_session.name,
"content": "Test message",
"created_at": datetime.now(timezone.utc),
# seq_in_session is MISSING
}
# Mock the CRUD function and enable summary generation in settings
with (
patch("src.deriver.enqueue.crud.get_message_seq_in_session") as mock_crud,
patch("src.deriver.enqueue.settings.SUMMARY.ENABLED", True),
):
mock_crud.return_value = (
60 # Multiple of MESSAGES_PER_LONG_SUMMARY to trigger summary creation
)
mock_db_session = AsyncMock()
peers_config: dict[str, list[Any]] = {
test_peer.name: [
{"observe_me": True},
{"observe_others": True},
]
}
records = await generate_queue_records(
db_session=mock_db_session,
message=message_payload,
peers_with_configuration=peers_config,
session_id=test_session.id,
deriver_disabled=False,
)
# The CRUD function SHOULD have been called as fallback
mock_crud.assert_called_once_with(
mock_db_session,
workspace_name=test_workspace.name,
session_name=test_session.name,
message_id=12345,
)
# Verify that records were created with the fallback value
summary_records = [r for r in records if r["task_type"] == "summary"]
assert len(summary_records) > 0, "Expected summary records to be created"
for record in summary_records:
# Should use the value from CRUD fallback (60)
assert record["payload"]["message_seq_in_session"] == 60

14
tests/integration/test_representation.py
View File

@ -222,8 +222,8 @@ class TestDocumentCreationWorkflow:
observed=observed_peer.name,
content="User likes dogs",
session_name="test_session",
level="explicit",
internal_metadata={
"level": "explicit",
"message_ids": [(1, 1)],
"session_name": "test_session",
},
@ -274,7 +274,9 @@ class TestDocumentCreationWorkflow:
observed=observed_peer.name,
content="Highly derived observation",
session_name=session.name,
internal_metadata={"level": "explicit", "times_derived": 5},
level="explicit",
times_derived=5,
internal_metadata={},
embedding=[0.1] * 1536,
)
db_session.add(highly_derived_doc)
@ -286,7 +288,9 @@ class TestDocumentCreationWorkflow:
observed=observed_peer.name,
content="Less derived observation",
session_name=session.name,
internal_metadata={"level": "explicit", "times_derived": 2},
level="explicit",
times_derived=2,
internal_metadata={},
embedding=[0.2] * 1536,
)
db_session.add(less_derived_doc)
@ -315,8 +319,8 @@ class TestDocumentCreationWorkflow:
observer="test_peer",
observed="test_peer",
content="User said they like programming",
level="explicit",
internal_metadata={
"level": "explicit",
"message_ids": [(1, 1)],
},
session_name="test_session",
@ -329,8 +333,8 @@ class TestDocumentCreationWorkflow:
observer="test_peer",
observed="test_peer",
content="User is likely a software developer",
level="deductive",
internal_metadata={
"level": "deductive",
"message_ids": [(1, 1)],
"premises": ["User said they like programming"],
},

11
tests/routes/test_queue_status.py
View File

@ -3,7 +3,7 @@ from fastapi.testclient import TestClient
from sqlalchemy.ext.asyncio import AsyncSession
from src import models
from src.utils.work_unit import get_work_unit_key
from src.utils.work_unit import construct_work_unit_key
@pytest.mark.asyncio
@ -157,9 +157,10 @@ class TestDeriverStatusEndpoint:
queue_item = models.QueueItem(
session_id=session.id,
task_type="representation",
work_unit_key=get_work_unit_key(payload),
work_unit_key=construct_work_unit_key(workspace.name, payload),
payload=payload,
processed=False,
workspace_name=workspace.name,
)
queue_items.append(queue_item)
db_session.add_all(queue_items)
@ -231,9 +232,10 @@ class TestDeriverStatusEndpoint:
queue_item = models.QueueItem(
session_id=session.id,
task_type="representation",
work_unit_key=get_work_unit_key(payload),
work_unit_key=construct_work_unit_key(workspace.name, payload),
payload=payload,
processed=False,
workspace_name=workspace.name,
)
queue_items.append(queue_item)
db_session.add_all(queue_items)
@ -292,9 +294,10 @@ class TestDeriverStatusEndpoint:
queue_item = models.QueueItem(
session_id=session.id,
task_type="representation",
work_unit_key=get_work_unit_key(payload),
work_unit_key=construct_work_unit_key(workspace.name, payload),
payload=payload,
processed=False,
workspace_name=workspace.name,
)
db_session.add(queue_item)
await db_session.commit()

7
tests/routes/test_webhooks.py
View File

@ -95,13 +95,6 @@ async def test_list_webhook_endpoints_with_data(
assert "http://example2.com/webhook" in endpoint_urls
@pytest.mark.asyncio
async def test_list_webhook_endpoints_missing_workspace(client: TestClient):
response = client.get("/v2/workspaces/nonexistent-workspace/webhooks")
assert response.status_code == 404
assert response.json() == {"detail": "Workspace nonexistent-workspace not found"}
@pytest.mark.asyncio
async def test_delete_webhook_endpoint(
client: TestClient, sample_data: tuple[Workspace, Peer]

7
tests/test_schema_validations.py
View File

@ -95,17 +95,18 @@ class TestDocumentValidations:
def test_valid_document_create(self):
metadata = DocumentMetadata(
message_ids=[(1, 1)],
level="explicit",
premises=[],
message_created_at="2021-01-01T00:00:00Z",
)
doc = DocumentCreate(
content="test content",
session_name="test",
level="explicit",
metadata=metadata,
embedding=[0.1, 0.2, 0.3],
)
assert doc.content == "test content"
assert doc.level == "explicit"
assert doc.metadata == metadata
def test_document_content_too_short(self):
@ -113,9 +114,9 @@ class TestDocumentValidations:
DocumentCreate(
content="",
session_name="test",
level="explicit",
metadata=DocumentMetadata(
message_ids=[(1, 1)],
level="explicit",
premises=[],
message_created_at="2021-01-01T00:00:00Z",
),
@ -129,9 +130,9 @@ class TestDocumentValidations:
DocumentCreate(
content="a" * 100001,
session_name="test",
level="explicit",
metadata=DocumentMetadata(
message_ids=[(1, 1)],
level="explicit",
premises=[],
message_created_at="2021-01-01T00:00:00Z",
),

66
uv.lock
View File

@ -70,6 +70,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" },
]
[[package]]
name = "async-timeout"
version = "5.0.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" },
]
[[package]]
name = "attrs"
version = "25.3.0"
@ -118,6 +127,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" },
]
[[package]]
name = "cashews"
version = "7.4.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/9e/02/6c63550c84263219367e027038ccdac9bff900775262027ac35b6de91973/cashews-7.4.1.tar.gz", hash = "sha256:9d4ac7b0d0e20ec96680af60ae15dc26c19ccc267baa84a472c54bef86a93a8a", size = 91757, upload-time = "2025-07-14T22:39:48.137Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/e6/e77b27292b560725c35e478e90bdc9fe84c6ec849daac4360b4150083b4f/cashews-7.4.1-py3-none-any.whl", hash = "sha256:868019e9c8b0a75f345ea58b71197640b20dc2fe0892eb5ef6537f5652299ba4", size = 79356, upload-time = "2025-07-14T22:39:46.719Z" },
]
[package.optional-dependencies]
redis = [
{ name = "redis" },
]
[[package]]
name = "certifi"
version = "2025.8.3"
@ -464,6 +487,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },
]
[[package]]
name = "fakeredis"
version = "2.32.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "redis" },
{ name = "sortedcontainers" },
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e1/2e/94ca3f2ff35f086d7d3eeb924054e328b2ac851f0a20302d942c8d29726c/fakeredis-2.32.0.tar.gz", hash = "sha256:63d745b40eb6c8be4899cf2a53187c097ccca3afbca04fdbc5edc8b936cd1d59", size = 171097, upload-time = "2025-10-07T10:46:58.876Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/0e/1b/84ab7fd197eba5243b6625c78fbcffaa4cf6ac7dda42f95d22165f52187e/fakeredis-2.32.0-py3-none-any.whl", hash = "sha256:c9da8228de84060cfdb72c3cf4555c18c59ba7a5ae4d273f75e4822d6f01ecf8", size = 118422, upload-time = "2025-10-07T10:46:57.643Z" },
]
[[package]]
name = "fastapi"
version = "0.116.1"
@ -673,10 +710,11 @@ wheels = [
[[package]]
name = "honcho"
version = "2.4.1"
version = "2.4.2"
source = { virtual = "." }
dependencies = [
{ name = "alembic" },
{ name = "cashews", extra = ["redis"] },
{ name = "fastapi", extra = ["standard"] },
{ name = "fastapi-pagination" },
{ name = "google-genai" },
@ -695,6 +733,7 @@ dependencies = [
{ name = "pydantic-settings" },
{ name = "pyjwt" },
{ name = "python-dotenv" },
{ name = "redis" },
{ name = "rich" },
{ name = "sentry-sdk", extra = ["anthropic", "fastapi", "sqlalchemy"] },
{ name = "sqlalchemy" },
@ -707,6 +746,7 @@ dependencies = [
dev = [
{ name = "basedpyright" },
{ name = "coverage" },
{ name = "fakeredis" },
{ name = "honcho-ai" },
{ name = "interrogate" },
{ name = "pre-commit" },
@ -721,6 +761,7 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "alembic", specifier = ">=1.14.0" },
{ name = "cashews", extras = ["redis"], specifier = "==7.4.1" },
{ name = "fastapi", extras = ["standard"], specifier = ">=0.111.0" },
{ name = "fastapi-pagination", specifier = ">=0.12.24" },
{ name = "google-genai", specifier = ">=1.32.0" },
@ -739,6 +780,7 @@ requires-dist = [
{ name = "pydantic-settings", specifier = ">=2.10.1" },
{ name = "pyjwt", specifier = ">=2.10.0" },
{ name = "python-dotenv", specifier = ">=1.0.0" },
{ name = "redis", specifier = ">=6.0.0" },
{ name = "rich", specifier = ">=13.7.1" },
{ name = "sentry-sdk", extras = ["anthropic", "fastapi", "sqlalchemy"], specifier = ">=2.3.1" },
{ name = "sqlalchemy", specifier = ">=2.0.30" },
@ -751,6 +793,7 @@ requires-dist = [
dev = [
{ name = "basedpyright", specifier = ">=1.29.4" },
{ name = "coverage", specifier = ">=7.6.0" },
{ name = "fakeredis", specifier = ">=2.32.0" },
{ name = "honcho-ai", editable = "sdks/python" },
{ name = "interrogate", specifier = ">=1.7.0" },
{ name = "pre-commit", specifier = ">=4.2.0" },
@ -1967,6 +2010,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },
]
[[package]]
name = "redis"
version = "6.4.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "async-timeout", marker = "python_full_version < '3.11.3'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" },
]
[[package]]
name = "regex"
version = "2025.7.34"
@ -2261,6 +2316,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
]
[[package]]
name = "sortedcontainers"
version = "2.4.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" },
]
[[package]]
name = "sqlalchemy"
version = "2.0.42"