diff --git a/.env.template b/.env.template index 5879b5ed..eef1f096 100644 --- a/.env.template +++ b/.env.template @@ -21,6 +21,8 @@ LOG_LEVEL=INFO # LANGFUSE_HOST= # LANGFUSE_PUBLIC_KEY= +# NAMESPACE="honcho" + # ============================================================================= # Database Settings (REQUIRED) # ============================================================================= @@ -83,6 +85,7 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 # DERIVER_PROVIDER=google # DERIVER_MODEL=gemini-2.0-flash-lite +# DERIVER_DEDUPLICATE=true # DERIVER_MAX_OUTPUT_TOKENS=2500 # only applied when using Anthropic as provider # DERIVER_THINKING_BUDGET_TOKENS=1024 @@ -166,5 +169,14 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # ============================================================================= # Metrics (Optional) # ============================================================================= -# ENABLED=false -# NAMESPACE=honcho +# METRICS_ENABLED=false +# METRICS_NAMESPACE=honcho + +# ============================================================================= +# Cache +# ============================================================================= +# CACHE_ENABLED=false +# CACHE_URL="redis://localhost:6379/0" +# CACHE_NAMESPACE="honcho" +# CACHE_DEFAULT_TTL_SECONDS=300 +# CACHE_DEFAULT_LOCK_TTL_SECONDS=5 diff --git a/.gitignore b/.gitignore index f75448c4..17d9597c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ api/docker-compose.yml *.db data +redis-data docker-compose.yml compose.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7cc0de63..6d5a3f6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,11 +74,11 @@ repos: # Run Alembic tests only when migrations change - id: pytest-alembic name: pytest (alembic migrations) - entry: uv run pytest tests/alembic/ + entry: uv run python scripts/run_alembic_tests.py language: system - files: ^(migrations/.*\.py|tests/alembic/.*\.py)$ + files: ^(migrations/versions/.*\.py|tests/alembic/.*\.py)$ stages: [pre-push] - pass_filenames: false + pass_filenames: true require_serial: true # Ensure each alembic migration revision has a corresponding test file diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fdd2e90..68b7e316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [2.4.2] - 2025-11-03 + +### Fixed + +- Langfuse tracing to have readable waterfalls +- Alembic Migrations to match models.py +- message_in_seq correctly included in webhook payload + +### Changed + +- Alembic to always use a session pooler +- Statement timeout during alembic operations to 5 min + ## [2.4.1] - 2025-10-24 ### Added diff --git a/README.md b/README.md index 452f487b..b83733f9 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ --- -![Static Badge](https://img.shields.io/badge/Version-2.4.1-blue) +![Static Badge](https://img.shields.io/badge/Version-2.4.2-blue) [![PyPI version](https://img.shields.io/pypi/v/honcho-ai.svg)](https://pypi.org/project/honcho-ai/) [![NPM version](https://img.shields.io/npm/v/@honcho-ai/sdk.svg)](https://npmjs.org/package/@honcho-ai/sdk) [![Discord](https://img.shields.io/discord/1016845111637839922?style=flat&logo=discord&logoColor=23ffffff&label=Plastic%20Labs&labelColor=235865F2)](https://discord.gg/plasticlabs) @@ -396,13 +396,18 @@ cp config.toml.example config.toml Then modify the values as needed. The TOML file is organized into sections: -- `[app]` - Application-level settings (log level, host, port, embedding settings) +- `[app]` - Application-level settings (log level, session limits, embedding settings, metrics) - `[db]` - Database connection and pool settings - `[auth]` - Authentication configuration +- `[cache]` - Redis cache configuration - `[llm]` - LLM provider API keys and general settings - `[dialectic]` - Dialectic API configuration (provider, model, search settings) - `[deriver]` - Background worker settings and theory of mind configuration +- `[peer_card]` - Peer card generation settings - `[summary]` - Session summarization settings +- `[dream]` - Dream processing configuration +- `[webhook]` - Webhook configuration +- `[metrics]` - Metrics collection settings - `[sentry]` - Error tracking and monitoring settings ### Using Environment Variables diff --git a/config.toml.example b/config.toml.example index dddd8690..8ef1b5e1 100644 --- a/config.toml.example +++ b/config.toml.example @@ -15,7 +15,7 @@ MAX_EMBEDDING_TOKENS = 8192 MAX_EMBEDDING_TOKENS_PER_REQUEST = 300000 # LANGFUSE_HOST = "https://api.langfuse.com" # LANGFUSE_PUBLIC_KEY = "your-public-key-here" - +NAMESPACE="honcho" # Database settings [db] @@ -64,6 +64,7 @@ POLLING_SLEEP_INTERVAL_SECONDS = 1.0 STALE_SESSION_TIMEOUT_MINUTES = 5 PROVIDER = "google" MODEL = "gemini-2.5-flash-lite" +DEDUPLICATE = true MAX_OUTPUT_TOKENS = 2500 THINKING_BUDGET_TOKENS = 1024 # only applied when using Anthropic # BACKUP_PROVIDER = "google" @@ -132,3 +133,11 @@ MAX_WORKSPACE_LIMIT = 10 [metrics] ENABLED = false NAMESPACE = "honcho" + +# Cache settings +[cache] +ENABLED = false +URL = "redis://localhost:6379/0" +NAMESPACE="honcho" +DEFAULT_TTL_SECONDS = 300 +DEFAULT_LOCK_TTL_SECONDS = 5 diff --git a/docker-compose.yml.example b/docker-compose.yml.example index e11e93ad..123ec5c5 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -45,3 +45,15 @@ services: interval: 5s timeout: 5s retries: 5 + redis: + image: redis:8.2 + restart: always + ports: + - 6379:6379 + volumes: + - ./redis-data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli ping"] + interval: 5s + timeout: 5s + retries: 5 diff --git a/docs/changelog/compatibility-guide.mdx b/docs/changelog/compatibility-guide.mdx index edcc8fdd..883ee2d6 100644 --- a/docs/changelog/compatibility-guide.mdx +++ b/docs/changelog/compatibility-guide.mdx @@ -8,7 +8,7 @@ This guide helps you understand which versions of Honcho's API are compatible wi ## Version Compatibility -### Honcho API v2.4.1 (Current) +### Honcho API v2.4.2 (Current) @@ -30,12 +30,12 @@ This guide helps you understand which versions of Honcho's API are compatible wi - ## Version Compatibility Table | Honcho API Version | TypeScript SDK | Python SDK | |-------------------|---------------|------------| -| v2.4.1 (Current) | v1.5.0 | v1.5.0 | +| v2.4.2 (Current) | v1.5.0 | v1.5.0 | +| v2.4.1 | v1.5.0 | v1.5.0 | | v2.4.0 | v1.5.0 | v1.5.0 | | v2.3.3 | v1.4.1 | v1.4.1 | | v2.3.2 | v1.4.0 | v1.4.0 | diff --git a/docs/changelog/introduction.mdx b/docs/changelog/introduction.mdx index 4edab9dc..ff9946a5 100644 --- a/docs/changelog/introduction.mdx +++ b/docs/changelog/introduction.mdx @@ -27,7 +27,21 @@ Welcome to the Honcho changelog! This section documents all notable changes to t ### Honcho API and SDK Changelogs - + + ### Fixed + + - Langfuse tracing to have readable waterfalls + - Alembic Migrations to match models.py + - message_in_seq correctly included in webhook payload + + + ### Changed + + - Alembic to always use a session pooler + - Statement timeout during alembic operations to 5 min + + + ### Added - Alembic migration validation test suite diff --git a/docs/docs.json b/docs/docs.json index 4a9a6412..323fca93 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -3,27 +3,20 @@ "theme": "mint", "name": "Honcho", "colors": { - "primary": "#86BCF2", + "primary": "#66AAFF", "dark": "#151E27", - "light": "#B5D9FD" + "light": "#86BCF2" }, "favicon": "/favicon.svg", "contextual": { - "options": [ - "copy", - "view", - "chatgpt", - "claude" - ] + "options": ["copy", "view", "chatgpt", "claude"] }, "navigation": { "versions": [ { - "version": "v2.4.1", + "version": "v2.4.2", "api": { - "openapi": [ - "openapi.documented.yml" - ] + "openapi": ["openapi.documented.yml"] }, "tabs": [ { @@ -41,10 +34,19 @@ "group": "Core Concepts", "pages": [ "v2/documentation/core-concepts/architecture", - "v2/documentation/core-concepts/glossary", - "v2/documentation/core-concepts/features", + "v2/documentation/core-concepts/features/storing-data", + "v2/documentation/core-concepts/features/dialectic-endpoint", + "v2/documentation/core-concepts/features/get-context", + "v2/documentation/core-concepts/features/search", + "v2/documentation/core-concepts/features/working-rep", + "v2/documentation/core-concepts/features/streaming-response", + "v2/documentation/core-concepts/features/using-filters", + "v2/documentation/core-concepts/features/file-uploads", + "v2/documentation/core-concepts/features/queue-status", + "v2/documentation/core-concepts/features/local-vs-global", "v2/documentation/core-concepts/configuration", - "v2/documentation/core-concepts/summarizer" + "v2/documentation/core-concepts/summarizer", + "v2/documentation/core-concepts/glossary" ] }, { @@ -61,54 +63,21 @@ "groups": [ { "group": "Getting Started", - "pages": [ - "v2/guides/overview", - "v2/guides/mcp" - ] + "pages": ["v2/guides/overview", "v2/guides/mcp"] }, { "group": "Application Interfaces", - "pages": [ - "v2/guides/discord", - "v2/guides/telegram" - ] - }, - { - "group": "Design Patterns", - "pages": [ - "v2/guides/dialectic-endpoint", - "v2/guides/get-context", - "v2/guides/search", - "v2/guides/working-rep", - "v2/guides/streaming-response", - "v2/guides/using-filters", - "v2/guides/file-uploads" - ] - } - ] - }, - { - "tab": "Contributing", - "groups": [ - { - "group": "Contributing", - "pages": [ - "v2/contributing/guidelines", - "v2/contributing/self-hosting", - "v2/contributing/configuration", - "v2/contributing/license" - ] + "pages": ["v2/guides/discord", "v2/guides/telegram"] } ] }, + { "tab": "API Reference", "groups": [ { "group": "API Documentation", - "pages": [ - "v2/api-reference/introduction" - ] + "pages": ["v2/api-reference/introduction"] }, { "group": "workspaces", @@ -192,15 +161,27 @@ ] } ] + }, + { + "tab": "Contributing", + "groups": [ + { + "group": "Contributing", + "pages": [ + "v2/contributing/guidelines", + "v2/contributing/self-hosting", + "v2/contributing/configuration", + "v2/contributing/license" + ] + } + ] } ] }, { "version": "v1.1.0", "api": { - "openapi": [ - "openapi.json" - ] + "openapi": ["openapi.json"] }, "tabs": [ { @@ -230,23 +211,15 @@ "groups": [ { "group": "Getting Started", - "pages": [ - "v1/guides/overview", - "v1/guides/streaming-response" - ] + "pages": ["v1/guides/overview", "v1/guides/streaming-response"] }, { "group": "Application Interfaces", - "pages": [ - "v1/guides/discord", - "v1/guides/honcho-mcp" - ] + "pages": ["v1/guides/discord", "v1/guides/honcho-mcp"] }, { "group": "Personal Memory", - "pages": [ - "v1/guides/dialectic-endpoint" - ] + "pages": ["v1/guides/dialectic-endpoint"] } ] }, @@ -255,9 +228,7 @@ "groups": [ { "group": "API Documentation", - "pages": [ - "v1/api-reference/introduction" - ] + "pages": ["v1/api-reference/introduction"] }, { "group": "apps", @@ -305,9 +276,7 @@ }, { "group": "keys", - "pages": [ - "v1/api-reference/endpoint/keys/create-key" - ] + "pages": ["v1/api-reference/endpoint/keys/create-key"] }, { "group": "metamessages", @@ -360,9 +329,9 @@ "global": { "anchors": [ { - "anchor": "Managed Platform", + "anchor": "Dashboard", "href": "https://app.honcho.dev", - "icon": "book-open-cover" + "icon": "table-columns" }, { "anchor": "Community", diff --git a/docs/images/overview/honcho-overview-dark.svg b/docs/images/overview/honcho-overview-dark.svg new file mode 100644 index 00000000..753eb87b --- /dev/null +++ b/docs/images/overview/honcho-overview-dark.svg @@ -0,0 +1,4 @@ + + +Store Messages inHonchoIngestionHoncho Store Messages in LongTerm MemoryLong Term Memory StoreSends Message toReasoning EngineHoncho APIHonchoReasoningEngineMessagesPeer CardsSummariesRepresentationsSends Derived Insights to LongTerm MemoryMessagesHoncho APIGet ContextChatSearchWorking RepresentationsRetreival diff --git a/docs/images/overview/honcho-overview-light.png b/docs/images/overview/honcho-overview-light.png new file mode 100644 index 00000000..3a5b50e5 Binary files /dev/null and b/docs/images/overview/honcho-overview-light.png differ diff --git a/docs/images/overview/honcho-overview-light.svg b/docs/images/overview/honcho-overview-light.svg new file mode 100644 index 00000000..704f7ec9 --- /dev/null +++ b/docs/images/overview/honcho-overview-light.svg @@ -0,0 +1,4 @@ + + +Store Messages inHonchoIngestionHoncho Store Messages in LongTerm MemoryLong Term Memory StoreSends Message toReasoning EngineHoncho APIHonchoReasoningEngineMessagesPeer CardsSummariesRepresentationsSends Derived Insights to LongTerm MemoryMessagesHoncho APIGet ContextChatSearchWorking RepresentationsRetreival diff --git a/docs/images/overview/honcho-overview-no-background.png b/docs/images/overview/honcho-overview-no-background.png new file mode 100644 index 00000000..320ee0d8 Binary files /dev/null and b/docs/images/overview/honcho-overview-no-background.png differ diff --git a/docs/images/overview/honcho-overview.svg b/docs/images/overview/honcho-overview.svg new file mode 100644 index 00000000..a861b327 --- /dev/null +++ b/docs/images/overview/honcho-overview.svg @@ -0,0 +1,4 @@ + + +Store Messages inHonchoHoncho StoreMessages in LongTerm MemoryLong Term Memory StoreSends Message toReasoning EngineHoncho APIHonchoReasoningEngineMessagesPeer CardsSummariesRepresentationsSends DerivedInsights to LongTerm MemoryMessagesHoncho APIGet ContextChatSearchWorking Representations diff --git a/docs/_snippets/overview-shields.mdx b/docs/snippets/overview-shields.mdx similarity index 100% rename from docs/_snippets/overview-shields.mdx rename to docs/snippets/overview-shields.mdx diff --git a/docs/v2/contributing/configuration.mdx b/docs/v2/contributing/configuration.mdx index da5f2d6a..1233f511 100644 --- a/docs/v2/contributing/configuration.mdx +++ b/docs/v2/contributing/configuration.mdx @@ -46,14 +46,19 @@ cp config.toml.example config.toml Then modify the values as needed. The TOML file is organized into sections: -- `[app]` - Application-level settings (log level, host, port, embedding settings) -- `[db]` - Database connection and pool settings -- `[auth]` - Authentication configuration -- `[llm]` - LLM provider API keys and general settings -- `[dialectic]` - Dialectic API configuration (provider, model, search settings) -- `[deriver]` - Background worker settings and theory of mind configuration -- `[summary]` - Session summarization settings -- `[sentry]` - Error tracking and monitoring settings +- `[app]` - Application-level settings (log level, session limits, embedding settings, Langfuse integration, local metrics collection) +- `[db]` - Database connection and pool settings (connection URI, pool size, timeouts, connection recycling) +- `[auth]` - Authentication configuration (enable/disable auth, JWT secret) +- `[cache]` - Redis cache configuration (enable/disable caching, Redis URL, TTL settings, lock configuration for cache stampede prevention) +- `[llm]` - LLM provider API keys (Anthropic, OpenAI, Gemini, Groq, OpenAI-compatible endpoints) and general LLM settings +- `[dialectic]` - Dialectic API configuration (provider, model, query generation settings, semantic search parameters, context window size) +- `[deriver]` - Background worker settings (worker count, polling intervals, queue management) and theory of mind configuration (model, tokens, observation limits) +- `[peer_card]` - Peer card generation settings (provider, model, token limits) +- `[summary]` - Session summarization settings (frequency thresholds, provider, model, token limits for short and long summaries) +- `[dream]` - Dream processing configuration (enable/disable, thresholds, idle timeouts, dream types, LLM settings) +- `[webhook]` - Webhook configuration (webhook secret, workspace limits) +- `[metrics]` - Metrics collection settings (enable/disable metrics, namespace) +- `[sentry]` - Error tracking and monitoring settings (enable/disable, DSN, environment, sample rates) ### Using Environment Variables @@ -107,28 +112,33 @@ The application will use the production connection URI while keeping the pool si ### Application Settings +Application-level settings control core behavior of the Honcho server including logging, session limits, message handling, and optional integrations. + **Basic Application Configuration:** ```bash # Logging and server settings -LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR -SESSION_PEERS_LIMIT=10 -GET_CONTEXT_MAX_TOKENS=100000 +LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR, CRITICAL -# Embedding settings (optional) -EMBED_MESSAGES=false -MAX_EMBEDDING_TOKENS=8192 -MAX_EMBEDDING_TOKENS_PER_REQUEST=300000 +# Session and context limits +SESSION_OBSERVERS_LIMIT=10 # Maximum number of observers per session +GET_CONTEXT_MAX_TOKENS=100000 # Maximum tokens for context retrieval +MAX_MESSAGE_SIZE=25000 # Maximum message size in characters + +# Embedding settings +EMBED_MESSAGES=true # Enable vector embeddings for messages +MAX_EMBEDDING_TOKENS=8192 # Maximum tokens per embedding +MAX_EMBEDDING_TOKENS_PER_REQUEST=300000 # Batch embedding limit ``` -**Environment-specific settings:** +**Optional Integrations:** ```bash -# Development -LOG_LEVEL=DEBUG -FASTAPI_HOST=127.0.0.1 +# Langfuse integration for LLM observability +LANGFUSE_HOST=https://cloud.langfuse.com +LANGFUSE_PUBLIC_KEY=your-langfuse-public-key -# Production -LOG_LEVEL=WARNING -FASTAPI_HOST=0.0.0.0 +# Local metrics collection +COLLECT_METRICS_LOCAL=false +LOCAL_METRICS_FILE=metrics.jsonl ``` ### Database Configuration @@ -196,6 +206,36 @@ AUTH_JWT_SECRET=your-super-secret-jwt-key python scripts/generate_jwt_secret.py ``` +### Cache Configuration + +Honcho supports Redis caching to improve performance by caching frequently accessed data like peers, sessions, and working representations. Caching also includes lock mechanisms to prevent cache stampede scenarios. + +**Redis Cache Settings:** +```bash +# Enable/disable Redis caching +CACHE_ENABLED=false # Set to true to enable caching + +# Redis connection +CACHE_URL=redis://localhost:6379/0?suppress=false + +# Cache namespace and TTL +CACHE_NAMESPACE=honcho # Prefix for all cache keys +CACHE_DEFAULT_TTL_SECONDS=300 # How long items stay in cache (5 minutes) + +# Lock settings for preventing cache stampede +CACHE_DEFAULT_LOCK_TTL_SECONDS=5 # Lock duration when fetching from DB on cache miss +``` + +**When to Enable Caching:** +- High-traffic production environments +- Applications with many repeated reads of the same data +- When you need to reduce database load + +**Note:** Caching requires a Redis instance. You can run Redis locally with Docker: +```bash +docker run -d -p 6379:6379 redis:latest +``` + ## LLM Provider Configuration Honcho supports multiple LLM providers for different tasks. API keys are configured in the `[llm]` section, while specific features use their own configuration sections. @@ -221,6 +261,9 @@ LLM_OPENAI_COMPATIBLE_BASE_URL=https://your-openai-compatible-endpoint.com ```bash # Default settings for all LLM calls LLM_DEFAULT_MAX_TOKENS=2500 + +# Embedding provider (used when EMBED_MESSAGES=true) +LLM_EMBEDDING_PROVIDER=openai # Options: openai, gemini ``` ### Feature-Specific Model Configuration @@ -228,67 +271,147 @@ LLM_DEFAULT_MAX_TOKENS=2500 Different features can use different providers and models: **Dialectic API:** + +The Dialectic API provides theory-of-mind informed responses by integrating long-term facts with current context. + ```bash # Main dialectic model (default: Anthropic) DIALECTIC_PROVIDER=anthropic DIALECTIC_MODEL=claude-sonnet-4-20250514 DIALECTIC_MAX_OUTPUT_TOKENS=2500 -DIALECTIC_THINKING_BUDGET_TOKENS=1024 +DIALECTIC_THINKING_BUDGET_TOKENS=1024 # Only used with Anthropic provider +DIALECTIC_CONTEXT_WINDOW_SIZE=100000 # Maximum context window tokens -# Query generation for dialectic (default: Groq) +# Query generation for dialectic searches +DIALECTIC_PERFORM_QUERY_GENERATION=false # Enable query generation for semantic search DIALECTIC_QUERY_GENERATION_PROVIDER=groq DIALECTIC_QUERY_GENERATION_MODEL=llama-3.1-8b-instant # Semantic search settings -DIALECTIC_SEMANTIC_SEARCH_TOP_K=10 -DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE=0.85 +DIALECTIC_SEMANTIC_SEARCH_TOP_K=10 # Number of results to retrieve +DIALECTIC_SEMANTIC_SEARCH_MAX_DISTANCE=0.85 # Maximum distance for relevance ``` -**Deriver:** +**Deriver (Theory of Mind):** + +The Deriver is a background processing system that extracts facts from messages and builds theory-of-mind representations of peers. + ```bash -# Deriver model (default: Google) +# LLM settings for deriver DERIVER_PROVIDER=google -DERIVER_MODEL=gemini-2.0-flash-lite +DERIVER_MODEL=gemini-2.5-flash-lite +DERIVER_MAX_OUTPUT_TOKENS=10000 +DERIVER_THINKING_BUDGET_TOKENS=1024 # Only used with Anthropic provider +DERIVER_MAX_INPUT_TOKENS=23000 # Maximum input tokens for deriver # Worker settings -DERIVER_WORKERS=1 -DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 -DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0 +DERIVER_WORKERS=1 # Number of background worker processes +DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0 # Time between queue checks +DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 # Timeout for stale sessions -# Peer card settings -DERIVER_PEER_CARD_PROVIDER=openai -DERIVER_PEER_CARD_MODEL=gpt-5-nano-2025-08-07 -DERIVER_PEER_CARD_MAX_OUTPUT_TOKENS=2000 +# Queue management +DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000 # Keep errored items for 30 days -# Maximum number of observations to store in working representation -# This is applied to both explicit and deductive observations -DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100 +# Working representation settings +DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=50 # Max observations stored +DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=4096 # Max tokens per batch +``` + +**Peer Card:** + +Peer cards are short, structured summaries of peer identity and characteristics. + +```bash +# Enable/disable peer card generation +PEER_CARD_ENABLED=true + +# LLM settings for peer card generation +PEER_CARD_PROVIDER=openai +PEER_CARD_MODEL=gpt-5-nano-2025-08-07 +PEER_CARD_MAX_OUTPUT_TOKENS=4000 # Includes thinking tokens for GPT-5 models ``` **Summary Generation:** -```bash -# Summary model (default: Google) -SUMMARY_PROVIDER=google -SUMMARY_MODEL=gemini-1.5-flash-latest -SUMMARY_MAX_TOKENS_SHORT=1000 -SUMMARY_MAX_TOKENS_LONG=2000 -SUMMARY_THINKING_BUDGET_TOKENS=512 -# Summary frequency -SUMMARY_MESSAGES_PER_SHORT_SUMMARY=20 -SUMMARY_MESSAGES_PER_LONG_SUMMARY=60 +Session summaries provide compressed context for long conversations. Honcho creates two types: short summaries (frequent) and long summaries (comprehensive). + +```bash +# Enable/disable summarization +SUMMARY_ENABLED=true + +# LLM settings for summary generation +SUMMARY_PROVIDER=openai +SUMMARY_MODEL=gpt-4o-mini-2024-07-18 +SUMMARY_MAX_TOKENS_SHORT=1000 # Max tokens for short summaries +SUMMARY_MAX_TOKENS_LONG=4000 # Max tokens for long summaries +SUMMARY_THINKING_BUDGET_TOKENS=512 # Only used with Anthropic provider + +# Summary frequency thresholds +SUMMARY_MESSAGES_PER_SHORT_SUMMARY=20 # Create short summary every N messages +SUMMARY_MESSAGES_PER_LONG_SUMMARY=60 # Create long summary every N messages ``` ### Default Provider Usage By default, Honcho uses: -- **Anthropic** for dialectic API responses -- **Groq** for query generation -- **Google** for deriving theory of mind and summarization +- **Anthropic** (Claude) for dialectic API responses +- **Groq** for query generation (fast, cost-effective) +- **Google** (Gemini) for theory of mind derivation +- **OpenAI** (GPT) for peer cards and summarization - **OpenAI** for embeddings (if `EMBED_MESSAGES=true`) -You only need to set the API keys for the providers you plan to use. +You only need to set the API keys for the providers you plan to use. All providers are configurable per feature. +## Additional Features Configuration + +### Dream Processing + +Dream processing consolidates and refines peer representations during idle periods, similar to how human memory consolidation works during sleep. + +**Dream Settings:** +```bash +# Enable/disable dream processing +DREAM_ENABLED=true + +# Trigger thresholds +DREAM_DOCUMENT_THRESHOLD=50 # Minimum documents to trigger a dream +DREAM_IDLE_TIMEOUT_MINUTES=60 # Minutes of inactivity before dream can start +DREAM_MIN_HOURS_BETWEEN_DREAMS=8 # Minimum hours between dreams for a peer + +# Dream types to enable +DREAM_ENABLED_TYPES=["consolidate"] # Currently supported: consolidate + +# LLM settings for dream processing +DREAM_PROVIDER=openai +DREAM_MODEL=gpt-4o-mini-2024-07-18 +DREAM_MAX_OUTPUT_TOKENS=2000 +``` + +### Webhook Configuration + +Webhooks allow you to receive real-time notifications when events occur in Honcho (e.g., new messages, session updates). + +**Webhook Settings:** +```bash +# Webhook secret for signing payloads (optional but recommended) +WEBHOOK_SECRET=your-webhook-signing-secret + +# Limit on webhooks per workspace +WEBHOOK_MAX_WORKSPACE_LIMIT=10 +``` + +### Metrics Collection + +Enable metrics collection for monitoring Honcho performance and usage. + +**Metrics Settings:** +```bash +# Enable/disable metrics collection +METRICS_ENABLED=false + +# Namespace for metrics (used in metric names) +METRICS_NAMESPACE=honcho +``` ## Monitoring Configuration @@ -296,13 +419,17 @@ You only need to set the API keys for the providers you plan to use. **Sentry Settings:** ```bash -# Enable/disable Sentry +# Enable/disable Sentry error tracking SENTRY_ENABLED=false # Sentry configuration SENTRY_DSN=https://your-sentry-dsn@sentry.io/project-id -SENTRY_TRACES_SAMPLE_RATE=0.1 -SENTRY_PROFILES_SAMPLE_RATE=0.1 +SENTRY_RELEASE=2.4.0 # Optional: track which version errors come from +SENTRY_ENVIRONMENT=production # Environment name (development, staging, production) + +# Sampling rates (0.0 to 1.0) +SENTRY_TRACES_SAMPLE_RATE=0.1 # 10% of transactions tracked +SENTRY_PROFILES_SAMPLE_RATE=0.1 # 10% of transactions profiled ``` ## Environment-Specific Examples @@ -313,7 +440,7 @@ SENTRY_PROFILES_SAMPLE_RATE=0.1 ```toml [app] LOG_LEVEL = "DEBUG" -SESSION_PEERS_LIMIT = 10 +SESSION_OBSERVERS_LIMIT = 10 EMBED_MESSAGES = false [db] @@ -323,21 +450,40 @@ POOL_SIZE = 5 [auth] USE_AUTH = false +[cache] +ENABLED = false + [dialectic] PROVIDER = "anthropic" MODEL = "claude-sonnet-4-20250514" -QUERY_GENERATION_PROVIDER = "groq" -QUERY_GENERATION_MODEL = "llama-3.1-8b-instant" +PERFORM_QUERY_GENERATION = false MAX_OUTPUT_TOKENS = 2500 -[summary] -PROVIDER = "google" -MODEL = "gemini-1.5-flash-latest" -MAX_TOKENS_SHORT = 1000 -MAX_TOKENS_LONG = 2000 - [deriver] WORKERS = 1 +PROVIDER = "google" +MODEL = "gemini-2.5-flash-lite" + +[peer_card] +ENABLED = true +PROVIDER = "openai" +MODEL = "gpt-5-nano-2025-08-07" + +[summary] +ENABLED = true +PROVIDER = "openai" +MODEL = "gpt-4o-mini-2024-07-18" +MAX_TOKENS_SHORT = 1000 +MAX_TOKENS_LONG = 4000 + +[dream] +ENABLED = true + +[webhook] +MAX_WORKSPACE_LIMIT = 10 + +[metrics] +ENABLED = false [sentry] ENABLED = false @@ -349,7 +495,12 @@ ENABLED = false LOG_LEVEL=DEBUG DB_CONNECTION_URI=postgresql+psycopg://postgres:postgres@localhost:5432/honcho_dev AUTH_USE_AUTH=false -ANTHROPIC_API_KEY=your-dev-anthropic-key +CACHE_ENABLED=false + +# LLM Provider API Keys +LLM_ANTHROPIC_API_KEY=your-dev-anthropic-key +LLM_OPENAI_API_KEY=your-dev-openai-key +LLM_GEMINI_API_KEY=your-dev-gemini-key ``` ### Production Configuration @@ -358,7 +509,7 @@ ANTHROPIC_API_KEY=your-dev-anthropic-key ```toml [app] LOG_LEVEL = "WARNING" -SESSION_PEERS_LIMIT = 10 +SESSION_OBSERVERS_LIMIT = 10 EMBED_MESSAGES = true [db] @@ -369,27 +520,50 @@ MAX_OVERFLOW = 40 [auth] USE_AUTH = true +[cache] +ENABLED = true +URL = "redis://redis:6379/0" +DEFAULT_TTL_SECONDS = 300 + [dialectic] PROVIDER = "anthropic" MODEL = "claude-sonnet-4-20250514" -QUERY_GENERATION_PROVIDER = "groq" -QUERY_GENERATION_MODEL = "llama-3.1-8b-instant" +PERFORM_QUERY_GENERATION = false MAX_OUTPUT_TOKENS = 2500 -[summary] -PROVIDER = "google" -MODEL = "gemini-1.5-flash-latest" -MAX_TOKENS_SHORT = 1000 -MAX_TOKENS_LONG = 2000 - [deriver] WORKERS = 4 PROVIDER = "google" -MODEL = "gemini-2.0-flash-lite" +MODEL = "gemini-2.5-flash-lite" + +[peer_card] +ENABLED = true +PROVIDER = "openai" +MODEL = "gpt-5-nano-2025-08-07" + +[summary] +ENABLED = true +PROVIDER = "openai" +MODEL = "gpt-4o-mini-2024-07-18" +MAX_TOKENS_SHORT = 1000 +MAX_TOKENS_LONG = 4000 + +[dream] +ENABLED = true +PROVIDER = "openai" +MODEL = "gpt-4o-mini-2024-07-18" + +[webhook] +MAX_WORKSPACE_LIMIT = 10 + +[metrics] +ENABLED = true [sentry] ENABLED = true +ENVIRONMENT = "production" TRACES_SAMPLE_RATE = 0.1 +PROFILES_SAMPLE_RATE = 0.1 ``` **Environment variables for production:** @@ -397,11 +571,27 @@ TRACES_SAMPLE_RATE = 0.1 # .env.production LOG_LEVEL=WARNING DB_CONNECTION_URI=postgresql+psycopg://honcho_user:secure_password@prod-db:5432/honcho_prod + +# Authentication AUTH_USE_AUTH=true AUTH_JWT_SECRET=your-super-secret-jwt-key -ANTHROPIC_API_KEY=your-prod-anthropic-key -GEMINI_API_KEY=your-prod-gemini-key + +# Cache +CACHE_ENABLED=true +CACHE_URL=redis://redis:6379/0 + +# LLM Provider API Keys +LLM_ANTHROPIC_API_KEY=your-prod-anthropic-key +LLM_OPENAI_API_KEY=your-prod-openai-key +LLM_GEMINI_API_KEY=your-prod-gemini-key +LLM_GROQ_API_KEY=your-prod-groq-key + +# Webhooks +WEBHOOK_SECRET=your-webhook-signing-secret + +# Monitoring SENTRY_DSN=https://your-sentry-dsn@sentry.io/project-id +SENTRY_ENVIRONMENT=production ``` ## Migration Management diff --git a/docs/v2/documentation/core-concepts/architecture.mdx b/docs/v2/documentation/core-concepts/architecture.mdx index 5980d90b..3a58147d 100644 --- a/docs/v2/documentation/core-concepts/architecture.mdx +++ b/docs/v2/documentation/core-concepts/architecture.mdx @@ -7,17 +7,15 @@ sidebarTitle: "Architecture" The goal of this page is to build an intuition for the primitives in Honcho and how they fit together -Honcho has 3 main components that work together to manage agent identity and context. +Honcho has 2 main components that work together to manage agent identity and context. -- **The Storage API**: The Memory layer for storing interaction history for your agents -- **The Deriver**: The background processing layer that builds representations of users and agents -- **The Dialectic API**: The natural language API for chatting with representations +- **The Memory Layer**: The Memory layer for storing interaction history for your agents +- **The Reasoning Layer**: The background processing layer that builds representations of users and agents Below we'll deep dive into these different areas, discussing the data primitives, the flow of data through the system, artifacts Honcho produces, and how to use them. - ## Data Model Honcho has a hierarchical data model centered around the entities below. @@ -37,9 +35,9 @@ Honcho has a hierarchical data model centered around the entities below. style SM fill:#e8f5e9,stroke:#2e7d32,color:#000 ``` -There are `Workspaces` at the top that contain `Peers` and `Sessions`. A `Peer` -can be part of many `Sessions` and a `Session` can have many `Peers`. `Sessions` -hold messages that are sent by `Peers`. +- A `Workspaces` has `Peers` & `Sessions` +- A `Peer` can be in multiple `Sessions` and can send `Messages` in a `Session`. +- A `Session` can have many `Peers` and stores `Messages` sent by its `Peers`. ### Workspaces @@ -126,19 +124,38 @@ with a single peer and structure the data as messages. - File uploads (PDFs, text files, JSON documents) -## Deriver +## Reasoning Layer -At the core of developing representations of Peers, we have the Deriver. The -Deriver refers to a set of processes in Honcho that enqueue new messages sent -by peers and reasons over them to extract facts, insights, and context. +The raw data you store in Honcho is useful, but it's not in a format that's most +useful for an LLM to consume. There may be too many tokens that need to be +compacted, key facts about what happened may be hard to piece together because +they involve messages from across different sessions, etc. + +To solve this problem, Honcho has a reasoning layer that continually processes +incoming data to form the most informationally dense and useful representations of `Peers` +that we can then expose to agents. Honcho does the following tasks in +the reasoning engine. + +- **Fact Derivation** +- **Generate Summaries** +- **Generate Peer Cards** +- **Dreaming** + + +Honcho will reason about each `Message` it +ingests to generate new facts and insights that are spelled out and easy to +consume in an LLM prompt. + +We refer to this module of Honcho as the `Deriver`, because it's constantly +deriving new insights from messages. The sum total of all these generated +insights are what we refer to as a `Representation`, all the data related to who +and what a `Peer` is. Depending on the configuration of a `Peer` or `Session`, the deriver will behave differently and update different representations. -Facts derived here are used in the Dialectic chat endpoint to generate -context-aware responses that can correctly reference both concrete facts -extracted from messages and social insights deduced from facts, tone, and -opinion. +Facts derived here are used in the Dialectic chat endpoint, get_context +endpoint, Deriver tasks are processed in parallel, but tasks affecting the same peer representation will always be processed serially in order of message creation, so as to properly understand their cumulative effect. @@ -149,7 +166,7 @@ There are two types of tasks that the deriver currently does: - **Representation Tasks**: Generate/update peer representations - **Summary Tasks**: Generate conversation summaries -### Peer Representations +### Local & Global Representations Peer representations are more of an abstract concept, as they are made up of various pieces of data stored throughout Honcho. There are however diff --git a/docs/v2/documentation/core-concepts/configuration.mdx b/docs/v2/documentation/core-concepts/configuration.mdx index ff57dc8e..bc8635b9 100644 --- a/docs/v2/documentation/core-concepts/configuration.mdx +++ b/docs/v2/documentation/core-concepts/configuration.mdx @@ -1,5 +1,5 @@ --- -title: 'Configuration' +title: 'Configure Reasoning' description: 'Customizing how Honcho handles peers and sessions' icon: 'wrench' --- diff --git a/docs/v2/documentation/core-concepts/features.mdx b/docs/v2/documentation/core-concepts/features.mdx deleted file mode 100644 index cc1d402e..00000000 --- a/docs/v2/documentation/core-concepts/features.mdx +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: 'Features' -description: 'Key features and capabilities of Honcho' -icon: 'star' ---- - -This page is a quick overview of the features within Honcho. In-depth -guides are available for each feature in the [Spellbooks - Design Patterns](../../guides/overview#design-patterns) section. - -### Local vs Global Representation -Peers in Honcho are abstract entities that can represent humans, agents, or NPCs. Honcho has a two-layer approach to forming representations of Peers. -- **Global Representation**: Representation owned by a Peer that is constructed from everything the Peer has sent within Honcho. -- **Local Representation**: The representation that a Peer forms of other Peers, based on the messages those other Peers have sent (as observed by the Peer forming the representation). -- At the Session level, you can configure which Peers are able to observe messages from other Peers in that Session. This determines which Peers form representations of others within the Session. - -### Queue Status -To help developers understand when a Peer's representation is fully up to date, Honcho exposes the ability to poll the status of Peer-centric queues that construct representations. -- If no Session is specified, the queue status reflects pending work for the Peer's global representation. -- If a Session is specified, the queue status reflects pending work for the Peer's working representation in that Session. - -### Search -Honcho implements a powerful search endpoint that allows you to search for messages across a workspace, session, or peer with complex [filters](/v2/guides/using-filters). - -The search process combines full-text and semantic search using reciprocal rank fusion. By default, all messages ingested into Honcho have embeddings generated and stored in the database, enabling semantic search -- if this feature is disabled, the search process will only use full-text search. - -Results are returned in the form of a list of Message objects, and you may choose how many results to return. The default is 10 results, with a maximum of 100. - -In the SDK, search is available on `Workspace`, `Session`, and `Peer` objects, and an optional `filters` parameter may be used to apply a narrower search scope such as a time range or developer-defined metadata attached to messages. - -Note that results are not ordered by recency, only relevance. Results can be sorted by timestamp or a filter on the `created_at` field can limit results to recent messages. - -[Look here for examples of how to use search in the SDK](/v2/guides/search). - -### Scoped API Keys -Builders can create scoped API keys to control access to different resources within Honcho. -- **Workspace-Level Keys**: Access to everything scoped to a Workspace. -- **Peer-Level Keys**: Access to everything scoped to a Peer. -- **Session-Level Keys**: Access to everything scoped to a Session. - -### Get Context -Honcho provides a powerful context retrieval feature that delivers formatted conversation context from sessions, making it easy to integrate with LLMs like OpenAI, Anthropic, and others. -- By default, the context includes a blend of summary and messages which covers the entire history of the session. -- Summaries are generated automatically at intervals, and recent messages are included based on your specified token budget for the context. -- You can set any token limit, and if you prefer, you can disable summaries so that the context consists entirely of the most recent messages up to your chosen limit. diff --git a/docs/v2/guides/dialectic-endpoint.mdx b/docs/v2/documentation/core-concepts/features/dialectic-endpoint.mdx similarity index 100% rename from docs/v2/guides/dialectic-endpoint.mdx rename to docs/v2/documentation/core-concepts/features/dialectic-endpoint.mdx diff --git a/docs/v2/guides/file-uploads.mdx b/docs/v2/documentation/core-concepts/features/file-uploads.mdx similarity index 100% rename from docs/v2/guides/file-uploads.mdx rename to docs/v2/documentation/core-concepts/features/file-uploads.mdx diff --git a/docs/v2/guides/get-context.mdx b/docs/v2/documentation/core-concepts/features/get-context.mdx similarity index 99% rename from docs/v2/guides/get-context.mdx rename to docs/v2/documentation/core-concepts/features/get-context.mdx index 6722a548..fb4e7fab 100644 --- a/docs/v2/guides/get-context.mdx +++ b/docs/v2/documentation/core-concepts/features/get-context.mdx @@ -1,5 +1,5 @@ --- -title: 'Working with Session Context' +title: 'Get Context' description: 'Learn how to use get_context() to retrieve and format conversation context for LLM integration' icon: 'messages' --- diff --git a/docs/v2/documentation/core-concepts/features/local-vs-global.mdx b/docs/v2/documentation/core-concepts/features/local-vs-global.mdx new file mode 100644 index 00000000..e5670229 --- /dev/null +++ b/docs/v2/documentation/core-concepts/features/local-vs-global.mdx @@ -0,0 +1,68 @@ +--- +title: Local vs Global Representations +description: Model directional relationships between Peers in Honcho +icon: location-pin +--- + +One of the unique affordances of Honcho is that it allows developers to model +directional relationships between Peers. What I mean by this is you can model +how one `Peer` thinks about another `Peer`. + +There are many use cases where you don't want every agent or human to know +everything about another user such as games or multi-agent workflows. To +illustrate this, the following examples shows 2 conversations. + +Conversation #1 (With Bob and Alice) +``` +Alice: I had a great breakfast today. +Bob: What did you eat? +Alice: I had pancakes and eggs and bacon +``` + +Conversation #2 (With Alice and Charlie) +``` +Alice: I actually didn't eat any breakfast today. +Charlie: Oh that's too bad. +Alice: But I lied to Bob and told him I did, so back me up if you see them. +``` + +Alice told Bob a lie in this conversation. If we stored both of these +conversations in Honcho with Alice, Bob, and Charlie as `Peers` and let them +use Honcho to get insights on each other then Bob would immediately know this +deception. For example: + + + ```python Python + # Bob could run + alice.chat("What did Alice eat today?") + # Response: Alice did not eat anything today + ``` + + +This is a problem. Bob shouldn't be able to know everything about Alice in this +situation. So to support these situations we support what we call **Local +Representations**. + +By default insights generated for a `Peer` are scoped globally. This means every +message sent by that `Peer` in any conversation updates the same representation +of that `Peer`. However, we can enable **Local Representations** so Bob can +form a representation Alice based only on what they observe Alice do. + +This feature is illustrated in the graphic below: +Peer Representations + +We can enable local representation for a `Peer` by setting `observe_others=True`. +This is shown in the [Configure +Reasoning](/v2/documentation/core-concepts/configuration) page. + +Now if we used Bob's local representation of Alice then Bob would only get +insights on what they've seen Alice say to them. + +```python +bob.chat(target="alice", query="What did Alice eat today?") +# Response: Alice ate pancakes, eggs, and bacon +``` + + + Local Representations are turned off by default + diff --git a/docs/v2/documentation/core-concepts/features/queue-status.mdx b/docs/v2/documentation/core-concepts/features/queue-status.mdx new file mode 100644 index 00000000..ed28161f --- /dev/null +++ b/docs/v2/documentation/core-concepts/features/queue-status.mdx @@ -0,0 +1,132 @@ +--- +title: Queue Status +description: Learn how to check the status of the Deriver +icon: lines-leaning +--- + +Whenever `Messages` are stored in Honcho, a background process called the +[Deriver](/docs/v2/documentation/core-concepts/architecture#reasoning-layer) is +triggered to reason about the conversation and generate insights. + +The Deriver is an asynchronous process and, depending on load may not immediately +generated insights for the latest message you've sent. To help with this, Honcho +provides several utilities to check the status of the Deriver. + + +```python Python +from honcho import Honcho +honcho = Honcho() + +status = honcho.get_deriver_status() +honcho.poll_deriver_status() +``` + +```typescript typescript +import { Honcho } from '@honcho-ai/sdk'; + +const honcho = new Honcho({}); + +const status = await honcho.getDeriverStatus(); +await honcho.pollDeriverStatus(); +``` + + +Output types + + +```python Python +class DeriverStatus(BaseModel): + completed_work_units: int + """Completed work units""" + + in_progress_work_units: int + """Work units currently being processed""" + + pending_work_units: int + """Work units waiting to be processed""" + + total_work_units: int + """Total work units""" + + sessions: Optional[Dict[str, Sessions]] = None + """Per-session status when not filtered by session""" +``` +```typescript TypeScript +Promise<{ + totalWorkUnits: number + completedWorkUnits: number + inProgressWorkUnits: number + pendingWorkUnits: number + sessions?: Record + }> + +``` + + +Whenever a `Message` is sent it will generate several tasks. These could +be tasks such as generating insights, cleaning up a representation, summarizing +a conversation etc. These tasks are defined based on who is sending the +message, what `Session` the message is in, and potentially who is observing the +message. We call the combination of these parameters a `work_unit` + +This has a few different implications. + +- tasks within the same work_unit are processed sequentially, but multiple +work_units will be processed in parallel +- If local representations are turned in a Session then a `Message` will + generate an additional work unit for every `Peer` that has `observe_others=True` + +The `get_deriver_status` and `poll_deriver_status` methods can take additional +parameters to scope the status to a specific work unit + + +```python Python +def get_deriver_status( + self, + observer_id: str | None = None, + sender_id: str | None = None, + session_id: str | None = None, + ) -> DeriverStatus: +``` +```typescript TypeScript + +export const DeriverStatusOptionsSchema = z.object({ + observerId: z.string().optional(), + senderId: z.string().optional(), + sessionId: z.string().optional(), + timeoutMs: z + .number() + .positive('Timeout must be a positive number') + .optional(), +}) + +``` + + +Additionally, there are deriver status and polling deriver status methods +available on the `Session` objects in each of the SDKs. + +Below are the function signatures for the session level deriver status method + + +```python python +@validate_call + def get_deriver_status( + self, + observer_id: str | None = None, + sender_id: str | None = None, + ) -> DeriverStatus: +``` + +```typescript TypeScript +async getDeriverStatus( + options?: Omit + ): Promise<{ + totalWorkUnits: number + completedWorkUnits: number + inProgressWorkUnits: number + pendingWorkUnits: number + sessions?: Record + }> +``` + diff --git a/docs/v2/guides/search.mdx b/docs/v2/documentation/core-concepts/features/search.mdx similarity index 100% rename from docs/v2/guides/search.mdx rename to docs/v2/documentation/core-concepts/features/search.mdx diff --git a/docs/v2/documentation/core-concepts/features/storing-data.mdx b/docs/v2/documentation/core-concepts/features/storing-data.mdx new file mode 100644 index 00000000..0ec7f4b4 --- /dev/null +++ b/docs/v2/documentation/core-concepts/features/storing-data.mdx @@ -0,0 +1,61 @@ +--- +title: Storing Data +description: "Store Data in Honcho to Generate Memories and Insights" +icon: "memory" +--- + +The most basic building block of Honcho's data model is the `Message` object. +A `Message` is sent by a `Peer` and saved in a `Session` + + + + ```python Python + from honcho import Honcho + + honcho = Honcho() + + peer = honcho.peer("sample-peer") + + session = honcho.session("sample-session") + + message = peer.message("Hello, world!", session_id=session.id) + + session.add_messages([message]) + ``` + + ```typescript TypeScript + import { Honcho } from '@honcho-ai/sdk'; + + const honcho = new Honcho({}); + + const peer = await honcho.peer('sample-peer'); + + const session = await honcho.session('sample-session'); + + const message = peer.message('Hello, world!'); + + await session.addMessages([message]); +``` + + +Once a `Message` is saved in Honcho, it will kick off a background task that +looks at the new data to generate insights about the `Peer` that sent the `Message` + +This is the default behavior of Honcho and can be turned off by [configuring the +Peer or Session](/v2/documentation/core-concepts/configuration) + +This pattern of having a Peer, Session, and Messages is highly flexible and +works for many different use cases and agent setups. Some use cases may only +need a single Peer, but many Sessions. Others will only use a single `Session` +for their entire app. These are flexible components that work in any situation. + +## Chat Bots + +A common use case for Honcho to is to build a chatbot like ChatGPT or Claude. +In this case you can simply + +- Make a `Peer` for the User +- Make a `Peer` for the AI + +Then you can make a `Session` for each thread of conversation and save +`Messages` from the user and assistant in each turn of conversation diff --git a/docs/v2/guides/streaming-response.mdx b/docs/v2/documentation/core-concepts/features/streaming-response.mdx similarity index 100% rename from docs/v2/guides/streaming-response.mdx rename to docs/v2/documentation/core-concepts/features/streaming-response.mdx diff --git a/docs/v2/guides/using-filters.mdx b/docs/v2/documentation/core-concepts/features/using-filters.mdx similarity index 100% rename from docs/v2/guides/using-filters.mdx rename to docs/v2/documentation/core-concepts/features/using-filters.mdx diff --git a/docs/v2/guides/working-rep.mdx b/docs/v2/documentation/core-concepts/features/working-rep.mdx similarity index 100% rename from docs/v2/guides/working-rep.mdx rename to docs/v2/documentation/core-concepts/features/working-rep.mdx diff --git a/docs/v2/documentation/introduction/overview.mdx b/docs/v2/documentation/introduction/overview.mdx index 1fb4f199..e94a85de 100644 --- a/docs/v2/documentation/introduction/overview.mdx +++ b/docs/v2/documentation/introduction/overview.mdx @@ -5,108 +5,102 @@ icon: "brain" sidebarTitle: "Overview" --- -When building agents developers often run into the same walls: +Honcho is an AI-native memory library for building agents with +[state-of-the-art](https://blog.plasticlabs.ai/research/Introducing-Neuromancer-XR) +long-term memory. -> "My agent forgets everything between chats" +Agents using Honcho have perfect recall with a wide variety of tools to traverse +their history and get the exact context they need when they need it. -You need memory: session management, message storage, context handling. It's table stakes, but surprisingly complex to get right. +It then goes beyond basic memory by reasoning about the stored history +to expand the latent information available to your agent. Agents using Honcho +will understand who they are, who they are interacting with, what happened, and +when it happened — all without you having to think about it. -> "My agent treats everyone exactly the same" +Use it to build -You need personalization: user modeling, preference learning, behavioral adaptation. Now you're building a [social cognition](../core-concepts/glossary#social-cognition) engine. +- Highly personalized experiences +- Agents with social cognition +- Agents with rich identity that evolve over time +- Multi-agent systems with complex social dynamics -> "I'm writing infrastructure instead of features" - -You need Honcho - -Honcho's Hiearchy of Agents - -Honcho delivers production-ready memory infrastructure from day one. Store -conversations, manage sessions, get perfectly formatted context for any LLM. -But here's the magic: while your agents are chatting, Honcho is learning. It -builds Theory of Mind models automatically, transforming raw conversations into -rich psychological understanding. ```python -# Start simple - just add messages +# Start simple by just adding messages session.add_messages([alice.message("I learn best with examples")]) -# Get powerful - query user psychology +# Honcho will automatically reason about the message to generate insights about Alice + +# Get insights by chatting with the agent insight = peer.chat("How should I explain this concept?") # > "This user learns best through concrete examples..." ``` -Your agents evolve from goldfish to counselor, on the same infrastructure. That's Honcho. - Designed for developers and agents alike: -- **Natural Language Queries**: Chat with Honcho in natural language via the [Dialectic API](../core-concepts/architecture#dialectic-api) and let agents backchannel -- **Automatic Context Management**: Smart summarization that respects token limits -- **Native multi-agent support**: Break out of User/Assistant Paradigms and build complex multi-agent systems +- **Natural Language Queries**: Chat with Honcho in natural language via the [Dialectic API](../core-concepts/architecture#dialectic-api) to get insights about your users and agents +- **Automatic Context Management**: Smart conversation summaries to have infinite chats +- **Native multi-agent support**: Sessions can natively have as many participants as you need - **Agent-first interfaces**: MCP connections and APIs designed for agents to consume and use as tools - **Provider Agnostic**: Works with any LLM or Agent Framework ## How It Works -### Storage + + + High Level Honcho Diagram + + -Developers use Honcho to store information about their users and application via -two integrated layers: +At a high level Honcho works very simply: -Basic Honcho Flowchart +1. Store messages sent by users and agents in Honcho +2. Honcho reasons about the messages to generate insights about each entity in +the system +3. At runtime your agents can leverage insights from Honcho to get the exact +context they need -**Memory Layer**: Captures all user interactions - messages, preferences, and -behavioral patterns - in a peer-centric data model that scales from individual -conversations to complex multi-agent scenarios. This also queues up messages for -the reasoning layer to process. +There are several API endpoints to leverage the memory & insights in Honcho. -**Reasoning Layer**: Continuously analyzes stored interactions to build -psychological profiles using [theory of mind](../core-concepts/glossary#theory-of-mind) -inference, extracting patterns about communication style, decision-making -preferences, and mental models. +### Get Context -### Retrieval +This is the easiest way to leverage Honcho. simply call get context and get the +most relevant information for your conversation. This endpoint is highly +customizable so you can specify parameters such as: -Once data is stored and generated within Honcho, the API exposes several -different ways to retrieve and use those insights. +- A number of tokens you want +- An option to include summaries of the conversation +- An option to get a profile of a specific user (Peer Card & Representation) -**[Dialectic API](/v2/guides/dialectic-endpoint)**: This is the -flagship endpoint that allows developers to send natural language queries to -Honcho to chat with the representation of each user in your system to get -dynamic, in-context actionable insights. +### Search -Example Queries +This endpoint lets you search across Honcho for relevant messages using a +hybrid search strategy that combines full-text and semantic search. + +You can optionally scope the endpoint to a specific workspace, peer, or session. + +### Working Representation + +This endpoint gives you a snapshot of a user or what we call a +**Representation**. Essentially, a list of explicit and deductive facts about +the user that are relevant to the current conversation. + +Plug this into your prompt to get a quick overview of the user. + +### Dialectic API + +This endpoint lets you chat with Honcho about any entity in your system. Honcho +will leverage what it has remembered and learned about the entity to provide in-context actionable insights. + +This is especially helpful when you want your agent to back-channel with Honcho to +change its behavior at runtime. + +Example Queries: - "What's the best way to explain technical concepts to this user?" - "Is this user more task-oriented or relationship-oriented?" - "What time of day is this user most engaged?" - "How does this user prefer to receive feedback?" - "What are this user's core values based on our conversations?" -**[Get Context](/v2/guides/get-context)**: This endpoint abstracts context window -constraints and continuously retrieves the most relevant and recent data from a -conversation. Provide a token budget and Honcho will return a combination of -summaries and messages that provide session context. Use this for creating -long-running conversations. We crafted our summaries to provide the most -[coverage of a session possible](../core-concepts/summarizer). - -**[Search](/v2/guides/search)**: This endpoint allows you to search across Honcho -for relevant messages either at the workspace, peer, or session level. This -endpoint uses a hybrid search strategy that combines text search and cosine -similarity. - -**[Working Representations](/v2/guides/working-rep)**: Get a cached, snapshot -of a user in the context of a session. Instead of waiting for an LLM to -synthesize an in-context response via the Dialectic endpoint, use this to get -recent insights you can plug into your context window. - -## Ideal For - -**Personalized AI assistants** that need to understand individual psychology, not just remember conversations. - -**Customer-facing agents** that must adapt their approach based on user communication preferences and emotional context. - -**Multi-agent systems** where AI needs to understand human collaborators' working styles and decision-making patterns. - -**NPCs** where you want autonomous agents with a rich and deep personality that isn't the average sycophantic llm ## Getting Started @@ -115,7 +109,7 @@ Ready to integrate Honcho into your application? Get up and running with Honcho in minutes Understand Honcho's +href="/v2/documentation/core-concepts/architecture"> Understand Honcho's fundamental concepts ## Community & Support diff --git a/docs/v2/documentation/introduction/quickstart.mdx b/docs/v2/documentation/introduction/quickstart.mdx index 30175b5e..fb5ff35a 100644 --- a/docs/v2/documentation/introduction/quickstart.mdx +++ b/docs/v2/documentation/introduction/quickstart.mdx @@ -62,7 +62,7 @@ The Honcho client is the main entry point for interacting with Honcho's API. By from honcho import Honcho # Initialize client (uses demo environment and default workspace) -client = Honcho() +honcho = Honcho() ``` @@ -70,7 +70,7 @@ client = Honcho() import { Honcho } from '@honcho-ai/sdk'; // Initialize client (uses demo environment and default workspace) -const client = new Honcho({}); +const honcho = new Honcho({}); ``` @@ -83,7 +83,7 @@ import os from honcho import Honcho # Production environment with API key -client = Honcho( +honcho = Honcho( api_key=os.environ["HONCHO_API_KEY"], environment="production", # Create a workspace, otherwise set to "default" @@ -95,7 +95,7 @@ client = Honcho( import { Honcho } from '@honcho-ai/sdk'; // Production environment with API key -const client = new Honcho({ +const honcho = new Honcho({ apiKey: process.env.HONCHO_API_KEY!, environment: "production", // Create a workspace, otherwise set to "default" @@ -110,13 +110,13 @@ Peers represent individual users, AI agents, or any conversational entity in you ```python Python -alice = client.peer("alice") -bob = client.peer("bob") +alice = honcho.peer("alice") +bob = honcho.peer("bob") ``` ```typescript TypeScript -const alice = await client.peer("alice") -const bob = await client.peer("bob") +const alice = await honcho.peer("alice") +const bob = await honcho.peer("bob") ``` @@ -126,12 +126,12 @@ Sessions are independent conversations that can include multiple peers: ```python Python -session = client.session("session_1") +session = honcho.session("session_1") session.add_peers([alice, bob]) ``` ```typescript TypeScript -const session = await client.session("session_1") +const session = await honcho.session("session_1") await session.addPeers([alice, bob]) ``` @@ -171,7 +171,7 @@ Now ask Honcho what it's learned - this is where the magic happens: ```python Python # Ask what Bob is like -response = alice.chat("Tell me about Bob's interests and habits") +response = bob.chat("Tell me about Bob's interests and habits") print(response) # Returns rich context like: @@ -182,36 +182,128 @@ print(response) ``` ```typescript TypeScript -(async () => { - // Ask what Bob is like - const response = await alice.chat("Tell me about Bob's interests and habits"); - console.log(response); +bob.chat("Tell me about Bob's interests and habits").then((response) => { + console.log(response); + // Returns rich context like: + // "Bob is health-conscious and has been working on getting back in shape. + // He regularly goes to the gym, particularly in the evenings, and finds + // exercise helps him relax. He's encouraging about fitness and willing + // to share advice about workout routines." +}) +``` + - // Returns rich context like: - // "Bob is health-conscious and has been working on getting back in shape. - // He regularly goes to the gym, particularly in the evenings, and finds - // exercise helps him relax. He's encouraging about fitness and willing - // to share advice about workout routines." -})(); +## 7. Putting it all together + + +```python Python +import os +from honcho import Honcho + +# Create your client +honcho = Honcho( + api_key=os.environ["HONCHO_API_KEY"], + environment="production", + # Create a workspace, otherwise set to "default" + # workspaceId="your-workspace-id" +) + +# Get your Peers +alice = honcho.peer("alice") +bob = honcho.peer("bob") + +# Make a Session and add your Peers +session = honcho.session("session_1") +session.add_peers([alice, bob]) + +# Add messages sent by your Peers +session.add_messages([ + alice.message("Hi Bob, how are you?"), + bob.message("I'm good, thank you!"), + alice.message("What are you doing today after work?"), + bob.message("I'm going to the gym! I've been trying to get back in shape."), + alice.message("That's great! I should probably start exercising too."), + bob.message("You should! I find that evening workouts help me relax."), +]) + +# Get insights about your Peers +response = bob.chat("Tell me about Bob's interests and habits") +print(response) + +# Returns rich context like: +# "Bob is health-conscious and has been working on getting back in shape. +# He regularly goes to the gym, particularly in the evenings, and finds +# exercise helps him relax. He's encouraging about fitness and willing +# to share advice about workout routines." +``` + +```typescript TypeScript +import { Honcho } from '@honcho-ai/sdk'; + +// Create your client +const honcho = new Honcho({ + apiKey: process.env.HONCHO_API_KEY!, + environment: "production", + // Create a workspace, otherwise set to "default" + // workspace: "your-workspace-id" +}); + +// Get your Peers +const alice = await honcho.peer("alice") +const bob = await honcho.peer("bob") + +// Make a Session and add your peers +const session = await honcho.session("session_1") +await session.addPeers([alice, bob]) + +// Add messages sent by your Peers +await session.addMessages([ + alice.message("Hi Bob, how are you?"), + bob.message("I'm good, thank you!"), + alice.message("What are you doing today after work?"), + bob.message("I'm going to the gym! I've been trying to get back in shape."), + alice.message("That's great! I should probably start exercising too."), + bob.message("You should! I find that evening workouts help me relax."), +]) + +// Get insights about your peers +bob.chat("Tell me about Bob's interests and habits").then((response) => { + console.log(response); + // Returns rich context like: + // "Bob is health-conscious and has been working on getting back in shape. + // He regularly goes to the gym, particularly in the evenings, and finds + // exercise helps him relax. He's encouraging about fitness and willing + // to share advice about workout routines." +}) ``` ## What Just Happened? -Honcho automatically built rich psychological profiles from just a few messages: +You just got through building a simple conversation between two people, Alice +and Bob. We: -- **Theory of Mind Processing**: Understanding personality, preferences, and patterns -- **Ambient Learning**: No surveys or explicit training - just natural conversation -- **Rich Context**: Far more detailed than simple conversation history +1. Set up our connection to Honcho. +2. Setup who the participants of our conversation are, these are called `Peers`. +3. Made a `Session` and added our `Peers` to it. +4. Sent messages from our `Peers` +5. Chat with Honcho to get insights about one of the `Peers` in the conversation -The response isn't just retrieving stored text - it's synthesizing insights about Bob's personality, habits, and communication style. +As soon as you save a message in Honcho, it will start to reason about it to +pull out insights and develop a profile of the user. This is the default +behavior and can be toggled off via [the configuration](/v2/documentation/core-concepts/configuration). ## Next Steps -This covers the core concepts: **peers**, **sessions**, **messages**, and **dialectic queries**. - -- For production use, [sign up for the managed platform](https://app.honcho.dev) or get an [overview here](../reference/platform). -- For detailed API reference, check out our [SDK documentation](../reference/sdk). -- For more examples, explore our [guides](../guides/overview). - ---- + + + Learn about the data primitives in Honcho and how they work together + + + Sign up for Managed Honcho and get started building agents now. + + + Check out spellbooks to see different examples apps built with Honcho + + diff --git a/docs/v2/documentation/introduction/vibecoding.mdx b/docs/v2/documentation/introduction/vibecoding.mdx index 31f98ff6..7127c8e6 100644 --- a/docs/v2/documentation/introduction/vibecoding.mdx +++ b/docs/v2/documentation/introduction/vibecoding.mdx @@ -5,7 +5,17 @@ description: "Universal starter prompt for building with Honcho" sidebarTitle: 'Vibecoding Setup' --- -Copy this prompt into Cursor, Claude, or any AI coding assistant to start building with Honcho. +These docs are designed to be easily consumable for LLMs. Each page has a button +the lets you copy the page as Markdown or paste directly into ChatGPT or Claude. + +Additionally, we follow the llms.txt standard. There are both an llms.txt and +llms-full.txt available. + +- [llms.txt](/llms.txt) +- [llms-full.txt](/llms-full.txt) + +Additionally, we provide a starter prompt to paste into a coding assistant to +quickly get started building with Honcho. ## 🚀 Universal Starter Prompt diff --git a/docs/v2/documentation/reference/platform.mdx b/docs/v2/documentation/reference/platform.mdx index 98b3cbb3..3b01717a 100644 --- a/docs/v2/documentation/reference/platform.mdx +++ b/docs/v2/documentation/reference/platform.mdx @@ -1,8 +1,8 @@ --- -title: "Managed Honcho Platform" +title: "The Honcho Dashboard" icon: "rocket" description: "Build socially intelligent agents without worrying about infrastructure" -sidebarTitle: "Platform Overview" +sidebarTitle: "Dashboard Overview" --- diff --git a/docs/v2/guides/overview.mdx b/docs/v2/guides/overview.mdx index 79a08be8..dcaf5b48 100644 --- a/docs/v2/guides/overview.mdx +++ b/docs/v2/guides/overview.mdx @@ -11,40 +11,20 @@ AI development often feels like magic - you craft the right prompt and get exact Whether you're integrating Honcho into existing platforms, exploring advanced features, or getting up and running quickly, these guides provide concrete examples and implementation patterns. -## What You'll Find Here +Each spellbook focuses on a specific use case with working code you can adapt to your needs. The goal is to get you from idea to working prototype as quickly as possible, then provide the depth you need to scale and customize. -### Getting Started -**[Overview](/v2/guides/overview)** - You are here - -**[MCP Integration](/v2/guides/mcp)** - Get Honcho running with a single prompt in Cursor or Claude Code - -### Application Interfaces +## Application Interfaces Ready-to-use integration patterns for popular platforms: -**[Discord Bot](/v2/guides/discord)** - Build a Discord bot that remembers users across conversations - -**[Telegram Bot](/v2/guides/telegram)** - Create a Telegram bot with persistent user understanding - -### Design Patterns -Implementation patterns for Honcho's core capabilities: - -**[Dialectic Endpoint](/v2/guides/dialectic-endpoint)** - Query user psychology in natural language - -**[Working with Session Context](/v2/guides/get-context)** - Manage conversation flow and context windows - -**[Search](/v2/guides/search)** - Search your data using natural language - -**[Working Representations](/v2/guides/working-rep)** - Understanding and customizing user models - -**[Streaming Responses](/v2/guides/streaming-response)** - Handle real-time interactions efficiently - -**[Using Filters](/v2/guides/using-filters)** - Control what data gets processed and how - -**[File Uploads](/v2/guides/file-uploads)** - Upload PDF, text, or JSON files to create messages - -## Philosophy - -These aren't just API documentation - they're implementation patterns that solve real problems. Each spellbook focuses on a specific use case with working code you can adapt to your needs. - -The goal is to get you from idea to working prototype as quickly as possible, then provide the depth you need to scale and customize. + + + Build a Discord bot that remembers users across conversations + + + Create a Telegram bot with persistent user understanding + + +Get Honcho running with a single prompt in Cursor or Claude Code + + diff --git a/migrations/env.py b/migrations/env.py index f0a8197c..4ee654bd 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,4 +1,4 @@ -import logging +import logging # noqa: I001 import sys from logging.config import fileConfig from pathlib import Path @@ -12,6 +12,10 @@ from src.config import settings # Import your models from src.db import Base +# Import all models so they register with Base.metadata +import src.models # noqa: F401 + + # Set up logging more verbosely logging.basicConfig() logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) @@ -166,6 +170,13 @@ def run_migrations_online() -> None: connection=connection, target_metadata=target_metadata, version_table_schema=target_metadata.schema, + include_schemas=True, + include_object=lambda obj, name, type_, reflected, compare_to: ( + # Only include objects from our target schema + getattr(obj, "schema", None) == target_metadata.schema + if hasattr(obj, "schema") + else True + ), ) with context.begin_transaction(): diff --git a/migrations/script.py.mako b/migrations/script.py.mako index fbc4b07d..bba30146 100644 --- a/migrations/script.py.mako +++ b/migrations/script.py.mako @@ -5,18 +5,20 @@ Revises: ${down_revision | comma,n} Create Date: ${create_date} """ -from typing import Sequence, Union +from collections.abc import Sequence from alembic import op import sqlalchemy as sa ${imports if imports else ""} +from migrations.utils import get_schema # revision identifiers, used by Alembic. revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} +down_revision: str | None = ${repr(down_revision)} +branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} +depends_on: str | Sequence[str] | None = ${repr(depends_on)} +schema = get_schema() def upgrade() -> None: ${upgrades if upgrades else "pass"} diff --git a/migrations/utils.py b/migrations/utils.py index 4333ae46..1dc74d86 100644 --- a/migrations/utils.py +++ b/migrations/utils.py @@ -71,3 +71,89 @@ def constraint_exists( else: raise ValueError(f"Invalid constraint type: {type}") return any(constraint["name"] == constraint_name for constraint in constraints) + + +def make_column_non_nullable_safe(table_name: str, column_name: str) -> None: + """ + Make a column non-nullable using a non-blocking approach to minimize lock duration. + + WARNING: Only use this if you can guarantee that: + 1. No NULL values currently exist in the column + 2. The application code is already writing non-NULL values to this column or + 3. The column has never accepted NULLs in practice + + This uses a 4-step process to avoid long exclusive locks: + 1. Add CHECK constraint with NOT VALID (instant, no scan) + 2. Validate the constraint (scans but allows concurrent read/writes to the table) + 3. Set column NOT NULL (fast since we've validated the constraint) + 4. Drop the redundant CHECK constraint + + Args: + table_name: The name of the table + column_name: The name of the column to make non-nullable + """ + schema = get_schema() + conn = op.get_bind() + constraint_name = f"{table_name}_{column_name}_not_null" + + # Step 1: Check if the column is already non-nullable + inspector = sa.inspect(op.get_bind()) + columns = inspector.get_columns(table_name, schema=schema) + column_info = next((col for col in columns if col["name"] == column_name), None) + if column_info is None: + raise ValueError(f"Column {table_name}.{column_name} does not exist") + if not column_info["nullable"]: + print(f"Column {table_name}.{column_name} is already non-nullable, skipping...") + return + + # Step 2: Add CHECK constraint without validation (instant) + # Note: op.create_check_constraint() doesn't support NOT VALID, so use raw SQL + + # Get the identifier preparer for safe quoting + dialect = conn.dialect + preparer = dialect.identifier_preparer + + quoted_schema = preparer.quote(schema) + quoted_table = preparer.quote(table_name) + quoted_constraint = preparer.quote(constraint_name) + quoted_column = preparer.quote(column_name) + + # Step 2: Add CHECK constraint without validation (instant) + # Note: op.create_check_constraint() doesn't support NOT VALID, so use raw SQL + if not constraint_exists(table_name, constraint_name, "check"): + conn.execute( + sa.text( + f""" + ALTER TABLE {quoted_schema}.{quoted_table} + ADD CONSTRAINT {quoted_constraint} + CHECK ({quoted_column} IS NOT NULL) + NOT VALID + """ + ) + ) + + # Step 3: Validate constraint (scans but allows concurrent operations) + conn.execute( + sa.text( + f""" + ALTER TABLE {quoted_schema}.{quoted_table} + VALIDATE CONSTRAINT {quoted_constraint} + """ + ) + ) + + # Step 4: Set NOT NULL (fast with validated constraint) + op.alter_column( + table_name, + column_name, + nullable=False, + schema=schema, + ) + + # Step 5: Drop the redundant CHECK constraint + op.drop_constraint( + constraint_name, + table_name, + type_="check", + schema=schema, + ) diff --git a/migrations/versions/066e87ca5b07_align_schema_with_declarative_models.py b/migrations/versions/066e87ca5b07_align_schema_with_declarative_models.py new file mode 100644 index 00000000..72966bf0 --- /dev/null +++ b/migrations/versions/066e87ca5b07_align_schema_with_declarative_models.py @@ -0,0 +1,341 @@ +"""align_schema_with_declarative_models + +Revision ID: 066e87ca5b07 +Revises: bb6fb3a7a643 +Create Date: 2025-10-27 12:36:51.614959 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +from migrations.utils import ( + column_exists, + constraint_exists, + fk_exists, + get_schema, + index_exists, + make_column_non_nullable_safe, +) + +# revision identifiers, used by Alembic. +revision: str = "066e87ca5b07" +down_revision: str | None = "bb6fb3a7a643" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +schema = get_schema() + + +def upgrade() -> None: + """ + The application code has been previously updated to ensure none of the following columns have NULL values but the actual DB schema is out of sync with our SQLAlchemy model definitions. + This migration fixes this by making the columns non-nullable using a non-blocking approach to minimize lock duration. + """ + conn = op.get_bind() + + # Make peers.workspace_name non-nullable + if column_exists("peers", "workspace_name"): + make_column_non_nullable_safe("peers", "workspace_name") + + # Make sessions.workspace_name non-nullable + if column_exists("sessions", "workspace_name"): + make_column_non_nullable_safe("sessions", "workspace_name") + + # Make active_queue_sessions.work_unit_key non-nullable + if column_exists("active_queue_sessions", "work_unit_key"): + make_column_non_nullable_safe("active_queue_sessions", "work_unit_key") + + # Make documents.embedding non-nullable + if column_exists("documents", "embedding"): + make_column_non_nullable_safe("documents", "embedding") + + # Add primary key constraint to message_embeddings.id + if column_exists("message_embeddings", "id") and not constraint_exists( + "message_embeddings", "pk_message_embeddings", "primary" + ): + conn.execute( + sa.text( + f""" + ALTER TABLE {schema}.message_embeddings + ADD CONSTRAINT pk_message_embeddings + PRIMARY KEY (id) + """ + ) + ) + + # Rename indexes on peers table + inspector = sa.inspect(conn) + index_renames = [ + ("peers", "ix_users_created_at", "ix_peers_created_at"), + ("peers", "ix_users_name", "ix_peers_name"), + ("workspaces", "ix_apps_created_at", "ix_workspaces_created_at"), + ("workspaces", "ix_apps_name", "ix_workspaces_name"), + ] + for table_name, old_name, new_name in index_renames: + if index_exists(table_name, old_name, inspector): + conn.execute( + sa.text(f"ALTER INDEX {schema}.{old_name} RENAME TO {new_name}") + ) + + # Drop redundant indexes + if index_exists("workspaces", "ix_apps_public_id", inspector): + op.drop_index("ix_apps_public_id", table_name="workspaces", schema=schema) + if index_exists("peers", "ix_users_public_id", inspector): + op.drop_index("ix_users_public_id", table_name="peers", schema=schema) + if index_exists("sessions", "ix_sessions_public_id", inspector): + op.drop_index("ix_sessions_public_id", table_name="sessions", schema=schema) + if index_exists("documents", "ix_documents_public_id", inspector): + op.drop_index("ix_documents_public_id", table_name="documents", schema=schema) + if index_exists("collections", "ix_collections_public_id", inspector): + op.drop_index( + "ix_collections_public_id", table_name="collections", schema=schema + ) + + # Drop redundant unique constraints + if constraint_exists("workspaces", "uq_apps_public_id", "unique", inspector): + op.drop_constraint( + "uq_apps_public_id", "workspaces", type_="unique", schema=schema + ) + + if constraint_exists("peers", "uq_users_public_id", "unique", inspector): + op.drop_constraint("uq_users_public_id", "peers", type_="unique", schema=schema) + + if constraint_exists("sessions", "uq_sessions_public_id", "unique", inspector): + op.drop_constraint( + "uq_sessions_public_id", "sessions", type_="unique", schema=schema + ) + + if constraint_exists( + "collections", "uq_collections_public_id", "unique", inspector + ): + op.drop_constraint( + "uq_collections_public_id", "collections", type_="unique", schema=schema + ) + + if constraint_exists("documents", "uq_documents_public_id", "unique", inspector): + op.drop_constraint( + "uq_documents_public_id", "documents", type_="unique", schema=schema + ) + + # Drop unnecessary index on active queue + if index_exists( + "active_queue_sessions", + f"ix_{schema}_active_queue_sessions_work_unit_key", + inspector, + ): + op.drop_index( + f"ix_{schema}_active_queue_sessions_work_unit_key", + table_name="active_queue_sessions", + schema=schema, + ) + + # Add FK constraint on queue.session_id to sessions.id + if not fk_exists("queue", "fk_queue_session_id"): + # Add constraint without validation (fast, doesn't scan) + conn.execute( + sa.text( + f""" + ALTER TABLE {schema}.queue + ADD CONSTRAINT fk_queue_session_id + FOREIGN KEY (session_id) + REFERENCES {schema}.sessions(id) + NOT VALID + """ + ) + ) + # Validate constraint (scans but allows concurrent reads) + conn.execute( + sa.text( + f"ALTER TABLE {schema}.queue VALIDATE CONSTRAINT fk_queue_session_id" + ) + ) + + # Create missing indexes + if not index_exists("peers", "ix_peers_workspace_name", inspector): + op.create_index( + "ix_peers_workspace_name", "peers", ["workspace_name"], schema=schema + ) + + if not index_exists("collections", "ix_collections_workspace_name", inspector): + op.create_index( + "ix_collections_workspace_name", + "collections", + ["workspace_name"], + schema=schema, + ) + + if not index_exists("documents", "ix_documents_workspace_name", inspector): + op.create_index( + "ix_documents_workspace_name", + "documents", + ["workspace_name"], + schema=schema, + ) + + if not fk_exists("session_peers", "fk_session_peers_workspace_name", inspector): + # Add constraint without validation (fast, doesn't scan) + conn.execute( + sa.text( + f""" + ALTER TABLE {schema}.session_peers + ADD CONSTRAINT fk_session_peers_workspace_name + FOREIGN KEY (workspace_name) + REFERENCES {schema}.workspaces(name) + NOT VALID + """ + ) + ) + # Validate constraint (scans but allows concurrent reads) + conn.execute( + sa.text( + f"ALTER TABLE {schema}.session_peers VALIDATE CONSTRAINT fk_session_peers_workspace_name" + ) + ) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + + if fk_exists("session_peers", "fk_session_peers_workspace_name", inspector): + op.drop_constraint( + "fk_session_peers_workspace_name", + table_name="session_peers", + type_="foreignkey", + schema=schema, + ) + + if index_exists("documents", "ix_documents_workspace_name", inspector): + op.drop_index( + "ix_documents_workspace_name", table_name="documents", schema=schema + ) + + if index_exists("peers", "ix_peers_workspace_name", inspector): + op.drop_index("ix_peers_workspace_name", table_name="peers", schema=schema) + if index_exists("collections", "ix_collections_workspace_name", inspector): + op.drop_index( + "ix_collections_workspace_name", table_name="collections", schema=schema + ) + + # First, drop the FK constraint (we'll recreate it later if needed) + if fk_exists("queue", "fk_queue_session_id"): + op.drop_constraint( + "fk_queue_session_id", + "queue", + type_="foreignkey", + schema=schema, + ) + + if not index_exists( + "active_queue_sessions", + f"ix_{schema}_active_queue_sessions_work_unit_key", + inspector, + ): + op.create_index( + f"ix_{schema}_active_queue_sessions_work_unit_key", + table_name="active_queue_sessions", + columns=["work_unit_key"], + schema=schema, + ) + + # Recreate the redundant unique constraints + if not constraint_exists("sessions", "uq_sessions_public_id", "unique", inspector): + op.create_unique_constraint( + "uq_sessions_public_id", "sessions", ["id"], schema=schema + ) + if not constraint_exists("peers", "uq_users_public_id", "unique", inspector): + op.create_unique_constraint( + "uq_users_public_id", "peers", ["id"], schema=schema + ) + if not constraint_exists("workspaces", "uq_apps_public_id", "unique", inspector): + op.create_unique_constraint( + "uq_apps_public_id", "workspaces", ["id"], schema=schema + ) + + if not constraint_exists( + "collections", "uq_collections_public_id", "unique", inspector + ): + op.create_unique_constraint( + "uq_collections_public_id", "collections", ["id"], schema=schema + ) + if not constraint_exists( + "documents", "uq_documents_public_id", "unique", inspector + ): + op.create_unique_constraint( + "uq_documents_public_id", "documents", ["id"], schema=schema + ) + + # Recreate the redundant indexes + if not index_exists("sessions", "ix_sessions_public_id", inspector): + op.create_index("ix_sessions_public_id", "sessions", ["id"], schema=schema) + if not index_exists("peers", "ix_users_public_id", inspector): + op.create_index("ix_users_public_id", "peers", ["id"], schema=schema) + if not index_exists("workspaces", "ix_apps_public_id", inspector): + op.create_index("ix_apps_public_id", "workspaces", ["id"], schema=schema) + if not index_exists("documents", "ix_documents_public_id", inspector): + op.create_index("ix_documents_public_id", "documents", ["id"], schema=schema) + if not index_exists("collections", "ix_collections_public_id", inspector): + op.create_index( + "ix_collections_public_id", + "collections", + ["id"], + schema=schema, + ) + + # Rename indexes on peers table back to original names + index_renames = [ + ("peers", "ix_peers_created_at", "ix_users_created_at"), + ("peers", "ix_peers_name", "ix_users_name"), + ("workspaces", "ix_workspaces_created_at", "ix_apps_created_at"), + ("workspaces", "ix_workspaces_name", "ix_apps_name"), + ] + for table_name, new_name, old_name in index_renames: + if index_exists(table_name, new_name, inspector): + conn.execute( + sa.text(f"ALTER INDEX {schema}.{new_name} RENAME TO {old_name}") + ) + + # Drop primary key constraint from message_embeddings.id + if constraint_exists("message_embeddings", "pk_message_embeddings", "primary"): + op.drop_constraint( + "pk_message_embeddings", "message_embeddings", "primary", schema=schema + ) + + # Make documents.embedding nullable + if column_exists("documents", "embedding"): + op.alter_column( + "documents", + "embedding", + nullable=True, + schema=schema, + ) + + # Make active_queue_sessions.work_unit_key nullable + if column_exists("active_queue_sessions", "work_unit_key"): + op.alter_column( + "active_queue_sessions", + "work_unit_key", + nullable=True, + schema=schema, + ) + + # Make sessions.workspace_name nullable + if column_exists("sessions", "workspace_name"): + op.alter_column( + "sessions", + "workspace_name", + nullable=True, + schema=schema, + ) + + # Make peers.workspace_name nullable + if column_exists("peers", "workspace_name"): + op.alter_column( + "peers", + "workspace_name", + nullable=True, + schema=schema, + ) diff --git a/migrations/versions/08894082221a_replace_collection_name_with_observer_.py b/migrations/versions/08894082221a_replace_collection_name_with_observer_.py index 481ae33f..e38111b0 100644 --- a/migrations/versions/08894082221a_replace_collection_name_with_observer_.py +++ b/migrations/versions/08894082221a_replace_collection_name_with_observer_.py @@ -53,7 +53,7 @@ def upgrade() -> None: connection.execute( text( f""" - INSERT INTO {schema}.sessions (id, name, workspace_name, is_active) VALUES (:session_id, '__global_observations__', :workspace_name, true) ON CONFLICT DO NOTHING + INSERT INTO {schema}.sessions (id, name, workspace_name, is_active, metadata, internal_metadata, configuration, created_at) VALUES (:session_id, '__global_observations__', :workspace_name, true, '{{}}', '{{}}', '{{}}', NOW()) ON CONFLICT DO NOTHING """ ), {"session_id": session_id, "workspace_name": workspace_name}, @@ -446,7 +446,10 @@ def upgrade() -> None: schema=schema, ) - # Step 17: Drop the name column from collections + # Step 17: Drop the name_length check constraint before dropping the name column from collections + if constraint_exists("collections", "name_length", "check", inspector): + op.drop_constraint("name_length", "collections", schema=schema) + if column_exists("collections", "name", inspector): op.drop_column("collections", "name", schema=schema) diff --git a/migrations/versions/20f89a421aff_rename_metamessage_type_to_label.py b/migrations/versions/20f89a421aff_rename_metamessage_type_to_label.py index fb04c186..114eefa2 100644 --- a/migrations/versions/20f89a421aff_rename_metamessage_type_to_label.py +++ b/migrations/versions/20f89a421aff_rename_metamessage_type_to_label.py @@ -11,6 +11,8 @@ from collections.abc import Sequence import sqlalchemy as sa from alembic import op +from migrations.utils import constraint_exists + # revision identifiers, used by Alembic. revision: str = "20f89a421aff" down_revision: str | None = "556a16564f50" @@ -61,7 +63,8 @@ def upgrade() -> None: ) # Rename check constraint - op.execute("ALTER TABLE metamessages DROP CONSTRAINT metamessage_type_length;") + if constraint_exists("metamessages", "metamessage_type_length", "check"): + op.execute("ALTER TABLE metamessages DROP CONSTRAINT metamessage_type_length;") op.create_check_constraint("label_length", "metamessages", "length(label) <= 512") # ### end Alembic commands ### diff --git a/migrations/versions/b8183c5ffb48_codify_document_level_and_times_derived.py b/migrations/versions/b8183c5ffb48_codify_document_level_and_times_derived.py new file mode 100644 index 00000000..2e8c2a6b --- /dev/null +++ b/migrations/versions/b8183c5ffb48_codify_document_level_and_times_derived.py @@ -0,0 +1,164 @@ +"""codify_document_level_and_times_derived + +Revision ID: b8183c5ffb48 +Revises: ec8f94139b02 +Create Date: 2025-10-31 12:48:54.597269 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import text + +from migrations.utils import column_exists, constraint_exists, get_schema + +# revision identifiers, used by Alembic. +revision: str = "b8183c5ffb48" +down_revision: str | None = "ec8f94139b02" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +schema = get_schema() + + +def upgrade() -> None: + """Codify level and times_derived from internal_metadata into explicit columns.""" + connection = op.get_bind() + inspector = sa.inspect(connection) + + # Step 1: Add level column (nullable initially) + if not column_exists("documents", "level", inspector): + op.add_column( + "documents", + sa.Column( + "level", + sa.TEXT(), + nullable=True, + ), + schema=schema, + ) + + # Step 2: Add times_derived column (nullable initially) + if not column_exists("documents", "times_derived", inspector): + op.add_column( + "documents", + sa.Column("times_derived", sa.Integer(), nullable=True), + schema=schema, + ) + + # Step 3: Populate level and times_derived from internal_metadata in batches + # Default to 'explicit' for level and 1 for times_derived if not present in metadata + batch_size = 5000 + while True: + result = connection.execute( + text( + f""" + WITH batch AS ( + SELECT id + FROM {schema}.documents + WHERE level IS NULL OR times_derived IS NULL + LIMIT :batch_size + ) + UPDATE {schema}.documents d + SET + level = COALESCE( + d.internal_metadata->>'level', + 'explicit' + ), + times_derived = COALESCE( + (d.internal_metadata->>'times_derived')::integer, + 1 + ) + FROM batch + WHERE d.id = batch.id + """ + ), + {"batch_size": batch_size}, + ) + if result.rowcount == 0: + break + + # Step 4: Make level NOT NULL with server default + op.alter_column( + "documents", + "level", + nullable=False, + server_default=text("'explicit'"), + schema=schema, + ) + + # Step 5: Make times_derived NOT NULL with server default + op.alter_column( + "documents", + "times_derived", + nullable=False, + server_default=text("1"), + schema=schema, + ) + + # Step 6: Add CHECK constraint for level + if not constraint_exists("documents", "level_valid", "check", inspector): + op.create_check_constraint( + "level_valid", + "documents", + "level IN ('explicit', 'deductive')", + schema=schema, + ) + + +def downgrade() -> None: + """Restore level and times_derived to internal_metadata.""" + connection = op.get_bind() + inspector = sa.inspect(connection) + + # Step 1: Drop CHECK constraint for level + if constraint_exists("documents", "level_valid", "check", inspector): + op.drop_constraint( + "level_valid", + "documents", + type_="check", + schema=schema, + ) + + # Step 2: Copy level and times_derived back to internal_metadata in batches (optional, for safety) + batch_size = 5000 + while True: + result = connection.execute( + text( + f""" + WITH batch AS ( + SELECT id + FROM {schema}.documents + WHERE internal_metadata IS NULL + OR NOT (internal_metadata ? 'level') + OR NOT (internal_metadata ? 'times_derived') + LIMIT :batch_size + ) + UPDATE {schema}.documents d + SET internal_metadata = jsonb_set( + jsonb_set( + COALESCE(d.internal_metadata, '{{}}'::jsonb), + '{{level}}', + to_jsonb(d.level) + ), + '{{times_derived}}', + to_jsonb(d.times_derived) + ) + FROM batch + WHERE d.id = batch.id + """ + ), + {"batch_size": batch_size}, + ) + if result.rowcount == 0: + break + + # Step 3: Drop the level column + if column_exists("documents", "level", inspector): + op.drop_column("documents", "level", schema=schema) + + # Step 4: Drop the times_derived column + if column_exists("documents", "times_derived", inspector): + op.drop_column("documents", "times_derived", schema=schema) diff --git a/migrations/versions/e9b705f9adf9_add_server_defaults_to_timestamp_.py b/migrations/versions/e9b705f9adf9_add_server_defaults_to_timestamp_.py new file mode 100644 index 00000000..b23ba032 --- /dev/null +++ b/migrations/versions/e9b705f9adf9_add_server_defaults_to_timestamp_.py @@ -0,0 +1,387 @@ +"""add server defaults to timestamp boolean and jsonb columns + +Revision ID: e9b705f9adf9 +Revises: 066e87ca5b07 +Create Date: 2025-10-29 12:08:36.803611 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +from migrations.utils import get_schema + +# revision identifiers, used by Alembic. +revision: str = "e9b705f9adf9" +down_revision: str | None = "066e87ca5b07" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +schema = get_schema() + + +def upgrade() -> None: + # Add server defaults for timestamp columns + op.alter_column( + "workspaces", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "peers", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "sessions", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "messages", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "message_embeddings", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "collections", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "documents", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "queue", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "webhook_endpoints", + "created_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "session_peers", + "joined_at", + server_default=sa.func.now(), + schema=schema, + ) + op.alter_column( + "active_queue_sessions", + "last_updated", + server_default=sa.func.now(), + schema=schema, + ) + + # Add server defaults for JSONB columns + op.alter_column( + "workspaces", + "metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "workspaces", + "internal_metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "workspaces", + "configuration", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "peers", + "metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "peers", + "internal_metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "peers", + "configuration", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "sessions", + "metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "sessions", + "internal_metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "sessions", + "configuration", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "messages", + "metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "messages", + "internal_metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "collections", + "metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "collections", + "internal_metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "documents", + "internal_metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "session_peers", + "configuration", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + op.alter_column( + "session_peers", + "internal_metadata", + server_default=sa.text("'{}'::jsonb"), + schema=schema, + ) + + # Add server defaults for boolean columns + op.alter_column( + "sessions", + "is_active", + server_default=sa.text("true"), + schema=schema, + ) + op.alter_column( + "queue", + "processed", + server_default=sa.text("false"), + schema=schema, + ) + + +def downgrade() -> None: + # Remove server defaults for timestamp columns + op.alter_column( + "workspaces", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "peers", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "sessions", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "messages", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "message_embeddings", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "collections", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "documents", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "queue", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "webhook_endpoints", + "created_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "session_peers", + "joined_at", + server_default=None, + schema=schema, + ) + op.alter_column( + "active_queue_sessions", + "last_updated", + server_default=None, + schema=schema, + ) + + # Remove server defaults for JSONB columns + op.alter_column( + "workspaces", + "metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "workspaces", + "internal_metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "workspaces", + "configuration", + server_default=None, + schema=schema, + ) + op.alter_column( + "peers", + "metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "peers", + "internal_metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "peers", + "configuration", + server_default=None, + schema=schema, + ) + op.alter_column( + "sessions", + "metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "sessions", + "internal_metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "sessions", + "configuration", + server_default=None, + schema=schema, + ) + op.alter_column( + "messages", + "metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "messages", + "internal_metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "collections", + "metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "collections", + "internal_metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "documents", + "internal_metadata", + server_default=None, + schema=schema, + ) + op.alter_column( + "session_peers", + "configuration", + server_default=None, + schema=schema, + ) + op.alter_column( + "session_peers", + "internal_metadata", + server_default=None, + schema=schema, + ) + + # Remove server defaults for boolean columns + op.alter_column( + "sessions", + "is_active", + server_default=None, + schema=schema, + ) + op.alter_column( + "queue", + "processed", + server_default=None, + schema=schema, + ) diff --git a/migrations/versions/ec8f94139b02_codify_workspace_name_and_message_id_in_.py b/migrations/versions/ec8f94139b02_codify_workspace_name_and_message_id_in_.py new file mode 100644 index 00000000..ae7210cb --- /dev/null +++ b/migrations/versions/ec8f94139b02_codify_workspace_name_and_message_id_in_.py @@ -0,0 +1,252 @@ +"""codify workspace_name and message_id in queue table + +Revision ID: ec8f94139b02 +Revises: e9b705f9adf9 +Create Date: 2025-10-28 17:39:51.778665 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +from migrations.utils import column_exists, fk_exists, get_schema, index_exists + +# revision identifiers, used by Alembic. +revision: str = "ec8f94139b02" +down_revision: str | None = "e9b705f9adf9" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +schema = get_schema() + + +def upgrade() -> None: + # Step 1: Drop rows where workspace_name is NULL in payload + # This removes invalid/corrupted queue items + conn = op.get_bind() + batch_size = 10000 + while True: + result = conn.execute( + sa.text( + f""" + DELETE FROM "{schema}".queue + WHERE id IN ( + SELECT id FROM "{schema}".queue + WHERE payload->>'workspace_name' IS NULL + LIMIT :batch_size + ) + """ + ), + {"batch_size": batch_size}, + ) + if result.rowcount == 0: + break + + # Step 2: Add workspace_name column (nullable initially for backfill) + op.add_column( + "queue", + sa.Column("workspace_name", sa.TEXT(), nullable=True), + schema=schema, + ) + + # Step 3: Add message_id column (nullable, as not all tasks have message_id) + op.add_column( + "queue", + sa.Column("message_id", sa.BigInteger(), nullable=True), + schema=schema, + ) + + # Step 4: Backfill workspace_name and message_id from payload in batches + batch_size = 5000 + while True: + result = conn.execute( + sa.text( + f""" + WITH batch AS ( + SELECT id + FROM "{schema}".queue + WHERE workspace_name IS NULL + OR (message_id IS NULL AND payload ? 'message_id' AND payload->>'message_id' IS NOT NULL) + LIMIT :batch_size + ) + UPDATE "{schema}".queue q + SET + workspace_name = COALESCE(q.workspace_name, q.payload->>'workspace_name'), + message_id = COALESCE( + q.message_id, + CASE + WHEN q.payload ? 'message_id' AND q.payload->>'message_id' IS NOT NULL + THEN (q.payload->>'message_id')::bigint + END + ) + FROM batch + WHERE q.id = batch.id + """ + ), + {"batch_size": batch_size}, + ) + if result.rowcount == 0: + break + + # Step 5: Remove workspace_name and message_id from JSONB payloads in batches + while True: + result = conn.execute( + sa.text( + f""" + WITH batch AS ( + SELECT id + FROM "{schema}".queue + WHERE payload ? 'workspace_name' OR payload ? 'message_id' + LIMIT :batch_size + ) + UPDATE "{schema}".queue q + SET payload = q.payload - 'workspace_name' - 'message_id' + FROM batch + WHERE q.id = batch.id + """ + ), + {"batch_size": batch_size}, + ) + if result.rowcount == 0: + break + + # Step 6: Make workspace_name non-nullable + op.alter_column("queue", "workspace_name", nullable=False, schema=schema) + + # Step 7: Add foreign key constraint on workspace_name -> workspaces.name + op.create_foreign_key( + "fk_queue_workspace_name", + "queue", + "workspaces", + ["workspace_name"], + ["name"], + source_schema=schema, + referent_schema=schema, + ) + + # Step 7b: Add foreign key constraint on message_id -> messages.id + op.create_foreign_key( + "fk_queue_message_id", + "queue", + "messages", + ["message_id"], + ["id"], + source_schema=schema, + referent_schema=schema, + ) + + # Step 8: Add index on workspace_name (for FK performance and filtering) + op.create_index( + op.f("ix_queue_workspace_name"), + "queue", + ["workspace_name"], + unique=False, + schema=schema, + ) + + # Step 9: Add partial index on message_id WHERE message_id IS NOT NULL + # This optimizes JOINs with the messages table + op.create_index( + "ix_queue_message_id_not_null", + "queue", + ["message_id"], + unique=False, + schema=schema, + postgresql_where=sa.text("message_id IS NOT NULL"), + ) + + # Step 10: Add composite index on (workspace_name, processed) + # This optimizes queries that filter unprocessed items by workspace + op.create_index( + "ix_queue_workspace_name_processed", + "queue", + ["workspace_name", "processed"], + unique=False, + schema=schema, + ) + + # Step 11: Add composite index on (work_unit_key, processed, id) + # This is critical for the hot path: "get next unprocessed item for this work unit" + # Covers: WHERE work_unit_key = ? AND NOT processed ORDER BY id + op.create_index( + "ix_queue_work_unit_key_processed_id", + "queue", + ["work_unit_key", "processed", "id"], + unique=False, + schema=schema, + ) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + + # Drop indexes + if index_exists("queue", "ix_queue_work_unit_key_processed_id", inspector): + op.drop_index( + "ix_queue_work_unit_key_processed_id", table_name="queue", schema=schema + ) + + if index_exists("queue", "ix_queue_workspace_name_processed", inspector): + op.drop_index( + "ix_queue_workspace_name_processed", table_name="queue", schema=schema + ) + + if index_exists("queue", "ix_queue_message_id_not_null", inspector): + op.drop_index("ix_queue_message_id_not_null", table_name="queue", schema=schema) + + if index_exists("queue", "ix_queue_workspace_name", inspector): + op.drop_index( + op.f("ix_queue_workspace_name"), table_name="queue", schema=schema + ) + + # Drop foreign key constraints + if fk_exists("queue", "fk_queue_message_id", inspector): + op.drop_constraint("fk_queue_message_id", "queue", schema=schema) + + if fk_exists("queue", "fk_queue_workspace_name", inspector): + op.drop_constraint("fk_queue_workspace_name", "queue", schema=schema) + + # Restore workspace_name and message_id to payload in batches + conn = op.get_bind() + batch_size = 5000 + + if column_exists("queue", "workspace_name", inspector) or column_exists( + "queue", "message_id", inspector + ): + while True: + result = conn.execute( + sa.text( + f""" + WITH batch AS ( + SELECT id + FROM "{schema}".queue + WHERE (workspace_name IS NOT NULL AND NOT (payload ? 'workspace_name')) + OR (message_id IS NOT NULL AND NOT (payload ? 'message_id')) + LIMIT :batch_size + ) + UPDATE "{schema}".queue q + SET payload = q.payload + || CASE WHEN q.workspace_name IS NOT NULL AND NOT (q.payload ? 'workspace_name') + THEN jsonb_build_object('workspace_name', q.workspace_name) + ELSE '{{}}'::jsonb END + || CASE WHEN q.message_id IS NOT NULL AND NOT (q.payload ? 'message_id') + THEN jsonb_build_object('message_id', q.message_id) + ELSE '{{}}'::jsonb END + FROM batch + WHERE q.id = batch.id + """ + ), + {"batch_size": batch_size}, + ) + if result.rowcount == 0: + break + + # Drop columns + if column_exists("queue", "message_id", inspector): + op.drop_column("queue", "message_id", schema=schema) + + if column_exists("queue", "workspace_name", inspector): + op.drop_column("queue", "workspace_name", schema=schema) diff --git a/pyproject.toml b/pyproject.toml index 6c9b037c..da2400ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "honcho" -version = "2.4.1" +version = "2.4.2" description = "Honcho Server" authors = [ {name = "Plastic Labs", email = "hello@plasticlabs.ai"}, @@ -33,6 +33,8 @@ dependencies = [ "typing-extensions>=4.11.0", "prometheus-client>=0.20.0", "json-repair>=0.49.0", + "redis>=6.0.0", + "cashews[redis]==7.4.1", ] [tool.uv] dev-dependencies = [ @@ -47,6 +49,7 @@ dev-dependencies = [ "pre-commit>=4.2.0", "pytest-cov>=6.2.1", "honcho-ai", + "fakeredis>=2.32.0", ] [tool.uv.workspace] @@ -85,6 +88,15 @@ addopts = "--strict-markers --cov=src/ --cov=sdks/python/src/honcho --cov-report testpaths = ["tests"] pythonpath = ["src"] +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] [tool.basedpyright] # BasedPyright currently seems like the best type checker option, much faster diff --git a/scripts/run_alembic_tests.py b/scripts/run_alembic_tests.py new file mode 100755 index 00000000..8a2439e6 --- /dev/null +++ b/scripts/run_alembic_tests.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Run alembic tests selectively based on changed files. + +Note that this script is actively used within our precommit hooks and should not be removed. +If this script is moved, the corresponding precommit hook will need to be updated. + +This script determines which specific alembic tests to run based on: +1. If a test file changed, run the test for that revision +2. If a migration file changed, run the corresponding test + +The alembic test system uses a parameterized test in test_pipeline.py that runs +for each revision. We filter these tests using pytest's -k flag with the revision IDs. + +Usage: python scripts/run_alembic_tests.py ... +""" + +import re +import subprocess +import sys +from pathlib import Path + + +def extract_revision_id(filepath: Path) -> str | None: + """Extract the revision ID from a migration or test filename. + + Migration files: {revision_id}_{description}.py + Test files: test_{revision_id}_{description}.py + + Returns the revision_id (e.g., "05486ce795d5") or None if not found. + """ + filename = filepath.name + + # Remove .py extension + if not filename.endswith(".py"): + return None + + filename = filename[:-3] + + # Remove test_ prefix if present + if filename.startswith("test_"): + filename = filename[5:] + + # Extract revision ID (first part before underscore) + # Revision IDs are typically 12 characters of hex + match = re.match(r"^([a-f0-9]{12})_", filename) + if match: + return match.group(1) + + return None + + +def main(): + if len(sys.argv) < 2: + print("No files to check, skipping alembic tests") + sys.exit(0) + + changed_files = [Path(f) for f in sys.argv[1:]] + + # Paths + repo_root = Path(__file__).parent.parent + migrations_dir = repo_root / "migrations" / "versions" + tests_dir = repo_root / "tests" / "alembic" / "revisions" + alembic_tests_dir = repo_root / "tests" / "alembic" + + # Collect revision IDs to test + revision_ids: set[str] = set() + run_full_suite = False + + for filepath in changed_files: + filepath = Path(filepath).resolve() + + # Check if file is under tests/alembic (including subdirectories) + # If it's not a revision-specific test file, run full suite + if ( + filepath.parent == alembic_tests_dir + or alembic_tests_dir in filepath.parents + ) and not (filepath.parent == tests_dir and filepath.name.startswith("test_")): + run_full_suite = True + print( + f"Infrastructure file changed: {filepath.name} -> will run full test suite" + ) + continue + + # Case 1: Test file changed - extract its revision ID + if filepath.parent == tests_dir and filepath.name.startswith("test_"): + revision_id = extract_revision_id(filepath) + if revision_id: + revision_ids.add(revision_id) + print( + f"Test file changed: {filepath.name} -> testing revision {revision_id}" + ) + + # Case 2: Migration file changed - extract its revision ID + elif filepath.parent == migrations_dir: + revision_id = extract_revision_id(filepath) + if revision_id: + revision_ids.add(revision_id) + print( + f"Migration changed: {filepath.name} -> testing revision {revision_id}" + ) + + if run_full_suite: + # Run full test suite without -k filter + print("\nRunning full alembic test suite due to infrastructure file changes\n") + cmd = [ + "uv", + "run", + "pytest", + "tests/alembic/test_pipeline.py", + ] + elif revision_ids: + # Build a -k expression to filter tests by revision ID + # pytest -k "rev1 or rev2 or rev3" + k_expression = " or ".join(sorted(revision_ids)) + + print( + f"\nRunning tests for {len(revision_ids)} revision(s): {', '.join(sorted(revision_ids))}" + ) + print() + + # Run pytest on test_pipeline.py with -k filter + cmd = [ + "uv", + "run", + "pytest", + "tests/alembic/test_pipeline.py", + "-k", + k_expression, + ] + else: + print("No alembic tests to run") + sys.exit(0) + + result = subprocess.run(cmd, cwd=repo_root) + + sys.exit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/src/cache/__init__.py b/src/cache/__init__.py new file mode 100644 index 00000000..44e5b5fe --- /dev/null +++ b/src/cache/__init__.py @@ -0,0 +1,11 @@ +"""Cache module for Honcho.""" + +from src.cache.client import ( + close_cache, + init_cache, +) + +__all__ = [ + "close_cache", + "init_cache", +] diff --git a/src/cache/client.py b/src/cache/client.py new file mode 100644 index 00000000..8f06ca46 --- /dev/null +++ b/src/cache/client.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import asyncio +import logging +from typing import cast + +import sentry_sdk +from cashews import cache +from cashews.picklers import PicklerType +from redis import exceptions as redis_exc +from tenacity import ( + AsyncRetrying, + retry_if_exception_type, + stop_after_delay, + wait_exponential_jitter, +) + +from src.config import settings + +logger = logging.getLogger(__name__) + + +_cache_lock = asyncio.Lock() + + +def is_cache_enabled() -> bool: + return settings.CACHE.ENABLED + + +def get_cache_namespace() -> str: + # CACHE.NAMESPACE is guaranteed to be non-None by AppSettings.propagate_namespace validator + return cast(str, settings.CACHE.NAMESPACE) + + +async def init_cache() -> None: + """Initialize and verify cache connection if enabled.""" + async with _cache_lock: + # Close existing backends to force recreation with new ContextVars + await cache.close() + + if not is_cache_enabled(): + # Use in-memory cache when caching is disabled + logger.info("Cache disabled, using in-memory cache") + cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType] + return + + # Setup cache with Redis backend + try: + cache.setup( # pyright: ignore[reportUnknownMemberType] + settings.CACHE.URL, + pickle_type=PicklerType.SQLALCHEMY, + ) + + except Exception as setup_err: + logger.warning( + "Cache setup failed for %s: %s. Falling back to in-memory cache", + settings.CACHE.URL, + setup_err, + ) + if settings.SENTRY.ENABLED: + sentry_sdk.capture_exception(setup_err) + # Fallback to in-memory cache + cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType] + return + + cache.enable() + # Retry Redis ping with exponential backoff + try: + async for attempt in AsyncRetrying( + wait=wait_exponential_jitter(initial=0.2, max=2.0), + stop=stop_after_delay(5), # give it a bit more headroom + retry=retry_if_exception_type( + ( + redis_exc.TimeoutError, + redis_exc.ConnectionError, + asyncio.TimeoutError, + TimeoutError, + ) + ), + reraise=True, + ): + with attempt: + async with asyncio.timeout(2): + await cache.ping() + logger.info("Connected to cache at %s", settings.CACHE.URL) + except ( + redis_exc.TimeoutError, + redis_exc.ConnectionError, + asyncio.TimeoutError, + TimeoutError, + ) as e: + logger.warning( + "Failed to connect to cache at %s: %s. Falling back to in-memory cache", + settings.CACHE.URL, + e, + ) + if settings.SENTRY.ENABLED: + sentry_sdk.capture_exception(e) + # Fallback to in-memory cache + await cache.close() + cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType] + except Exception as e: + logger.warning( + "Unexpected cache error at %s: %s. Falling back to in-memory cache", + settings.CACHE.URL, + e, + ) + if settings.SENTRY.ENABLED: + sentry_sdk.capture_exception(e) + # Fallback to in-memory cache + await cache.close() + cache.setup("mem://", pickle_type=PicklerType.SQLALCHEMY) # pyright: ignore[reportUnknownMemberType] + + +async def close_cache() -> None: + await cache.close() + + +__all__ = [ + "init_cache", + "close_cache", + "cache", +] diff --git a/src/config.py b/src/config.py index a0219789..492d3125 100644 --- a/src/config.py +++ b/src/config.py @@ -59,6 +59,7 @@ class TomlConfigSettingsSource(PydanticBaseSettingsSource): "DB": "db", "AUTH": "auth", "SENTRY": "sentry", + "CACHE": "cache", "LLM": "llm", "DERIVER": "deriver", "PEER_CARD": "peer_card", @@ -228,6 +229,9 @@ class DeriverSettings(BackupLLMSettingsMixin, HonchoSettings): PROVIDER: SupportedProviders = "google" MODEL: str = "gemini-2.5-flash-lite" + # Whether to deduplicate documents when creating them + DEDUPLICATE: bool = True + MAX_OUTPUT_TOKENS: Annotated[int, Field(default=10_000, gt=0, le=100_000)] = 10_000 # Thinking budget tokens are only applied when using Anthropic as provider THINKING_BUDGET_TOKENS: Annotated[int, Field(default=1024, gt=0, le=5000)] = 1024 @@ -319,7 +323,22 @@ class MetricsSettings(HonchoSettings): model_config = SettingsConfigDict(env_prefix="METRICS_", extra="ignore") # pyright: ignore ENABLED: bool = False - NAMESPACE: str = "honcho" + NAMESPACE: str | None = None + + +class CacheSettings(HonchoSettings): + model_config = SettingsConfigDict(env_prefix="CACHE_", extra="ignore") # pyright: ignore + + ENABLED: bool = False + URL: str = "redis://localhost:6379/0?suppress=false" + NAMESPACE: str | None = None + DEFAULT_TTL_SECONDS: Annotated[int, Field(default=300, ge=1, le=86_400)] = ( + 300 # how long to keep items in cache + ) + + DEFAULT_LOCK_TTL_SECONDS: Annotated[int, Field(default=5, ge=1, le=86_400)] = ( + 5 # how long to hold a lock on a resource when fetching DB after cache miss + ) class DreamSettings(BackupLLMSettingsMixin, HonchoSettings): @@ -363,6 +382,8 @@ class AppSettings(HonchoSettings): COLLECT_METRICS_LOCAL: bool = False LOCAL_METRICS_FILE: str = "metrics.jsonl" + NAMESPACE: str = "honcho" # Top-level namespace for all settings, can be overridden by nested-model settings + # Nested settings models DB: DBSettings = Field(default_factory=DBSettings) AUTH: AuthSettings = Field(default_factory=AuthSettings) @@ -374,6 +395,7 @@ class AppSettings(HonchoSettings): SUMMARY: SummarySettings = Field(default_factory=SummarySettings) WEBHOOK: WebhookSettings = Field(default_factory=WebhookSettings) METRICS: MetricsSettings = Field(default_factory=MetricsSettings) + CACHE: CacheSettings = Field(default_factory=CacheSettings) DREAM: DreamSettings = Field(default_factory=DreamSettings) @field_validator("LOG_LEVEL") @@ -383,6 +405,19 @@ class AppSettings(HonchoSettings): raise ValueError(f"Invalid log level: {v}") return log_level + @model_validator(mode="after") + def propagate_namespace(self) -> "AppSettings": + """Propagate top-level NAMESPACE to nested settings if not explicitly set. + + After this validator runs, CACHE.NAMESPACE and METRICS.NAMESPACE are guaranteed + to exist. + """ + if self.CACHE.NAMESPACE is None: + self.CACHE.NAMESPACE = self.NAMESPACE + if self.METRICS.NAMESPACE is None: + self.METRICS.NAMESPACE = self.NAMESPACE + return self + # Create a single global instance of the settings settings: AppSettings = AppSettings() diff --git a/src/crud/document.py b/src/crud/document.py index 127b5d7b..8708515d 100644 --- a/src/crud/document.py +++ b/src/crud/document.py @@ -103,9 +103,10 @@ async def create_documents( *, observer: str, observed: str, + deduplicate: bool = False, ) -> int: """ - Create multiple documents with NO duplicate detection. + Create multiple documents with optional duplicate detection. Args: db: Database session @@ -120,6 +121,16 @@ async def create_documents( honcho_documents: list[models.Document] = [] for doc in documents: try: + # for each document, if deduplicate is True, perform a process + # that checks against existing documents and either rejects this document + # as a duplicate OR deletes an existing document that is a duplicate. + if deduplicate: + is_duplicate = await is_rejected_duplicate( + db, doc, workspace_name, observer=observer, observed=observed + ) + if is_duplicate: + continue + metadata_dict = doc.metadata.model_dump(exclude_none=True) honcho_documents.append( models.Document( @@ -127,6 +138,8 @@ async def create_documents( observer=observer, observed=observed, content=doc.content, + level=doc.level, + times_derived=doc.times_derived, internal_metadata=metadata_dict, embedding=doc.embedding, session_name=doc.session_name, @@ -147,3 +160,68 @@ async def create_documents( ) from e return len(honcho_documents) + + +async def is_rejected_duplicate( + db: AsyncSession, + doc: schemas.DocumentCreate, + workspace_name: str, + *, + observer: str, + observed: str, +) -> bool: + """ + Check if a document is a duplicate of an existing document. + + Uses: 1) Cosine similarity (>=0.95), 2) Token diff for retention. + + Returns True if both: + - the document is deemed a duplicate of an existing document + - the existing document is deemed a superior duplicate + + If the document is not a duplicate, returns False. + + If the document is a duplicate AND the new document is superior, + deletes the existing document and returns False. + """ + # Step 1: Find potential duplicates using cosine similarity + similar_docs = await query_documents( + db=db, + workspace_name=workspace_name, + query=doc.content, + observer=observer, + observed=observed, + max_distance=0.05, + top_k=1, + embedding=doc.embedding, + ) + + if not similar_docs: + return False + + existing_doc = similar_docs[0] + + # Step 2: Determine which has more information using token set difference + tokens_new = set(embedding_client.encoding.encode(doc.content)) + tokens_existing = set(embedding_client.encoding.encode(existing_doc.content)) + + unique_new = len(tokens_new - tokens_existing) + unique_existing = len(tokens_existing - tokens_new) + + score_new = len(tokens_new) + (unique_new * 10) + score_existing = len(tokens_existing) + (unique_existing * 10) + + # If new document has more or equal information, keep it and delete existing + if score_new >= score_existing: + logger.warning( + f"[DUPLICATE DETECTION] Deleting existing in favor of new. new='{doc.content}', existing='{existing_doc.content}'." + ) + await db.delete(existing_doc) + await db.flush() # Flush to make deletion visible in this transaction + return False # Don't reject the new document + + # Existing document has more information, reject the new one + logger.warning( + f"[DUPLICATE DETECTION] Rejecting new in favor of existing. new='{doc.content}', existing='{existing_doc.content}'." + ) + return True diff --git a/src/crud/peer.py b/src/crud/peer.py index cad234c8..acacd235 100644 --- a/src/crud/peer.py +++ b/src/crud/peer.py @@ -1,16 +1,35 @@ from logging import getLogger from typing import Any +from cashews import NOT_NONE from sqlalchemy import Select, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from src import models, schemas +from src.cache.client import cache, get_cache_namespace +from src.config import settings +from src.crud.workspace import get_or_create_workspace from src.exceptions import ConflictException, ResourceNotFoundException from src.utils.filter import apply_filter logger = getLogger(__name__) +PEER_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}:peer:{peer_name}" +PEER_LOCK_PREFIX = f"{get_cache_namespace()}:lock" + + +def peer_cache_key(workspace_name: str, peer_name: str) -> str: + """Generate cache key for peer.""" + return ( + get_cache_namespace() + + ":" + + PEER_CACHE_KEY_TEMPLATE.format( + workspace_name=workspace_name, + peer_name=peer_name, + ) + ) + async def get_or_create_peers( db: AsyncSession, @@ -35,6 +54,8 @@ async def get_or_create_peers( Raises: ConflictException: If we fail to get or create the peers """ + + await get_or_create_workspace(db, schemas.WorkspaceCreate(name=workspace_name)) peer_names = [p.name for p in peers] stmt = ( select(models.Peer) @@ -75,8 +96,6 @@ async def get_or_create_peers( try: db.add_all(new_peers) await db.commit() - # Return combined list of existing and new peers - return existing_peers + new_peers except IntegrityError: await db.rollback() if _retry: @@ -85,6 +104,37 @@ async def get_or_create_peers( ) from None return await get_or_create_peers(db, workspace_name, peers, _retry=True) + # Invalidate cache for all updated/created peers - read-through pattern + for peer_obj in existing_peers + new_peers: + cache_key = peer_cache_key(workspace_name, peer_obj.name) + await cache.delete(cache_key) + + # Return combined list of existing and new peers + return existing_peers + new_peers + + +@cache( + key=PEER_CACHE_KEY_TEMPLATE, + ttl=f"{settings.CACHE.DEFAULT_TTL_SECONDS}s", + prefix=get_cache_namespace(), + condition=NOT_NONE, +) +@cache.locked( + key=PEER_CACHE_KEY_TEMPLATE, + ttl=f"{settings.CACHE.DEFAULT_LOCK_TTL_SECONDS}s", + prefix=PEER_LOCK_PREFIX, +) +async def _fetch_peer( + db: AsyncSession, + workspace_name: str, + peer_name: str, +) -> models.Peer | None: + return await db.scalar( + select(models.Peer) + .where(models.Peer.workspace_name == workspace_name) + .where(models.Peer.name == peer_name) + ) + async def get_peer( db: AsyncSession, @@ -105,21 +155,16 @@ async def get_peer( Raises: ResourceNotFoundException: If the peer does not exist """ - # Try to get the existing peer - stmt = ( - select(models.Peer) - .where(models.Peer.workspace_name == workspace_name) - .where(models.Peer.name == peer.name) - ) - result = await db.execute(stmt) - existing_peer = result.scalar_one_or_none() + existing_peer = await _fetch_peer(db, workspace_name, peer.name) + if existing_peer is None: + raise ResourceNotFoundException( + f"Peer {peer.name} not found in workspace {workspace_name}" + ) - if existing_peer is not None: - return existing_peer + # Merge cached object into session (cached objects are detached) + existing_peer = await db.merge(existing_peer, load=False) - raise ResourceNotFoundException( - f"Peer {peer.name} not found in workspace {workspace_name}" - ) + return existing_peer async def get_peers( @@ -166,7 +211,13 @@ async def update_peer( honcho_peer.configuration = peer.configuration await db.commit() - logger.debug(f"Peer {peer_name} updated successfully") + await db.refresh(honcho_peer) + + # Invalidate cache - read-through pattern + cache_key = peer_cache_key(workspace_name, honcho_peer.name) + await cache.delete(cache_key) + + logger.debug("Peer %s updated successfully", peer_name) return honcho_peer diff --git a/src/crud/peer_card.py b/src/crud/peer_card.py index 295d0de8..f04205ed 100644 --- a/src/crud/peer_card.py +++ b/src/crud/peer_card.py @@ -7,7 +7,8 @@ from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession from src import exceptions, models, schemas -from src.crud.peer import get_peer +from src.cache.client import cache +from src.crud.peer import get_peer, peer_cache_key logger = logging.getLogger(__name__) @@ -81,14 +82,20 @@ async def set_peer_card( } ) ) + .returning(models.Peer) ) result = await db.execute(stmt) - if result.rowcount == 0: + updated_peer = result.scalar_one_or_none() + if updated_peer is None: raise exceptions.ResourceNotFoundException( f"Peer {observer} not found in workspace {workspace_name}" ) await db.commit() + # Invalidate cache - read-through pattern + cache_key = peer_cache_key(workspace_name, observer) + await cache.delete(cache_key) + def construct_peer_card_label(*, observer: str, observed: str) -> str: if observer == observed: diff --git a/src/crud/representation.py b/src/crud/representation.py index 13aefce7..14c1dd56 100644 --- a/src/crud/representation.py +++ b/src/crud/representation.py @@ -145,7 +145,6 @@ class RepresentationManager: metadata: schemas.DocumentMetadata = schemas.DocumentMetadata( message_ids=[message_id_range], - level=obs_level, premises=obs_premises, message_created_at=format_datetime_utc(message_created_at), ) @@ -154,18 +153,20 @@ class RepresentationManager: schemas.DocumentCreate( content=obs_content, session_name=session_name, + level=obs_level, metadata=metadata, embedding=embedding, ) ) - # Use bulk creation with NO duplicate detection + # Use bulk creation with optional duplicate detection new_documents = await crud.create_documents( db, documents_to_create, self.workspace_name, observer=self.observer, observed=self.observed, + deduplicate=settings.DERIVER.DEDUPLICATE, ) try: @@ -175,41 +176,6 @@ class RepresentationManager: return new_documents - async def get_relevant_observations( - self, - query: str, - *, - top_k: int = 5, - max_distance: float = 0.3, - level: str | None = None, - conversation_context: str = "", - ) -> Representation: - """ - Unified method to get relevant observations with flexible options. - - Args: - query: The search query - top_k: Number of results to return - max_distance: Maximum distance for semantic similarity - level: Optional reasoning level to filter by - conversation_context: Additional conversation context - - Returns: - Representation - """ - async with tracked_db("representation_manager.get_relevant_observations") as db: - documents = await self._get_observations_internal( - db, - query, - top_k, - max_distance, - level, - conversation_context, - ) - - # convert documents to representation - return Representation.from_documents(documents) - async def get_working_representation( self, *, @@ -330,7 +296,6 @@ class RepresentationManager: top_k: int, max_distance: float | None = None, level: str | None = None, - conversation_context: str = "", ) -> list[models.Document]: """Query documents by semantic similarity.""" try: @@ -339,7 +304,6 @@ class RepresentationManager: db, query, level, - conversation_context, top_k, max_distance, ) @@ -349,7 +313,7 @@ class RepresentationManager: workspace_name=self.workspace_name, observer=self.observer, observed=self.observed, - query=self._build_truncated_query(query, conversation_context), + query=query, max_distance=max_distance, top_k=top_k, ) @@ -397,7 +361,7 @@ class RepresentationManager: models.Document.observer == self.observer, models.Document.observed == self.observed, ) - .order_by(models.Document.internal_metadata["times_derived"].desc()) + .order_by(models.Document.times_derived.desc()) ) result = await db.execute(stmt) @@ -412,11 +376,10 @@ class RepresentationManager: top_k: int, max_distance: float, level: str | None, - conversation_context: str, ) -> list[models.Document]: """Internal method that does the actual observation retrieval.""" return await self._query_documents_semantic( - db, query, top_k, max_distance, level, conversation_context + db, query, top_k, max_distance, level ) async def _query_documents_for_level( @@ -424,7 +387,6 @@ class RepresentationManager: db: AsyncSession, query: str, level: str, - conversation_context: str, count: int, max_distance: float | None = None, ) -> list[models.Document]: @@ -434,7 +396,7 @@ class RepresentationManager: workspace_name=self.workspace_name, observer=self.observer, observed=self.observed, - query=self._build_truncated_query(query, conversation_context), + query=query, max_distance=max_distance, top_k=count * FILTER_OVERSAMPLING_FACTOR, filters=self._build_filter_conditions(level), @@ -454,76 +416,13 @@ class RepresentationManager: conditions: list[dict[str, Any]] = [] if level: - conditions.append({"internal_metadata": {"level": level}}) + conditions.append({"level": level}) if not conditions: return {} return conditions[0] if len(conditions) == 1 else {"AND": conditions} - def _build_truncated_query( - self, - query: str, - conversation_context: str = "", - max_tokens: int | None = None, - ) -> str: - """Build a query that fits within token limits with clear priorities. - - Args: - query: The search query - conversation_context: Optional conversation context to include - max_tokens: Maximum tokens allowed (defaults to setting with buffer) - - Returns: - Truncated query string that fits within token limits - """ - max_tokens = max_tokens or (settings.MAX_EMBEDDING_TOKENS - 100) - encoding = embedding_client.encoding - - # Pre-calculate all token counts once - query_prefix = "Current message: " - context_prefix = "\nContext: " - - prefix_tokens = len(encoding.encode(query_prefix)) - context_prefix_tokens = len(encoding.encode(context_prefix)) - query_tokens = encoding.encode(query) - - # Simple case: query alone fits - if prefix_tokens + len(query_tokens) <= max_tokens: - if not conversation_context: - return f"{query_prefix}{query}" - - # Try to add context - context_tokens = encoding.encode(conversation_context) - total_without_context = ( - prefix_tokens + len(query_tokens) + context_prefix_tokens - ) - - if total_without_context + len(context_tokens) <= max_tokens: - return f"{query_prefix}{query}{context_prefix}{conversation_context}" - - # Truncate context to fit - available_context_tokens = max_tokens - total_without_context - if available_context_tokens > 0: - truncated_context = encoding.decode( - context_tokens[-available_context_tokens:] - ) - return f"{query_prefix}{query}{context_prefix}{truncated_context}" - else: - # No room left for context; keep full query intact - return f"{query_prefix}{query}" - - # Query itself is too long - truncate it - available_query_tokens = max_tokens - prefix_tokens - if available_query_tokens > 0: - # Keep the end (recency) of the query - truncated_query = encoding.decode(query_tokens[-available_query_tokens:]) - return f"{query_prefix}{truncated_query}" - - # Pathological case - just return what we can - logger.warning("Token limit too restrictive: %s", max_tokens) - return encoding.decode(query_tokens[:max_tokens]) - # Module-level functions for backward compatibility and convenience diff --git a/src/crud/session.py b/src/crud/session.py index cfb77660..625626e7 100644 --- a/src/crud/session.py +++ b/src/crud/session.py @@ -1,6 +1,7 @@ from logging import getLogger from typing import Any +from cashews import NOT_NONE from nanoid import generate as generate_nanoid from sqlalchemy import Select, case, cast, func, insert, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -9,6 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.types import BigInteger, Boolean from src import models, schemas +from src.cache.client import cache, get_cache_namespace from src.config import settings from src.exceptions import ( ConflictException, @@ -18,12 +20,48 @@ from src.exceptions import ( from src.utils.filter import apply_filter from .peer import get_or_create_peers, get_peer - -# Import workspace and peer functions that are needed from .workspace import get_or_create_workspace logger = getLogger(__name__) +SESSION_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}:session:{session_name}" +SESSION_LOCK_PREFIX = f"{get_cache_namespace()}:lock" + + +def session_cache_key(workspace_name: str, session_name: str) -> str: + """Generate cache key for session.""" + return ( + get_cache_namespace() + + ":" + + SESSION_CACHE_KEY_TEMPLATE.format( + workspace_name=workspace_name, + session_name=session_name, + ) + ) + + +@cache( + key=SESSION_CACHE_KEY_TEMPLATE, + ttl=f"{settings.CACHE.DEFAULT_TTL_SECONDS}s", + prefix=get_cache_namespace(), + condition=NOT_NONE, +) +@cache.locked( + key=SESSION_CACHE_KEY_TEMPLATE, + ttl=f"{settings.CACHE.DEFAULT_LOCK_TTL_SECONDS}s", + prefix=SESSION_LOCK_PREFIX, +) +async def _fetch_session( + db: AsyncSession, + workspace_name: str, + session_name: str, +) -> models.Session | None: + return await db.scalar( + select(models.Session) + .where(models.Session.workspace_name == workspace_name) + .where(models.Session.name == session_name) + ) + def count_observers_in_config( peer_configs: dict[str, schemas.SessionPeerConfig], @@ -79,15 +117,14 @@ async def get_or_create_session( ConflictException: If we fail to get or create the session """ - stmt = ( - select(models.Session) - .where(models.Session.workspace_name == workspace_name) - .where(models.Session.name == session.name) - ) + if not session.name: + raise ValueError("Session name must be provided") - result = await db.execute(stmt) + honcho_session = await _fetch_session(db, workspace_name, session.name) - honcho_session = result.scalar_one_or_none() + # Merge cached object into session if it exists (cached objects are detached) + if honcho_session is not None: + honcho_session = await db.merge(honcho_session, load=False) # Check if session already exists if honcho_session is None: @@ -148,6 +185,12 @@ async def get_or_create_session( ) await db.commit() + await db.refresh(honcho_session) + + cache_key = session_cache_key(workspace_name, session.name) + await cache.set( + cache_key, honcho_session, expire=settings.CACHE.DEFAULT_TTL_SECONDS + ) return honcho_session @@ -170,22 +213,17 @@ async def get_session( Raises: ResourceNotFoundException: If the session does not exist """ - stmt = ( - select(models.Session) - .where(models.Session.workspace_name == workspace_name) - .where(models.Session.name == session_name) - ) + session = await _fetch_session(db, workspace_name, session_name) - result = await db.execute(stmt) - - honcho_session = result.scalar_one_or_none() - - if honcho_session is None: + if session is None: raise ResourceNotFoundException( f"Session {session_name} not found in workspace {workspace_name}" ) - return honcho_session + # Merge cached object into session (cached objects are detached) + session = await db.merge(session, load=False) + + return session async def update_session( @@ -220,6 +258,12 @@ async def update_session( honcho_session.configuration = session.configuration await db.commit() + await db.refresh(honcho_session) + + # Invalidate cache - read-through pattern + cache_key = session_cache_key(workspace_name, session_name) + await cache.delete(cache_key) + logger.debug("Session %s updated successfully", session_name) return honcho_session @@ -241,22 +285,16 @@ async def delete_session( Raises: ResourceNotFoundException: If the session does not exist """ - stmt = ( - select(models.Session) - .where(models.Session.workspace_name == workspace_name) - .where(models.Session.name == session_name) - ) - result = await db.execute(stmt) - honcho_session = result.scalar_one_or_none() - - if honcho_session is None: - logger.warning( - f"Session {session_name} not found in workspace {workspace_name}" - ) - raise ResourceNotFoundException("Session not found") + honcho_session = await get_session(db, session_name, workspace_name) honcho_session.is_active = False await db.commit() + await db.refresh(honcho_session) + + # Invalidate cache - read-through pattern + cache_key = session_cache_key(workspace_name, session_name) + await cache.delete(cache_key) + logger.debug("Session %s marked as inactive", session_name) return True @@ -359,7 +397,10 @@ async def clone_session( db.add(new_session_peer) await db.commit() + await db.refresh(new_session) logger.debug("Session %s cloned successfully", original_session_name) + + # Cache will be populated on next read - read-through pattern return new_session @@ -385,18 +426,7 @@ async def remove_peers_from_session( ResourceNotFoundException: If the session does not exist """ # Verify session exists - stmt = ( - select(models.Session) - .where(models.Session.workspace_name == workspace_name) - .where(models.Session.name == session_name) - ) - result = await db.execute(stmt) - session = result.scalar_one_or_none() - - if session is None: - raise ResourceNotFoundException( - f"Session {session_name} not found in workspace {workspace_name}" - ) + await get_session(db, session_name, workspace_name) # Soft delete specified session peers by setting left_at timestamp update_stmt = ( @@ -409,7 +439,7 @@ async def remove_peers_from_session( ) .values(left_at=func.now()) ) - result = await db.execute(update_stmt) + await db.execute(update_stmt) await db.commit() return True diff --git a/src/crud/webhook.py b/src/crud/webhook.py index 1dcc29a8..65b05569 100644 --- a/src/crud/webhook.py +++ b/src/crud/webhook.py @@ -63,7 +63,7 @@ async def get_or_create_webhook_endpoint( async def list_webhook_endpoints( - db: AsyncSession, workspace_name: str + workspace_name: str, ) -> Select[tuple[models.WebhookEndpoint]]: """ List all webhook endpoints, optionally filtered by workspace. @@ -75,9 +75,6 @@ async def list_webhook_endpoints( Returns: List of webhook endpoints """ - # Verify workspace exists - await get_workspace(db, workspace_name) - return select(models.WebhookEndpoint).where( models.WebhookEndpoint.workspace_name == workspace_name ) diff --git a/src/crud/workspace.py b/src/crud/workspace.py index d9af943c..c872b728 100644 --- a/src/crud/workspace.py +++ b/src/crud/workspace.py @@ -1,16 +1,51 @@ from logging import getLogger from typing import Any +from cashews import NOT_NONE from sqlalchemy import Select, delete, func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from src import models, schemas +from src.cache.client import cache, get_cache_namespace +from src.config import settings from src.exceptions import ConflictException, ResourceNotFoundException from src.utils.filter import apply_filter logger = getLogger(__name__) +WORKSPACE_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}" +WORKSPACE_LOCK_PREFIX = f"{get_cache_namespace()}:lock" + + +def workspace_cache_key(workspace_name: str) -> str: + """Generate cache key for workspace.""" + return ( + get_cache_namespace() + + ":" + + WORKSPACE_CACHE_KEY_TEMPLATE.format(workspace_name=workspace_name) + ) + + +@cache( + key=WORKSPACE_CACHE_KEY_TEMPLATE, + ttl=f"{settings.CACHE.DEFAULT_TTL_SECONDS}s", + prefix=get_cache_namespace(), + condition=NOT_NONE, +) +@cache.locked( + key=WORKSPACE_CACHE_KEY_TEMPLATE, + ttl=f"{settings.CACHE.DEFAULT_LOCK_TTL_SECONDS}s", + prefix=WORKSPACE_LOCK_PREFIX, +) +async def _fetch_workspace( + db: AsyncSession, workspace_name: str +) -> models.Workspace | None: + """Fetch a workspace from the database.""" + return await db.scalar( + select(models.Workspace).where(models.Workspace.name == workspace_name) + ) + async def get_or_create_workspace( db: AsyncSession, @@ -31,14 +66,17 @@ async def get_or_create_workspace( Raises: ConflictException: If we fail to get or create the workspace """ - # Try to get the existing workspace - stmt = select(models.Workspace).where(models.Workspace.name == workspace.name) - result = await db.execute(stmt) - existing_workspace = result.scalar_one_or_none() + if not workspace.name: + raise ValueError("Workspace name must be provided") + + # Check if workspace already exists + existing_workspace = await _fetch_workspace(db, workspace.name) if existing_workspace is not None: # Workspace already exists logger.debug("Found existing workspace: %s", workspace.name) + # Merge cached object into session (cached objects are detached) + existing_workspace = await db.merge(existing_workspace, load=False) return existing_workspace # Workspace doesn't exist, create a new one @@ -50,7 +88,14 @@ async def get_or_create_workspace( try: db.add(honcho_workspace) await db.commit() + await db.refresh(honcho_workspace) + logger.debug("Workspace created successfully: %s", workspace.name) + + cache_key = workspace_cache_key(workspace.name) + await cache.set( + cache_key, honcho_workspace, expire=settings.CACHE.DEFAULT_TTL_SECONDS + ) return honcho_workspace except IntegrityError: await db.rollback() @@ -94,15 +139,15 @@ async def get_workspace( Raises: ResourceNotFoundException: If the workspace does not exist """ - # Try to get the existing peer - stmt = select(models.Workspace).where(models.Workspace.name == workspace_name) - result = await db.execute(stmt) - existing_workspace = result.scalar_one_or_none() + existing_workspace = await _fetch_workspace(db, workspace_name) - if existing_workspace is not None: - return existing_workspace + if existing_workspace is None: + raise ResourceNotFoundException(f"Workspace {workspace_name} not found") - raise ResourceNotFoundException(f"Workspace {workspace_name} not found") + # Merge cached object into session (cached objects are detached) + existing_workspace = await db.merge(existing_workspace, load=False) + + return existing_workspace async def update_workspace( @@ -134,6 +179,12 @@ async def update_workspace( honcho_workspace.configuration = workspace.configuration await db.commit() + await db.refresh(honcho_workspace) + + # Invalidate cache + cache_key = workspace_cache_key(workspace_name) + await cache.delete(cache_key) + logger.debug("Workspace with id %s updated successfully", honcho_workspace.id) return honcho_workspace @@ -241,10 +292,18 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor ) await db.delete(honcho_workspace) await db.commit() + + cache_key = workspace_cache_key(workspace_name) + workspace_pattern = f"{cache_key}*" + await cache.delete_match(workspace_pattern) + logger.debug("Workspace %s deleted", workspace_name) - except Exception as e: - logger.error("Failed to delete workspace %s: %s", workspace_name, e) + except Exception: + logger.exception( + "Failed to delete workspace %s", + workspace_name, + ) await db.rollback() - raise e + raise return workspace_snapshot diff --git a/src/db.py b/src/db.py index 9d37d321..100663d2 100644 --- a/src/db.py +++ b/src/db.py @@ -46,6 +46,8 @@ SessionLocal = async_sessionmaker( ) table_schema = settings.DB.SCHEMA +# Note: column_0_N_name expands to include all columns in multi-column constraints +# e.g., "workspace_id_tenant_id" for a composite constraint on both columns meta = MetaData() meta.schema = table_schema Base = declarative_base(metadata=meta) diff --git a/src/deriver/consumer.py b/src/deriver/consumer.py index 6cafc0fc..9dbfb6bb 100644 --- a/src/deriver/consumer.py +++ b/src/deriver/consumer.py @@ -1,5 +1,4 @@ import logging -from typing import Any import sentry_sdk from pydantic import ValidationError @@ -26,8 +25,12 @@ logging.getLogger("sqlalchemy.engine.Engine").disabled = True console = Console(markup=True) -async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None: +async def process_item(queue_item: models.QueueItem) -> None: """Process a single item from the queue.""" + task_type = queue_item.task_type + queue_payload = queue_item.payload + workspace_name = queue_item.workspace_name + if task_type == "webhook": try: validated = WebhookPayload(**queue_payload) @@ -39,7 +42,7 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None: ) raise ValueError(f"Invalid payload structure: {str(e)}") from e async with tracked_db() as db: - await webhook_delivery.deliver_webhook(db, validated) + await webhook_delivery.deliver_webhook(db, validated, workspace_name) elif task_type == "summary": try: @@ -52,17 +55,21 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None: ) raise ValueError(f"Invalid payload structure: {str(e)}") from e + # Use workspace_name and message_id from QueueItem columns + message_id = queue_item.message_id + + if message_id is None: + raise ValueError("Summary tasks require a message_id") + message_public_id = validated.message_public_id if not message_public_id: - logger.debug( - "Fetching message public ID for message %s", validated.message_id - ) + logger.debug("Fetching message public ID for message %s", message_id) async with tracked_db(operation_name="summary_fallback") as db: stmt = ( select(models.Message) - .where(models.Message.workspace_name == validated.workspace_name) + .where(models.Message.workspace_name == workspace_name) .where(models.Message.session_name == validated.session_name) - .where(models.Message.id == validated.message_id) + .where(models.Message.id == message_id) ) result = await db.execute(stmt) @@ -70,22 +77,20 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None: if message is None: logger.error( "Failed to fetch message with ID %s for process_summary_task", - validated.message_id, + message_id, ) return message_public_id = message.public_id with sentry_sdk.start_transaction(name="process_summary_task", op="deriver"): await summarizer.summarize_if_needed( - validated.workspace_name, + workspace_name, validated.session_name, - validated.message_id, + message_id, validated.message_seq_in_session, message_public_id, ) - log_performance_metrics( - "summary", f"{validated.workspace_name}_{validated.message_id}" - ) + log_performance_metrics("summary", f"{workspace_name}_{message_id}") elif task_type == "dream": with sentry_sdk.start_transaction(name="process_dream_task", op="deriver"): @@ -98,7 +103,7 @@ async def process_item(task_type: str, queue_payload: dict[str, Any]) -> None: queue_payload, ) raise ValueError(f"Invalid payload structure: {str(e)}") from e - await process_dream(validated) + await process_dream(validated, workspace_name) else: raise ValueError(f"Invalid task type: {task_type}") @@ -109,29 +114,21 @@ async def process_representation_batch( observer: str | None, observed: str | None, ) -> None: - """Validate incoming queue payloads and dispatch to the appropriate handler. - - This function centralizes payload validation using a simple mapping from - task type to Pydantic model. After validation, routes the request to - the correct processor without repeating type checks elsewhere. - + """Prepares and processes a batch of messages for representation tasks. Args: - task_type: The type of task to process - queue_payloads: List of payload dictionaries to process - observed (optional): For representation tasks, the observed from work_unit_key - to identify which messages should be focused on - observer (optional): For representation tasks, the observer from work_unit_key - to identify which messages should be focused on + messages: List of messages to process + observer: The observer of the messages + observed: The observed of the messages """ if not messages or not messages[0]: - logger.debug("process_representation_batch received no payloads") + logger.debug("process_representation_batch received no messages") return if observed is None or observer is None: raise ValueError("observed and observer are required for representation tasks") logger.debug( - "process_representation_batch received %s payloads", + "process_representation_batch received %s messages", len(messages), ) diff --git a/src/deriver/enqueue.py b/src/deriver/enqueue.py index 9731b91c..cf15c5d0 100644 --- a/src/deriver/enqueue.py +++ b/src/deriver/enqueue.py @@ -11,7 +11,7 @@ from src.dreamer.dream_scheduler import get_affected_dream_keys, get_dream_sched from src.exceptions import ValidationException from src.models import QueueItem from src.utils.queue_payload import create_payload -from src.utils.work_unit import get_work_unit_key +from src.utils.work_unit import construct_work_unit_key logger = logging.getLogger(__name__) @@ -159,8 +159,16 @@ def create_representation_record( session_id: Optional session ID Returns: - Queue record dictionary + Queue record dictionary with workspace_name and message_id as separate fields """ + workspace_name = message.get("workspace_name") + message_id = message.get("message_id") + + if not isinstance(workspace_name, str): + raise TypeError("workspace_name is required and must be a string") + if not isinstance(message_id, int): + raise TypeError("message_id is required and must be an integer") + processed_payload = create_payload( message=message, task_type="representation", @@ -168,10 +176,12 @@ def create_representation_record( observed=observed, ) return { - "work_unit_key": get_work_unit_key(processed_payload), + "work_unit_key": construct_work_unit_key(workspace_name, processed_payload), "payload": processed_payload, "session_id": session_id, "task_type": "representation", + "workspace_name": workspace_name, + "message_id": message_id, } @@ -185,23 +195,32 @@ def create_summary_record( Args: message: The message payload - observed: Name of the sender - observer: Name of the target session_id: Session ID + message_seq_in_session: The sequence number of the message in the session Returns: - Queue record dictionary + Queue record dictionary with workspace_name and message_id as separate fields """ + workspace_name = message.get("workspace_name") + message_id = message.get("message_id") + + if not isinstance(workspace_name, str): + raise ValueError("workspace_name is required and must be a string") + if not isinstance(message_id, int): + raise ValueError("message_id is required and must be an integer") + processed_payload = create_payload( message=message, task_type="summary", message_seq_in_session=message_seq_in_session, ) return { - "work_unit_key": get_work_unit_key(processed_payload), + "work_unit_key": construct_work_unit_key(workspace_name, processed_payload), "payload": processed_payload, "session_id": session_id, "task_type": "summary", + "workspace_name": workspace_name, + "message_id": message_id, } diff --git a/src/deriver/prompts.py b/src/deriver/prompts.py index 2ef199eb..4a414f88 100644 --- a/src/deriver/prompts.py +++ b/src/deriver/prompts.py @@ -65,11 +65,9 @@ Current understanding of {peer_id}: You are an agent who critically analyzes messages from {peer_id} through rigorous logical reasoning to produce only conclusions about them that are CERTAIN. TARGET USER TO ANALYZE -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ You are analyzing: {peer_id} The conversation may include messages from multiple participants, but you MUST focus ONLY on deriving conclusions about {peer_id}. Only use other participants' messages as context for understanding {peer_id}. -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ IMPORTANT NAMING RULES • When you write a conclusion about {peer_id}, always start the sentence with their name (e.g. "Anthony is 25 years old"). @@ -87,7 +85,7 @@ Here are strict definitions for the reasoning modes you are to employ: - Current date and time (which is: {message_created_at}) - Timestamps from conversation history - Follow strict literal necessity--if stated directly in message, extract a conclusion - - Latest message MUST be a premise, previous messages and timestamps may be used to contextualize + - New turn MUST be a premise, previous messages and timestamps may be used to contextualize - Transforms a single message (premise) into ONE OR MULTIPLE conclusions - Derive EVERYTHING that can be explicitly concluded - Make sure EVERY conclusion is sufficiently contextualized, i.e. ensure each conclusion contains enough specific information about subjects and objects to make it self-contained and useful (e.g. instead of "Ann is nervous about the interview", use "Ann is nervous about the job interview at the pharmacy") @@ -128,7 +126,7 @@ Based on our definitions and examples, here's a summary of the logical reasoning - Explicit/Deductive/Temporal/General → Further Deductive: Can use certain conclusions and known facts to deduce additional certain conclusions - Probabilistic Deductive ↛ Further Deductive: If a deductive conclusion includes probabilistic qualifiers (likely, potentially, typically, might, etc) it may NOT be used as a premise for further deductions -**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to the latest message, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions. +**INSTRUCTIONS:** Given the above, first think critically about what it means to do explicit and deductive reasoning, then consider how to apply that to all new turns, finally do explicit and deductive reasoning about the user to reach useful, contextually-rich conclusions. You must extract observations from all new turns. {peer_card_section} diff --git a/src/deriver/queue_manager.py b/src/deriver/queue_manager.py index 356ae3ba..df2f1465 100644 --- a/src/deriver/queue_manager.py +++ b/src/deriver/queue_manager.py @@ -10,12 +10,13 @@ import sentry_sdk from dotenv import load_dotenv from nanoid import generate as generate_nanoid from sentry_sdk.integrations.asyncio import AsyncioIntegration -from sqlalchemy import BigInteger, and_, delete, select, update +from sqlalchemy import and_, delete, select, update from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql import func from src import models, prometheus +from src.cache.client import close_cache, init_cache from src.config import settings from src.dependencies import tracked_db from src.deriver.consumer import ( @@ -463,9 +464,7 @@ class QueueManager: break try: - await process_item( - work_unit.task_type, queue_item.payload - ) + await process_item(queue_item) await self.mark_queue_items_as_processed( [queue_item], work_unit_key ) @@ -614,8 +613,7 @@ class QueueManager: .select_from(models.QueueItem) .join( models.Message, - func.cast(models.QueueItem.payload["message_id"].astext, BigInteger) - == models.Message.id, + models.QueueItem.message_id == models.Message.id, ) .where(~models.QueueItem.processed) .where(models.Message.session_name == parsed_key.session_name) @@ -661,10 +659,7 @@ class QueueManager: and_( models.QueueItem.work_unit_key == work_unit_key, ~models.QueueItem.processed, - func.cast( - models.QueueItem.payload["message_id"].astext, BigInteger - ) - == models.Message.id, + models.QueueItem.message_id == models.Message.id, ), ) .where(allowed_condition) @@ -689,7 +684,11 @@ class QueueManager: if items_to_process: max_queue_item_message_id = max( - [qi.payload["message_id"] for qi in items_to_process] + [ + qi.message_id + for qi in items_to_process + if qi.message_id is not None + ] ) messages_context = [ # remove any messages that are after the last message_id from queue items m for m in messages_context if m.id <= max_queue_item_message_id @@ -766,6 +765,13 @@ class QueueManager: async def main(): logger.debug("Starting queue manager") + try: + await init_cache() + except Exception as e: + logger.warning( + "Error initializing cache in queue manager; proceeding without cache: %s", e + ) + manager = QueueManager() try: await manager.initialize() @@ -773,4 +779,5 @@ async def main(): logger.error(f"Error in main: {str(e)}") sentry_sdk.capture_exception(e) finally: + await close_cache() logger.debug("Main function exiting") diff --git a/src/dreamer/dream_scheduler.py b/src/dreamer/dream_scheduler.py index a56edd3f..f28dead9 100644 --- a/src/dreamer/dream_scheduler.py +++ b/src/dreamer/dream_scheduler.py @@ -12,7 +12,7 @@ from src import models from src.config import settings from src.dependencies import tracked_db from src.utils.queue_payload import create_dream_payload -from src.utils.work_unit import get_work_unit_key, parse_work_unit_key +from src.utils.work_unit import construct_work_unit_key, parse_work_unit_key logger = getLogger(__name__) @@ -48,13 +48,13 @@ def get_affected_dream_keys(message: dict[str, Any]) -> list[str]: return [] # Generate dream work unit key for this peer's collection - dream_key = get_work_unit_key( + dream_key = construct_work_unit_key( + workspace_name, { "task_type": "dream", - "workspace_name": workspace_name, "observer": peer_name, "observed": peer_name, - } + }, ) return [dream_key] @@ -193,7 +193,6 @@ class DreamScheduler: ) -> None: """Execute the dream by enqueueing it and updating collection metadata.""" dream_payload = create_dream_payload( - workspace_name=workspace_name, dream_type="consolidate", observer=observer, observed=observed, @@ -205,6 +204,8 @@ class DreamScheduler: "payload": dream_payload, "session_id": None, "task_type": "dream", + "workspace_name": workspace_name, + "message_id": None, # Dreams don't have a message_id } await db.execute(insert(models.QueueItem), [dream_record]) @@ -325,13 +326,13 @@ async def check_and_schedule_dream( dream_scheduler = get_dream_scheduler() if dream_scheduler: - collection_work_unit_key = get_work_unit_key( + collection_work_unit_key = construct_work_unit_key( + collection.workspace_name, { "task_type": "dream", - "workspace_name": collection.workspace_name, "observer": collection.observer, "observed": collection.observed, - } + }, ) await dream_scheduler.schedule_dream( diff --git a/src/dreamer/dreamer.py b/src/dreamer/dreamer.py index 19a5751e..5da9d1b6 100644 --- a/src/dreamer/dreamer.py +++ b/src/dreamer/dreamer.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) @sentry_sdk.trace async def process_dream( payload: DreamPayload, + workspace_name: str, ) -> None: """ Process a dream task by performing collection maintenance operations. @@ -32,12 +33,12 @@ async def process_dream( payload: The dream task payload containing workspace, peer, and dream type information """ logger.info( - f"Processing dream task: {payload.dream_type} for {payload.workspace_name}/{payload.observer}/{payload.observed}" + f"Processing dream task: {payload.dream_type} for {workspace_name}/{payload.observer}/{payload.observed}" ) try: if payload.dream_type == "consolidate": - await _process_consolidate_dream(payload) + await _process_consolidate_dream(payload, workspace_name) ## TODO other dream types except Exception as e: @@ -50,7 +51,9 @@ async def process_dream( # Don't re-raise - we want to mark the dream task as processed even if it fails -async def _process_consolidate_dream(payload: DreamPayload) -> None: +async def _process_consolidate_dream( + payload: DreamPayload, workspace_name: str +) -> None: """ Process a consolidation dream task. @@ -63,7 +66,7 @@ async def _process_consolidate_dream(payload: DreamPayload) -> None: logger.info( f""" (っ- ‸ - ς)ᶻ z 𐰁 ᶻ z 𐰁 ᶻ z 𐰁\n -DREAM: consolidating documents for {payload.workspace_name}/{payload.observer}/{payload.observed}\n +DREAM: consolidating documents for {workspace_name}/{payload.observer}/{payload.observed}\n 𐰁 z ᶻ 𐰁 z ᶻ 𐰁 z ᶻ(っ- ‸ - ς)""" ) @@ -71,7 +74,7 @@ DREAM: consolidating documents for {payload.workspace_name}/{payload.observer}/{ async with tracked_db("dream_consolidate") as db: documents = await crud.get_all_documents( db, - payload.workspace_name, + workspace_name, observer=payload.observer, observed=payload.observed, ) @@ -88,7 +91,7 @@ DREAM: consolidating documents for {payload.workspace_name}/{payload.observer}/{ for cluster in clusters: await _consolidate_cluster( cluster, - payload.workspace_name, + workspace_name, db, observer=payload.observer, observed=payload.observed, @@ -118,9 +121,7 @@ async def _consolidate_cluster( logger.info("consolidated representation:\n%s", consolidated_representation) # TODO: less hacky preservation of times_derived - total_times_derived = sum( - doc.internal_metadata.get("times_derived", 1) for doc in cluster - ) + total_times_derived = sum(doc.times_derived for doc in cluster) new_documents = [ *consolidated_representation.explicit, @@ -141,10 +142,8 @@ async def _consolidate_cluster( # NOTE: other kinds of observations here in the future metadata = schemas.DocumentMetadata( - times_derived=total_times_derived, message_ids=obs.message_ids, message_created_at=format_datetime_utc(obs.created_at), - level=level, premises=premises, ) @@ -154,6 +153,8 @@ async def _consolidate_cluster( schemas.DocumentCreate( content=content, session_name=obs.session_name, + level=level, + times_derived=total_times_derived, metadata=metadata, embedding=embedding, ) diff --git a/src/main.py b/src/main.py index 412822d2..bb354dec 100644 --- a/src/main.py +++ b/src/main.py @@ -16,6 +16,7 @@ from sentry_sdk.integrations.fastapi import FastApiIntegration from sentry_sdk.integrations.starlette import StarletteIntegration from src import prometheus +from src.cache.client import close_cache, init_cache from src.config import settings from src.db import engine, request_context from src.exceptions import HonchoException @@ -109,8 +110,18 @@ if SENTRY_ENABLED: @asynccontextmanager async def lifespan(_: FastAPI): - yield - await engine.dispose() + try: + await init_cache() + except Exception as e: + logger.warning( + "Error initializing cache in api process; proceeding without cache: %s", e + ) + + try: + yield + finally: + await close_cache() + await engine.dispose() app = FastAPI( @@ -123,7 +134,7 @@ app = FastAPI( title="Honcho API", summary="The Identity Layer for the Agentic World", description="""Honcho is a platform for giving agents user-centric memory and social cognition""", - version="2.4.1", + version="2.4.2", contact={ "name": "Plastic Labs", "url": "https://honcho.dev", diff --git a/src/models.py b/src/models.py index b66e31cf..f0042cd0 100644 --- a/src/models.py +++ b/src/models.py @@ -1,6 +1,6 @@ import datetime from logging import getLogger -from typing import Any, Literal, final +from typing import Any, final from dotenv import load_dotenv from nanoid import generate as generate_nanoid @@ -26,6 +26,8 @@ from sqlalchemy.orm.properties import MappedColumn from sqlalchemy.sql import func from typing_extensions import override +from src.utils.types import DocumentLevel, TaskType + from .db import Base load_dotenv(override=True) @@ -51,13 +53,25 @@ session_peers_table = Table( nullable=False, ), Column("peer_name", TEXT, primary_key=True, nullable=False), - Column("configuration", JSONB, default=dict), - Column("internal_metadata", JSONB, default=dict), + Column( + "configuration", + JSONB, + default=dict, + nullable=False, + server_default=text("'{}'::jsonb"), + ), + Column( + "internal_metadata", + JSONB, + default=dict, + nullable=False, + server_default=text("'{}'::jsonb"), + ), Column( "joined_at", DateTime(timezone=True), nullable=False, - default=func.now(), + server_default=func.now(), ), Column( "left_at", @@ -81,22 +95,28 @@ session_peers_table = Table( class Workspace(Base): __tablename__: str = "workspaces" id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True) - name: Mapped[str] = mapped_column(TEXT, index=True, unique=True) + name: Mapped[str] = mapped_column(TEXT, unique=True) peers = relationship("Peer", back_populates="workspace") webhook_endpoints = relationship("WebhookEndpoint", back_populates="workspace") created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() + ) + h_metadata: Mapped[dict[str, Any]] = mapped_column( + "metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") ) - h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict) internal_metadata: Mapped[dict[str, Any]] = mapped_column( "internal_metadata", JSONB, default=dict ) - configuration: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict) + configuration: Mapped[dict[str, Any]] = mapped_column( + JSONB, default=dict, server_default=text("'{}'::jsonb") + ) __table_args__ = ( CheckConstraint("length(id) = 21", name="id_length"), CheckConstraint("length(name) <= 512", name="name_length"), CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"), + Index("ix_workspaces_created_at", "created_at"), + Index("ix_workspaces_name", "name"), ) @@ -104,18 +124,22 @@ class Workspace(Base): class Peer(Base): __tablename__: str = "peers" id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True) - name: Mapped[str] = mapped_column(TEXT, index=True) - h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict) + name: Mapped[str] = mapped_column(TEXT, nullable=False) + h_metadata: Mapped[dict[str, Any]] = mapped_column( + "metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") + ) internal_metadata: Mapped[dict[str, Any]] = mapped_column( - "internal_metadata", JSONB, default=dict + "internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") ) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() ) workspace_name: Mapped[str] = mapped_column( - ForeignKey("workspaces.name"), index=True, nullable=False + ForeignKey("workspaces.name"), nullable=False + ) + configuration: Mapped[dict[str, Any]] = mapped_column( + JSONB, default=dict, server_default=text("'{}'::jsonb") ) - configuration: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict) workspace = relationship("Workspace", back_populates="peers") sessions = relationship( @@ -128,6 +152,9 @@ class Peer(Base): CheckConstraint("length(name) <= 512", name="name_length"), CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"), Index("idx_peers_workspace_lookup", "workspace_name", "name"), + Index("ix_peers_created_at", "created_at"), + Index("ix_peers_name", "name"), + Index("ix_peers_workspace_name", "workspace_name"), ) def __repr__(self) -> str: @@ -138,20 +165,24 @@ class Peer(Base): class Session(Base): __tablename__: str = "sessions" id: Mapped[str] = mapped_column(TEXT, primary_key=True, default=generate_nanoid) - name: Mapped[str] = mapped_column(TEXT, index=True) - is_active: Mapped[bool] = mapped_column(default=True) - h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict) + name: Mapped[str] = mapped_column(TEXT) + is_active: Mapped[bool] = mapped_column(default=True, server_default=text("true")) + h_metadata: Mapped[dict[str, Any]] = mapped_column( + "metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") + ) internal_metadata: Mapped[dict[str, Any]] = mapped_column( - "internal_metadata", JSONB, default=dict + "internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") ) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() ) messages = relationship("Message", back_populates="session") workspace_name: Mapped[str] = mapped_column( - ForeignKey("workspaces.name"), index=True, nullable=False + ForeignKey("workspaces.name"), nullable=False + ) + configuration: Mapped[dict[str, Any]] = mapped_column( + JSONB, default=dict, server_default=text("'{}'::jsonb") ) - configuration: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict) peers = relationship( "Peer", secondary=session_peers_table, back_populates="sessions" @@ -162,6 +193,7 @@ class Session(Base): CheckConstraint("length(name) <= 512", name="name_length"), CheckConstraint("length(id) = 21", name="id_length"), CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"), + Index("ix_sessions_created_at", "created_at"), ) def __repr__(self) -> str: @@ -175,26 +207,30 @@ class Message(Base): BigInteger, Identity(), primary_key=True, autoincrement=True ) public_id: Mapped[str] = mapped_column( - TEXT, index=True, unique=True, default=generate_nanoid + TEXT, + unique=True, + default=generate_nanoid, ) # NOTE: Messages in Honcho 2.0 could historically be stored outside of a session. # We have since assigned all of these messages to a default session. - session_name: Mapped[str] = mapped_column(index=True, nullable=False) + session_name: Mapped[str] = mapped_column(TEXT, nullable=False) content: Mapped[str] = mapped_column(TEXT) - h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict) + h_metadata: Mapped[dict[str, Any]] = mapped_column( + "metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") + ) internal_metadata: Mapped[dict[str, Any]] = mapped_column( - "internal_metadata", JSONB, default=dict + "internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") ) token_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - seq_in_session: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True) + seq_in_session: Mapped[int] = mapped_column(BigInteger, nullable=False) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() ) session = relationship("Session", back_populates="messages") - peer_name: Mapped[str] = mapped_column(index=True) + peer_name: Mapped[str] = mapped_column(TEXT) workspace_name: Mapped[str] = mapped_column( - ForeignKey("workspaces.name"), index=True + ForeignKey("workspaces.name"), ) __table_args__ = ( @@ -229,6 +265,11 @@ class Message(Base): text("to_tsvector('english', content)"), postgresql_using="gin", ), + Index("ix_messages_created_at", "created_at"), + Index("ix_messages_id", "id"), + Index("ix_messages_peer_name", "peer_name"), + Index("ix_messages_public_id", "public_id"), + Index("ix_messages_workspace_name", "workspace_name"), ) @override @@ -246,15 +287,15 @@ class MessageEmbedding(Base): content: Mapped[str] = mapped_column(TEXT) embedding: MappedColumn[Any] = mapped_column(Vector(1536)) message_id: Mapped[str] = mapped_column( - ForeignKey("messages.public_id"), index=True + ForeignKey("messages.public_id"), ) workspace_name: Mapped[str] = mapped_column( - ForeignKey("workspaces.name"), index=True + ForeignKey("workspaces.name"), ) - session_name: Mapped[str] = mapped_column(TEXT, index=True, nullable=False) - peer_name: Mapped[str | None] = mapped_column(TEXT, index=True) + session_name: Mapped[str] = mapped_column(TEXT, nullable=False) + peer_name: Mapped[str] = mapped_column(TEXT) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() ) # Relationship to Message @@ -278,6 +319,11 @@ class MessageEmbedding(Base): postgresql_with={"m": 16, "ef_construction": 64}, postgresql_ops={"embedding": "vector_cosine_ops"}, ), + Index("idx_message_embeddings_created_at", "created_at"), + Index("idx_message_embeddings_message_id", "message_id"), + Index("idx_message_embeddings_peer_name", "peer_name"), + Index("idx_message_embeddings_session_name", "session_name"), + Index("idx_message_embeddings_workspace_name", "workspace_name"), ) @@ -286,20 +332,22 @@ class Collection(Base): __tablename__: str = "collections" id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True) - observer: Mapped[str] = mapped_column(TEXT, index=True) - observed: Mapped[str] = mapped_column(TEXT, index=True) + observer: Mapped[str] = mapped_column(TEXT) + observed: Mapped[str] = mapped_column(TEXT) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() + ) + h_metadata: Mapped[dict[str, Any]] = mapped_column( + "metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") ) - h_metadata: Mapped[dict[str, Any]] = mapped_column("metadata", JSONB, default=dict) internal_metadata: Mapped[dict[str, Any]] = mapped_column( - "internal_metadata", JSONB, default=dict + "internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") ) documents = relationship( "Document", back_populates="collection", cascade="all, delete, delete-orphan" ) workspace_name: Mapped[str] = mapped_column( - ForeignKey("workspaces.name"), index=True + ForeignKey("workspaces.name"), ) __table_args__ = ( @@ -321,6 +369,10 @@ class Collection(Base): ["observed", "workspace_name"], ["peers.name", "peers.workspace_name"], ), + Index("idx_collections_observer", "observer"), + Index("idx_collections_observed", "observed"), + Index("ix_collections_created_at", "created_at"), + Index("ix_collections_workspace_name", "workspace_name"), ) @@ -329,26 +381,31 @@ class Document(Base): __tablename__: str = "documents" id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True) internal_metadata: Mapped[dict[str, Any]] = mapped_column( - "internal_metadata", JSONB, default=dict + "internal_metadata", JSONB, default=dict, server_default=text("'{}'::jsonb") ) content: Mapped[str] = mapped_column(TEXT) + level: Mapped[DocumentLevel] = mapped_column( + TEXT, nullable=False, server_default="explicit" + ) + times_derived: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("1") + ) embedding: MappedColumn[Any] = mapped_column(Vector(1536)) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() ) - observer: Mapped[str] = mapped_column(TEXT, index=True) - observed: Mapped[str] = mapped_column(TEXT, index=True) - workspace_name: Mapped[str] = mapped_column( - ForeignKey("workspaces.name"), index=True - ) - session_name: Mapped[str] = mapped_column(TEXT, index=True) + observer: Mapped[str] = mapped_column(TEXT) + observed: Mapped[str] = mapped_column(TEXT) + workspace_name: Mapped[str] = mapped_column(ForeignKey("workspaces.name")) + session_name: Mapped[str] = mapped_column(TEXT) collection = relationship("Collection", back_populates="documents") __table_args__ = ( CheckConstraint("length(id) = 21", name="id_length"), CheckConstraint("length(content) <= 65535", name="content_length"), CheckConstraint("id ~ '^[A-Za-z0-9_-]+$'", name="id_format"), + CheckConstraint("level IN ('explicit', 'deductive')", name="level_valid"), # Composite foreign key constraint for collections ForeignKeyConstraint( ["observer", "observed", "workspace_name"], @@ -383,33 +440,62 @@ class Document(Base): "embedding": "vector_cosine_ops" }, # Cosine distance operator ), + Index("idx_documents_observer", "observer"), + Index("idx_documents_observed", "observed"), + Index("idx_documents_session_name", "session_name"), + Index("ix_documents_created_at", "created_at"), + Index("ix_documents_workspace_name", "workspace_name"), ) -TaskType = Literal["webhook", "summary", "representation", "dream"] - - @final class QueueItem(Base): __tablename__: str = "queue" id: Mapped[int] = mapped_column( BigInteger, Identity(), primary_key=True, autoincrement=True ) - session_id: Mapped[str] = mapped_column( - ForeignKey("sessions.id"), index=True, nullable=True - ) + session_id: Mapped[str] = mapped_column(ForeignKey("sessions.id"), nullable=True) work_unit_key: Mapped[str] = mapped_column(TEXT, nullable=False) task_type: Mapped[TaskType] = mapped_column(TEXT, nullable=False) payload: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) - processed: Mapped[bool] = mapped_column(Boolean, default=False) + processed: Mapped[bool] = mapped_column( + Boolean, default=False, server_default=text("false") + ) error: Mapped[str | None] = mapped_column(TEXT, nullable=True) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), index=True, default=func.now() + DateTime(timezone=True), server_default=func.now() + ) + workspace_name: Mapped[str] = mapped_column( + ForeignKey("workspaces.name"), nullable=False + ) + message_id: Mapped[int | None] = mapped_column( + BigInteger, ForeignKey("messages.id"), nullable=True + ) + + __table_args__ = ( + Index("ix_queue_created_at", "created_at"), + Index("ix_queue_session_id", "session_id"), + Index( + "ix_queue_workspace_name", + "workspace_name", + ), + Index( + "ix_queue_message_id_not_null", + "message_id", + postgresql_where=text("message_id IS NOT NULL"), + ), + Index("ix_queue_workspace_name_processed", "workspace_name", "processed"), + Index( + "ix_queue_work_unit_key_processed_id", + "work_unit_key", + "processed", + "id", + ), ) def __repr__(self) -> str: - return f"QueueItem(id={self.id}, session_id={self.session_id}, work_unit_key={self.work_unit_key}, task_type={self.task_type}, payload={self.payload}, processed={self.processed})" + return f"QueueItem(id={self.id}, session_id={self.session_id}, work_unit_key={self.work_unit_key}, task_type={self.task_type}, payload={self.payload}, processed={self.processed}, workspace_name={self.workspace_name}, message_id={self.message_id})" @final @@ -418,10 +504,10 @@ class ActiveQueueSession(Base): id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True) - work_unit_key: Mapped[str] = mapped_column(TEXT, unique=True, index=True) + work_unit_key: Mapped[str] = mapped_column(TEXT, unique=True) last_updated: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), default=func.now(), onupdate=func.now() + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() ) @@ -430,11 +516,11 @@ class WebhookEndpoint(Base): __tablename__: str = "webhook_endpoints" id: Mapped[str] = mapped_column(TEXT, default=generate_nanoid, primary_key=True) workspace_name: Mapped[str] = mapped_column( - ForeignKey("workspaces.name"), index=True, nullable=False + ForeignKey("workspaces.name"), nullable=False ) url: Mapped[str] = mapped_column(TEXT, nullable=False) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime(timezone=True), default=func.now() + DateTime(timezone=True), server_default=func.now() ) workspace = relationship("Workspace", back_populates="webhook_endpoints") diff --git a/src/prometheus.py b/src/prometheus.py index 85bda9ec..f19d30f3 100644 --- a/src/prometheus.py +++ b/src/prometheus.py @@ -5,6 +5,7 @@ This module defines all Prometheus metrics for all Honcho processes and exposes """ import logging +from typing import cast from prometheus_client import ( CONTENT_TYPE_LATEST, @@ -28,7 +29,8 @@ class NamespacedCounter(Counter): def labels(self, **kwargs: str) -> "NamespacedCounter": """Override labels to automatically appends namespace label""" - kwargs["namespace"] = settings.METRICS.NAMESPACE + # METRICS.NAMESPACE is guaranteed to be non-None by AppSettings.propagate_namespace validator + kwargs["namespace"] = cast(str, settings.METRICS.NAMESPACE) return super().labels(**kwargs) diff --git a/src/routers/messages.py b/src/routers/messages.py index 259fc9d7..5cbc9efe 100644 --- a/src/routers/messages.py +++ b/src/routers/messages.py @@ -71,7 +71,7 @@ async def create_messages_for_session( "peer_name": message.peer_name, "created_at": message.created_at, "message_public_id": message.public_id, - "message_seq_in_session": message.seq_in_session, + "seq_in_session": message.seq_in_session, } for message in created_messages ] @@ -135,7 +135,7 @@ async def create_messages_with_file( "peer_name": message.peer_name, "created_at": message.created_at, "message_public_id": message.public_id, - "message_seq_in_session": message.seq_in_session, + "seq_in_session": message.seq_in_session, } for message in created_messages ] diff --git a/src/routers/webhooks.py b/src/routers/webhooks.py index c6ae2017..aa3256c1 100644 --- a/src/routers/webhooks.py +++ b/src/routers/webhooks.py @@ -61,7 +61,7 @@ async def list_webhook_endpoints( if not jwt_params.ad and jwt_params.w is not None and jwt_params.w != workspace_id: raise AuthenticationException("Unauthorized access to resource") - stmt = await crud.list_webhook_endpoints(db, workspace_id) + stmt = await crud.list_webhook_endpoints(workspace_id) return await apaginate(db, stmt) diff --git a/src/schemas.py b/src/schemas.py index a1eac978..4d99dd42 100644 --- a/src/schemas.py +++ b/src/schemas.py @@ -288,20 +288,12 @@ class DocumentBase(BaseModel): class DocumentMetadata(BaseModel): - times_derived: int | None = Field( - default=None, - ge=1, - description="The number of times that a semantic duplicate document to this one has been derived", - ) message_ids: list[tuple[int, int]] = Field( description="The ID range(s) of the messages that this document was derived from. Acts as a link to the primary source of the document. Note that as a document gets deduplicated, additional ranges will be added, because the same document could be derived from completely separate message ranges." ) message_created_at: str = Field( description="The timestamp of the message that this document was derived from. Note that this is not the same as the created_at timestamp of the document. This timestamp is usually only saved with second-level precision." ) - level: Literal["explicit", "deductive"] = Field( - description="The level of the document (explicit or deductive)" - ) premises: list[str] | None = Field( default=None, description="The premises of the deduction -- only applicable for deductive observations", @@ -313,6 +305,15 @@ class DocumentCreate(DocumentBase): session_name: str = Field( description="The session from which the document was derived" ) + level: Literal["explicit", "deductive"] = Field( + default="explicit", + description="The level of the document (explicit or deductive)", + ) + times_derived: int = Field( + default=1, + ge=1, + description="The number of times that a semantic duplicate document to this one has been derived", + ) metadata: DocumentMetadata = Field() embedding: list[float] = Field() diff --git a/src/utils/queue_payload.py b/src/utils/queue_payload.py index e4d4a964..b267f04a 100644 --- a/src/utils/queue_payload.py +++ b/src/utils/queue_payload.py @@ -14,9 +14,7 @@ class RepresentationPayload(BasePayload): """Payload for representation tasks.""" task_type: Literal["representation"] = "representation" - workspace_name: str session_name: str - message_id: int content: str observer: str observed: str @@ -33,9 +31,7 @@ class SummaryPayload(BasePayload): """Payload for summary tasks.""" task_type: Literal["summary"] = "summary" - workspace_name: str session_name: str - message_id: int message_seq_in_session: int # Optional for backward compatibility with older queue items message_public_id: str | None = None @@ -45,7 +41,6 @@ class WebhookPayload(BasePayload): """Payload for webhook delivery tasks.""" task_type: Literal["webhook"] = "webhook" - workspace_name: str event_type: str data: dict[str, Any] @@ -54,35 +49,33 @@ class DreamPayload(BasePayload): """Payload for dream tasks.""" task_type: Literal["dream"] = "dream" - workspace_name: str dream_type: Literal["consolidate"] = "consolidate" observer: str observed: str def create_webhook_payload( - workspace_name: str, event_type: str, data: dict[str, Any], ) -> dict[str, Any]: - return WebhookPayload( - workspace_name=workspace_name, event_type=event_type, data=data - ).model_dump(mode="json") + """Create a webhook payload.""" + return WebhookPayload(event_type=event_type, data=data).model_dump( + mode="json", exclude_none=True + ) def create_dream_payload( - workspace_name: str, dream_type: Literal["consolidate"] = "consolidate", *, observer: str, observed: str, ) -> dict[str, Any]: + """Create a dream payload.""" return DreamPayload( - workspace_name=workspace_name, dream_type=dream_type, observer=observer, observed=observed, - ).model_dump(mode="json") + ).model_dump(mode="json", exclude_none=True) def create_payload( @@ -96,6 +89,10 @@ def create_payload( """ Create a processed payload from a message for queue processing. + Note: workspace_name and message_id are no longer included in the returned payload + as they are now stored in dedicated columns on the queue table. The caller is + responsible for extracting and passing these values separately. + Args: message: The original message dictionary task_type: Type of task ('representation' or 'summary') @@ -104,7 +101,7 @@ def create_payload( message_seq_in_session: Required for summary tasks, must be None for representation Returns: - Processed payload dictionary ready for queue processing + Processed payload dictionary ready for queue processing (without workspace_name and message_id) Raises: ValueError: If the payload doesn't match the expected schema @@ -142,9 +139,7 @@ def create_payload( validated_payload = RepresentationPayload( content=content, - workspace_name=workspace_name, session_name=session_name, - message_id=message_id, created_at=created_at, observer=observer, observed=observed, @@ -161,16 +156,14 @@ def create_payload( ) validated_payload = SummaryPayload( - workspace_name=workspace_name, session_name=session_name, - message_id=message_id, message_seq_in_session=message_seq_in_session, message_public_id=message_public_id, ) # Convert back to dict for compatibility with JSON serialization # mode='json' ensures datetime is converted to ISO string - payload = validated_payload.model_dump(mode="json") + payload = validated_payload.model_dump(mode="json", exclude_none=True) except Exception as e: raise ValueError(f"Failed to create valid payload: {str(e)}") from e diff --git a/src/utils/representation.py b/src/utils/representation.py index 30ebe71d..c0f499f0 100644 --- a/src/utils/representation.py +++ b/src/utils/representation.py @@ -271,7 +271,7 @@ class Representation(BaseModel): session_name=doc.session_name, ) for doc in documents - if doc.internal_metadata.get("level") == "explicit" + if doc.level == "explicit" ], deductive=[ DeductiveObservation( @@ -284,7 +284,7 @@ class Representation(BaseModel): premises=doc.internal_metadata.get("premises", []), ) for doc in documents - if doc.internal_metadata.get("level") == "deductive" + if doc.level == "deductive" ], ) diff --git a/src/utils/types.py b/src/utils/types.py index 317e545a..80dfc389 100644 --- a/src/utils/types.py +++ b/src/utils/types.py @@ -1,3 +1,5 @@ from typing import Literal SupportedProviders = Literal["anthropic", "openai", "google", "groq", "custom", "vllm"] +TaskType = Literal["webhook", "summary", "representation", "dream"] +DocumentLevel = Literal["explicit", "deductive"] diff --git a/src/utils/work_unit.py b/src/utils/work_unit.py index d7fd12bb..660baa16 100644 --- a/src/utils/work_unit.py +++ b/src/utils/work_unit.py @@ -15,11 +15,14 @@ class ParsedWorkUnit(BaseModel): observed: str | None -def get_work_unit_key(payload: dict[str, Any] | ParsedWorkUnit) -> str: +def construct_work_unit_key( + workspace_name: str, payload: dict[str, Any] | ParsedWorkUnit +) -> str: """ Generate a work unit key for a given task type, workspace name, and event type. Args: + workspace_name: The name of the workspace the work unit belongs to payload: Dictionary containing work unit information Returns: @@ -31,7 +34,6 @@ def get_work_unit_key(payload: dict[str, Any] | ParsedWorkUnit) -> str: if isinstance(payload, ParsedWorkUnit): payload = payload.model_dump() - workspace_name: str | None = payload.get("workspace_name") task_type: str | None = payload.get("task_type") if not workspace_name or not task_type: raise ValueError( diff --git a/src/webhooks/events.py b/src/webhooks/events.py index f942bb5d..a37eab89 100644 --- a/src/webhooks/events.py +++ b/src/webhooks/events.py @@ -7,7 +7,7 @@ from pydantic import BaseModel from src.dependencies import tracked_db from src.models import QueueItem from src.utils.queue_payload import create_webhook_payload -from src.utils.work_unit import get_work_unit_key +from src.utils.work_unit import construct_work_unit_key logger = logging.getLogger(__name__) @@ -51,23 +51,26 @@ async def publish_webhook_event(event: WebhookEvent) -> None: event: The webhook event to publish. """ try: + # Note: workspace_name is no longer included in the payload + # It's stored directly on the queue item payload = create_webhook_payload( - workspace_name=event.workspace_id, event_type=event.type.value, data=event.model_dump(mode="json", exclude={"type"}), ) async with tracked_db("publish_webhook_event") as db: queue_item = QueueItem( - work_unit_key=get_work_unit_key( + work_unit_key=construct_work_unit_key( + event.workspace_id, { "task_type": "webhook", - "workspace_name": event.workspace_id, - } + }, ), payload=payload, session_id=None, task_type="webhook", + workspace_name=event.workspace_id, + message_id=None, # Webhooks don't have a message_id ) db.add(queue_item) await db.commit() diff --git a/src/webhooks/webhook_delivery.py b/src/webhooks/webhook_delivery.py index d9587009..d26aa404 100644 --- a/src/webhooks/webhook_delivery.py +++ b/src/webhooks/webhook_delivery.py @@ -15,16 +15,18 @@ from src.utils.queue_payload import WebhookPayload logger = logging.getLogger(__name__) -async def deliver_webhook(db: AsyncSession, payload: WebhookPayload) -> None: +async def deliver_webhook( + db: AsyncSession, payload: WebhookPayload, workspace_name: str +) -> None: """ Deliver a single webhook event to its configured endpoints. """ async with httpx.AsyncClient(timeout=30.0) as client: try: - webhook_urls = await _get_webhook_urls(db, payload.workspace_name) + webhook_urls = await _get_webhook_urls(db, workspace_name) if not webhook_urls: logger.debug( - f"No webhook endpoints for workspace {payload.workspace_name}, skipping." + f"No webhook endpoints for workspace {workspace_name}, skipping." ) return @@ -72,7 +74,7 @@ async def deliver_webhook(db: AsyncSession, payload: WebhookPayload) -> None: ) except httpx.RequestError: - logger.exception(f"Error sending webhook for {payload.workspace_name}.") + logger.exception(f"Error sending webhook for {workspace_name}.") except Exception: logger.exception("Unexpected error delivering webhook.") @@ -82,7 +84,7 @@ async def _get_webhook_urls(db: AsyncSession, workspace_name: str) -> list[str]: Get all webhook endpoint URLs for a workspace. """ try: - endpoints = await list_webhook_endpoints(db, workspace_name) + endpoints = await list_webhook_endpoints(workspace_name) result = await db.execute(endpoints) return [endpoint.url for endpoint in result.scalars().all()] except Exception: diff --git a/tests/alembic/revisions/__init__.py b/tests/alembic/revisions/__init__.py index 359e7b1f..fedc98c5 100644 --- a/tests/alembic/revisions/__init__.py +++ b/tests/alembic/revisions/__init__.py @@ -2,6 +2,7 @@ from . import ( test_05486ce795d5_make_session_name_required_on_messages, + test_066e87ca5b07_align_schema_with_declarative_models, test_08894082221a_replace_collection_name_with_observer_, test_20f89a421aff_rename_metamessage_type_to_label, test_66e63cf2cf77_add_indexes_to_documents_table, @@ -12,13 +13,16 @@ from . import ( test_917195d9b5e9_add_messageembedding_table, test_a1b2c3d4e5f6_initial_schema, test_b765d82110bd_change_metamessages_to_user_level_with_, + test_b8183c5ffb48_codify_document_level_and_times_derived, test_bb6fb3a7a643_add_message_seq_in_session_column, test_c3828084f472_add_indexes_for_messages_and_, test_d429de0e5338_adopt_peer_paradigm, + test_ec8f94139b02_codify_workspace_name_and_message_id_in_, ) __all__ = [ "test_05486ce795d5_make_session_name_required_on_messages", + "test_066e87ca5b07_align_schema_with_declarative_models", "test_08894082221a_replace_collection_name_with_observer_", "test_20f89a421aff_rename_metamessage_type_to_label", "test_556a16564f50_add_user_id_and_app_id_to_tables", @@ -29,7 +33,9 @@ __all__ = [ "test_917195d9b5e9_add_messageembedding_table", "test_a1b2c3d4e5f6_initial_schema", "test_b765d82110bd_change_metamessages_to_user_level_with_", + "test_b8183c5ffb48_codify_document_level_and_times_derived", "test_bb6fb3a7a643_add_message_seq_in_session_column", "test_c3828084f472_add_indexes_for_messages_and_", "test_d429de0e5338_adopt_peer_paradigm", + "test_ec8f94139b02_codify_workspace_name_and_message_id_in_", ] diff --git a/tests/alembic/revisions/test_066e87ca5b07_align_schema_with_declarative_models.py b/tests/alembic/revisions/test_066e87ca5b07_align_schema_with_declarative_models.py new file mode 100644 index 00000000..40d396bc --- /dev/null +++ b/tests/alembic/revisions/test_066e87ca5b07_align_schema_with_declarative_models.py @@ -0,0 +1,42 @@ +"""Hooks for revision 066e87ca5b07 (align_schema_with_declarative_models).""" + +from __future__ import annotations + +from tests.alembic.registry import register_after_upgrade, register_before_upgrade +from tests.alembic.verifier import MigrationVerifier + + +@register_before_upgrade("066e87ca5b07") +def prepare_align_schema_with_declarative_models(verifier: MigrationVerifier) -> None: + """Seed state and assertions before upgrading to 066e87ca5b07.""" + # Assert columns exist but are nullable before migration + verifier.assert_column_exists( + "active_queue_sessions", "work_unit_key", exists=True, nullable=True + ) + verifier.assert_column_exists("documents", "embedding", exists=True, nullable=True) + + # Assert FK constraint does not exist yet + verifier.assert_constraint_exists( + "queue", "fk_queue_session_id", "foreign_key", exists=False + ) + + +@register_after_upgrade("066e87ca5b07") +def verify_align_schema_with_declarative_models(verifier: MigrationVerifier) -> None: + """Add assertions validating the effects of 066e87ca5b07.""" + # Assert columns are now non-nullable + verifier.assert_column_exists( + "peers", "workspace_name", exists=True, nullable=False + ) + verifier.assert_column_exists( + "sessions", "workspace_name", exists=True, nullable=False + ) + verifier.assert_column_exists( + "active_queue_sessions", "work_unit_key", exists=True, nullable=False + ) + verifier.assert_column_exists("documents", "embedding", exists=True, nullable=False) + + # Assert FK constraint now exists + verifier.assert_constraint_exists( + "queue", "fk_queue_session_id", "foreign_key", exists=True + ) diff --git a/tests/alembic/revisions/test_b8183c5ffb48_codify_document_level_and_times_derived.py b/tests/alembic/revisions/test_b8183c5ffb48_codify_document_level_and_times_derived.py new file mode 100644 index 00000000..c3e3d3a0 --- /dev/null +++ b/tests/alembic/revisions/test_b8183c5ffb48_codify_document_level_and_times_derived.py @@ -0,0 +1,315 @@ +"""Hooks for revision b8183c5ffb48 (codify_document_level_and_times_derived).""" + +from __future__ import annotations + +import json + +from nanoid import generate as generate_nanoid +from sqlalchemy import text + +from tests.alembic.registry import register_after_upgrade, register_before_upgrade +from tests.alembic.verifier import MigrationVerifier + +# Test data constants +WORKSPACE_NAME = "test-workspace" +OBSERVER_NAME = "test-observer" +OBSERVED_NAME = "test-observed" +SESSION_NAME = "test-session" + + +@register_before_upgrade("b8183c5ffb48") +def prepare_codify_document_level_and_times_derived( + verifier: MigrationVerifier, +) -> None: + """Seed state and assertions before upgrading to b8183c5ffb48.""" + # Verify columns don't exist yet + verifier.assert_column_exists("documents", "level", exists=False) + verifier.assert_column_exists("documents", "times_derived", exists=False) + + # Verify CHECK constraint doesn't exist yet + verifier.assert_constraint_exists("documents", "level_valid", "check", exists=False) + + conn = verifier.conn + schema = verifier.schema + + # Create workspace + conn.execute( + text( + f'INSERT INTO "{schema}"."workspaces" ("id", "name") VALUES (:ws_id, :ws_name)' + ), + {"ws_id": generate_nanoid(), "ws_name": WORKSPACE_NAME}, + ) + + # Create peers (observer and observed) + conn.execute( + text( + f'INSERT INTO "{schema}"."peers" ("id", "name", "workspace_name") ' + + "VALUES (:observer_id, :observer_name, :ws_name), " + + "(:observed_id, :observed_name, :ws_name)" + ), + { + "observer_id": generate_nanoid(), + "observer_name": OBSERVER_NAME, + "ws_name": WORKSPACE_NAME, + "observed_id": generate_nanoid(), + "observed_name": OBSERVED_NAME, + }, + ) + + # Create session + conn.execute( + text( + f'INSERT INTO "{schema}"."sessions" ("id", "name", "workspace_name") ' + + "VALUES (:session_id, :session_name, :ws_name)" + ), + { + "session_id": generate_nanoid(), + "session_name": SESSION_NAME, + "ws_name": WORKSPACE_NAME, + }, + ) + + # Create collection (required for documents) + conn.execute( + text( + f'INSERT INTO "{schema}"."collections" ("id", "observer", "observed", "workspace_name") ' + + "VALUES (:collection_id, :observer, :observed, :ws_name)" + ), + { + "collection_id": generate_nanoid(), + "observer": OBSERVER_NAME, + "observed": OBSERVED_NAME, + "ws_name": WORKSPACE_NAME, + }, + ) + + # Create documents with different scenarios + # Scenario 1: Document with both level and times_derived in internal_metadata + conn.execute( + text( + f'INSERT INTO "{schema}"."documents" ' + + '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") ' + + "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)" + ), + { + "id": generate_nanoid(), + "content": "Document with explicit level and times_derived=3", + "metadata": json.dumps({"level": "explicit", "times_derived": 3}), + "embedding": [0.1] * 1536, + "observer": OBSERVER_NAME, + "observed": OBSERVED_NAME, + "ws_name": WORKSPACE_NAME, + "session_name": SESSION_NAME, + }, + ) + + # Scenario 2: Document with deductive level in internal_metadata + conn.execute( + text( + f'INSERT INTO "{schema}"."documents" ' + + '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") ' + + "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)" + ), + { + "id": generate_nanoid(), + "content": "Document with deductive level and times_derived=5", + "metadata": json.dumps({"level": "deductive", "times_derived": 5}), + "embedding": [0.2] * 1536, + "observer": OBSERVER_NAME, + "observed": OBSERVED_NAME, + "ws_name": WORKSPACE_NAME, + "session_name": SESSION_NAME, + }, + ) + + # Scenario 3: Document without level or times_derived (should get defaults) + conn.execute( + text( + f'INSERT INTO "{schema}"."documents" ' + + '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") ' + + "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)" + ), + { + "id": generate_nanoid(), + "content": "Document without level or times_derived fields", + "metadata": json.dumps({"other_field": "value"}), + "embedding": [0.3] * 1536, + "observer": OBSERVER_NAME, + "observed": OBSERVED_NAME, + "ws_name": WORKSPACE_NAME, + "session_name": SESSION_NAME, + }, + ) + + # Scenario 4: Document with only level in metadata + conn.execute( + text( + f'INSERT INTO "{schema}"."documents" ' + + '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") ' + + "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)" + ), + { + "id": generate_nanoid(), + "content": "Document with only level field", + "metadata": json.dumps({"level": "explicit"}), + "embedding": [0.4] * 1536, + "observer": OBSERVER_NAME, + "observed": OBSERVED_NAME, + "ws_name": WORKSPACE_NAME, + "session_name": SESSION_NAME, + }, + ) + + # Scenario 5: Document with only times_derived in metadata + conn.execute( + text( + f'INSERT INTO "{schema}"."documents" ' + + '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") ' + + "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)" + ), + { + "id": generate_nanoid(), + "content": "Document with only times_derived field", + "metadata": json.dumps({"times_derived": 7}), + "embedding": [0.5] * 1536, + "observer": OBSERVER_NAME, + "observed": OBSERVED_NAME, + "ws_name": WORKSPACE_NAME, + "session_name": SESSION_NAME, + }, + ) + + # Verify we have exactly 5 documents + count = conn.execute(text(f'SELECT COUNT(*) FROM "{schema}"."documents"')).scalar() + assert count == 5, f"Expected 5 documents but found {count}" + + +@register_after_upgrade("b8183c5ffb48") +def verify_codify_document_level_and_times_derived(verifier: MigrationVerifier) -> None: + """Add assertions validating the effects of b8183c5ffb48.""" + # Verify columns were added with correct nullability + verifier.assert_column_exists("documents", "level", nullable=False) + verifier.assert_column_exists("documents", "times_derived", nullable=False) + + # Verify CHECK constraint exists + verifier.assert_constraint_exists("documents", "level_valid", "check") + + conn = verifier.conn + schema = verifier.schema + + # Verify all rows have non-null values after migration + verifier.assert_no_nulls("documents", "level") + verifier.assert_no_nulls("documents", "times_derived") + + # Verify data transformation: level extracted from internal_metadata + explicit_count = conn.execute( + text(f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE "level" = :level'), + {"level": "explicit"}, + ).scalar() + deductive_count = conn.execute( + text(f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE "level" = :level'), + {"level": "deductive"}, + ).scalar() + + # 3 documents should have explicit (scenarios 1, 3, 4) + # 1 document should have deductive (scenario 2) + # 1 document should have default explicit (scenario 5) + assert explicit_count == 4, f"Expected 4 explicit documents, got {explicit_count}" + assert deductive_count == 1, f"Expected 1 deductive document, got {deductive_count}" + + # Verify specific times_derived values were migrated correctly + times_derived_3 = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."documents" ' + + 'WHERE "times_derived" = 3 AND "content" LIKE :pattern' + ), + {"pattern": "%explicit level and times_derived=3%"}, + ).scalar() + assert ( + times_derived_3 == 1 + ), f"Expected 1 document with times_derived=3, got {times_derived_3}" + + times_derived_5 = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."documents" ' + + 'WHERE "times_derived" = 5 AND "content" LIKE :pattern' + ), + {"pattern": "%deductive level and times_derived=5%"}, + ).scalar() + assert ( + times_derived_5 == 1 + ), f"Expected 1 document with times_derived=5, got {times_derived_5}" + + times_derived_7 = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."documents" ' + + 'WHERE "times_derived" = 7 AND "content" LIKE :pattern' + ), + {"pattern": "%only times_derived field%"}, + ).scalar() + assert ( + times_derived_7 == 1 + ), f"Expected 1 document with times_derived=7, got {times_derived_7}" + + # Verify default times_derived=1 was applied to documents without it + times_derived_1 = conn.execute( + text(f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE "times_derived" = 1') + ).scalar() + assert ( + times_derived_1 == 2 + ), f"Expected 2 documents with times_derived=1, got {times_derived_1}" + + # Verify internal_metadata still contains the original data (NOT removed by migration) + level_in_metadata = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE internal_metadata ? \'level\'' + ) + ).scalar() + # 3 documents had level in metadata (scenarios 1, 2, 4) + assert ( + level_in_metadata == 3 + ), f"Expected 3 documents with level in metadata, got {level_in_metadata}" + + times_derived_in_metadata = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."documents" WHERE internal_metadata ? \'times_derived\'' + ) + ).scalar() + # 3 documents had times_derived in metadata (scenarios 1, 2, 5) + assert ( + times_derived_in_metadata == 3 + ), f"Expected 3 documents with times_derived in metadata, got {times_derived_in_metadata}" + + # Verify server defaults work for new documents + new_doc_id = generate_nanoid() + conn.execute( + text( + f'INSERT INTO "{schema}"."documents" ' + + '("id", "content", "internal_metadata", "embedding", "observer", "observed", "workspace_name", "session_name") ' + + "VALUES (:id, :content, :metadata, :embedding, :observer, :observed, :ws_name, :session_name)" + ), + { + "id": new_doc_id, + "content": "New document after migration", + "metadata": json.dumps({}), + "embedding": [0.6] * 1536, + "observer": OBSERVER_NAME, + "observed": OBSERVED_NAME, + "ws_name": WORKSPACE_NAME, + "session_name": SESSION_NAME, + }, + ) + + # Verify the new document got default values + new_doc = conn.execute( + text( + f'SELECT "level", "times_derived" FROM "{schema}"."documents" WHERE "id" = :id' + ), + {"id": new_doc_id}, + ).one() + assert ( + new_doc.level == "explicit" + ), f"Expected new document to have level='explicit', got {new_doc.level}" + assert ( + new_doc.times_derived == 1 + ), f"Expected new document to have times_derived=1, got {new_doc.times_derived}" diff --git a/tests/alembic/revisions/test_e9b705f9adf9_add_server_defaults_to_timestamp_.py b/tests/alembic/revisions/test_e9b705f9adf9_add_server_defaults_to_timestamp_.py new file mode 100644 index 00000000..31d9c6c4 --- /dev/null +++ b/tests/alembic/revisions/test_e9b705f9adf9_add_server_defaults_to_timestamp_.py @@ -0,0 +1,284 @@ +"""Hooks for revision e9b705f9adf9 (add server defaults to timestamp, boolean, and jsonb columns).""" + +from __future__ import annotations + +from nanoid import generate as generate_nanoid +from sqlalchemy import text + +from tests.alembic.registry import register_after_upgrade, register_before_upgrade +from tests.alembic.verifier import MigrationVerifier + +# Test data IDs +WORKSPACE_ID = generate_nanoid() +PEER_ID = generate_nanoid() +SESSION_ID = generate_nanoid() +MESSAGE_ID = generate_nanoid() +COLLECTION_ID = generate_nanoid() +DOCUMENT_ID = generate_nanoid() + + +@register_before_upgrade("e9b705f9adf9") +def prepare_add_server_defaults(verifier: MigrationVerifier) -> None: + """Seed state before upgrading to e9b705f9adf9. + + This migration adds server defaults to timestamp, JSONB, and boolean columns. + We verify that columns exist but don't have server defaults before the migration. + """ + conn = verifier.conn + schema = verifier.schema + inspector = verifier.get_inspector() + + # Sample timestamp columns to check - they should exist but without server defaults + for table, column in [ + ("workspaces", "created_at"), + ("peers", "created_at"), + ("sessions", "created_at"), + ("messages", "created_at"), + ("collections", "created_at"), + ("documents", "created_at"), + ("queue", "created_at"), + ]: + columns = inspector.get_columns(table, schema=schema) + col_info = next((c for c in columns if c["name"] == column), None) + assert ( + col_info is not None + ), f"Column {table}.{column} should exist before migration" + + # Create test data to ensure existing rows work after migration + conn.execute( + text( + f'INSERT INTO "{schema}"."workspaces" ' + + '("id", "name", "created_at", "metadata", "internal_metadata", "configuration") ' + + "VALUES (:id, :name, NOW(), :metadata, :internal_metadata, :configuration)" + ), + { + "id": WORKSPACE_ID, + "name": "test-workspace", + "metadata": "{}", + "internal_metadata": "{}", + "configuration": "{}", + }, + ) + + conn.execute( + text( + f'INSERT INTO "{schema}"."peers" ' + + '("id", "name", "workspace_name", "created_at", "metadata", "internal_metadata", "configuration") ' + + "VALUES (:id, :name, :workspace_name, NOW(), :metadata, :internal_metadata, :configuration)" + ), + { + "id": PEER_ID, + "name": "test-peer", + "workspace_name": "test-workspace", + "metadata": "{}", + "internal_metadata": "{}", + "configuration": "{}", + }, + ) + + conn.execute( + text( + f'INSERT INTO "{schema}"."sessions" ' + + '("id", "name", "workspace_name", "created_at", "is_active", "metadata", "internal_metadata", "configuration") ' + + "VALUES (:id, :name, :workspace_name, NOW(), true, :metadata, :internal_metadata, :configuration)" + ), + { + "id": SESSION_ID, + "name": "test-session", + "workspace_name": "test-workspace", + "metadata": "{}", + "internal_metadata": "{}", + "configuration": "{}", + }, + ) + + +@register_after_upgrade("e9b705f9adf9") +def verify_add_server_defaults(verifier: MigrationVerifier) -> None: + """Validate server defaults were added correctly to all columns.""" + conn = verifier.conn + schema = verifier.schema + inspector = verifier.get_inspector() + + # Verify timestamp columns have server defaults (now() function) + timestamp_columns = [ + ("workspaces", "created_at"), + ("peers", "created_at"), + ("sessions", "created_at"), + ("messages", "created_at"), + ("message_embeddings", "created_at"), + ("collections", "created_at"), + ("documents", "created_at"), + ("queue", "created_at"), + ("webhook_endpoints", "created_at"), + ("session_peers", "joined_at"), + ("active_queue_sessions", "last_updated"), + ] + + for table, column in timestamp_columns: + columns = inspector.get_columns(table, schema=schema) + col_info = next((c for c in columns if c["name"] == column), None) + assert ( + col_info is not None + ), f"Column {table}.{column} not found after migration" + + # Check that a server default exists + default = col_info.get("default") + assert default is not None, ( + f"Column {table}.{column} should have a server default after migration, " + f"but default is None" + ) + + # Verify JSONB columns have server defaults (empty object '{}') + jsonb_columns = [ + ("workspaces", "metadata"), + ("workspaces", "internal_metadata"), + ("workspaces", "configuration"), + ("peers", "metadata"), + ("peers", "internal_metadata"), + ("peers", "configuration"), + ("sessions", "metadata"), + ("sessions", "internal_metadata"), + ("sessions", "configuration"), + ("messages", "metadata"), + ("messages", "internal_metadata"), + ("collections", "metadata"), + ("collections", "internal_metadata"), + ("documents", "internal_metadata"), + ("session_peers", "configuration"), + ("session_peers", "internal_metadata"), + ] + + for table, column in jsonb_columns: + columns = inspector.get_columns(table, schema=schema) + col_info = next((c for c in columns if c["name"] == column), None) + assert ( + col_info is not None + ), f"Column {table}.{column} not found after migration" + + # Check that a server default exists + default = col_info.get("default") + assert default is not None, ( + f"Column {table}.{column} should have a server default after migration, " + f"but default is None" + ) + + # Verify boolean columns have server defaults + boolean_columns = [ + ("sessions", "is_active", "true"), + ("queue", "processed", "false"), + ] + + for table, column, expected_default in boolean_columns: + columns = inspector.get_columns(table, schema=schema) + col_info = next((c for c in columns if c["name"] == column), None) + assert ( + col_info is not None + ), f"Column {table}.{column} not found after migration" + + # Check that a server default exists + default = col_info.get("default") + assert default is not None, ( + f"Column {table}.{column} should have a server default after migration, " + f"but default is None" + ) + + assert ( + default == expected_default + ), f"Column {table}.{column} should have a server default of {expected_default} after migration, but default is {default}" + + # Test that defaults actually work by inserting rows without explicit values + test_workspace_id = generate_nanoid() + conn.execute( + text( + f'INSERT INTO "{schema}"."workspaces" ("id", "name") ' + + "VALUES (:id, :name)" + ), + {"id": test_workspace_id, "name": "test-defaults-workspace"}, + ) + + # Verify the inserted workspace has default values + workspace = conn.execute( + text( + 'SELECT "created_at", "metadata", "internal_metadata", "configuration" ' + + f'FROM "{schema}"."workspaces" WHERE "id" = :id' + ), + {"id": test_workspace_id}, + ).one() + + assert workspace.created_at is not None, "created_at should be auto-populated" + assert workspace.metadata == {}, "metadata should default to empty object" + assert ( + workspace.internal_metadata == {} + ), "internal_metadata should default to empty object" + assert workspace.configuration == {}, "configuration should default to empty object" + + # Test peer defaults + test_peer_id = generate_nanoid() + conn.execute( + text( + f'INSERT INTO "{schema}"."peers" ("id", "name", "workspace_name") ' + + "VALUES (:id, :name, :workspace_name)" + ), + { + "id": test_peer_id, + "name": "test-defaults-peer", + "workspace_name": "test-defaults-workspace", + }, + ) + + peer = conn.execute( + text( + 'SELECT "created_at", "metadata", "internal_metadata", "configuration" ' + + f'FROM "{schema}"."peers" WHERE "id" = :id' + ), + {"id": test_peer_id}, + ).one() + + assert peer.created_at is not None, "peer created_at should be auto-populated" + assert peer.metadata == {}, "peer metadata should default to empty object" + assert ( + peer.internal_metadata == {} + ), "peer internal_metadata should default to empty object" + assert peer.configuration == {}, "peer configuration should default to empty object" + + # Test session defaults (including boolean is_active) + test_session_id = generate_nanoid() + conn.execute( + text( + f'INSERT INTO "{schema}"."sessions" ("id", "name", "workspace_name") ' + + "VALUES (:id, :name, :workspace_name)" + ), + { + "id": test_session_id, + "name": "test-defaults-session", + "workspace_name": "test-defaults-workspace", + }, + ) + + session = conn.execute( + text( + 'SELECT "created_at", "is_active", "metadata", "internal_metadata", "configuration" ' + + f'FROM "{schema}"."sessions" WHERE "id" = :id' + ), + {"id": test_session_id}, + ).one() + + assert session.created_at is not None, "session created_at should be auto-populated" + assert session.is_active is True, "session is_active should default to true" + assert session.metadata == {}, "session metadata should default to empty object" + assert ( + session.internal_metadata == {} + ), "session internal_metadata should default to empty object" + assert ( + session.configuration == {} + ), "session configuration should default to empty object" + + # Verify pre-existing data still exists + existing_workspace = conn.execute( + text(f'SELECT "id" FROM "{schema}"."workspaces" WHERE "id" = :id'), + {"id": WORKSPACE_ID}, + ).one_or_none() + assert ( + existing_workspace is not None + ), "Pre-existing workspace should still exist after migration" diff --git a/tests/alembic/revisions/test_ec8f94139b02_codify_workspace_name_and_message_id_in_.py b/tests/alembic/revisions/test_ec8f94139b02_codify_workspace_name_and_message_id_in_.py new file mode 100644 index 00000000..cbef748a --- /dev/null +++ b/tests/alembic/revisions/test_ec8f94139b02_codify_workspace_name_and_message_id_in_.py @@ -0,0 +1,336 @@ +"""Hooks for revision ec8f94139b02 (codify_workspace_name_and_message_id_in_).""" + +from __future__ import annotations + +import json + +from nanoid import generate as generate_nanoid +from sqlalchemy import text + +from tests.alembic.registry import register_after_upgrade, register_before_upgrade +from tests.alembic.verifier import MigrationVerifier + +# Test data constants +WORKSPACE_NAME_1 = "test-workspace-1" +WORKSPACE_NAME_2 = "test-workspace-2" +PEER_NAME_1 = "test-peer-1" +PEER_NAME_2 = "test-peer-2" +SESSION_NAME_1 = "test-session-1" +SESSION_NAME_2 = "test-session-2" + +# Indexes that should be created by the migration +_INDEXES = ( + ("queue", "ix_queue_workspace_name"), + ("queue", "ix_queue_message_id_not_null"), + ("queue", "ix_queue_workspace_name_processed"), + ("queue", "ix_queue_work_unit_key_processed_id"), +) + + +@register_before_upgrade("ec8f94139b02") +def prepare_codify_workspace_name_and_message_id_in( + verifier: MigrationVerifier, +) -> None: + """Seed state and assertions before upgrading to ec8f94139b02.""" + # Verify columns don't exist yet + verifier.assert_column_exists("queue", "workspace_name", exists=False) + verifier.assert_column_exists("queue", "message_id", exists=False) + + # Verify foreign key constraints don't exist yet + verifier.assert_constraint_exists( + "queue", "fk_queue_workspace_name", "foreign_key", exists=False + ) + verifier.assert_constraint_exists( + "queue", "fk_queue_message_id", "foreign_key", exists=False + ) + + # Verify indexes don't exist yet + verifier.assert_indexes_not_exist(_INDEXES) + + conn = verifier.conn + schema = verifier.schema + + # Create workspaces + conn.execute( + text( + f'INSERT INTO "{schema}"."workspaces" ("id", "name") ' + + "VALUES (:ws_id_1, :ws_name_1), (:ws_id_2, :ws_name_2)" + ), + { + "ws_id_1": generate_nanoid(), + "ws_name_1": WORKSPACE_NAME_1, + "ws_id_2": generate_nanoid(), + "ws_name_2": WORKSPACE_NAME_2, + }, + ) + + # Create peers + conn.execute( + text( + f'INSERT INTO "{schema}"."peers" ("id", "name", "workspace_name") ' + + "VALUES (:peer_id_1, :peer_name_1, :ws_name_1), " + + "(:peer_id_2, :peer_name_2, :ws_name_2)" + ), + { + "peer_id_1": generate_nanoid(), + "peer_name_1": PEER_NAME_1, + "ws_name_1": WORKSPACE_NAME_1, + "peer_id_2": generate_nanoid(), + "peer_name_2": PEER_NAME_2, + "ws_name_2": WORKSPACE_NAME_2, + }, + ) + + # Create sessions + conn.execute( + text( + f'INSERT INTO "{schema}"."sessions" ("id", "name", "workspace_name") ' + + "VALUES (:session_id_1, :session_name_1, :ws_name_1), " + + "(:session_id_2, :session_name_2, :ws_name_2)" + ), + { + "session_id_1": generate_nanoid(), + "session_name_1": SESSION_NAME_1, + "ws_name_1": WORKSPACE_NAME_1, + "session_id_2": generate_nanoid(), + "session_name_2": SESSION_NAME_2, + "ws_name_2": WORKSPACE_NAME_2, + }, + ) + + # Create some messages to reference in queue items + message_ids: list[str] = [] + for i in range(10): + message_id: str = generate_nanoid() + message_ids.append(message_id) + workspace = WORKSPACE_NAME_1 if i % 2 == 0 else WORKSPACE_NAME_2 + session = SESSION_NAME_1 if i % 2 == 0 else SESSION_NAME_2 + peer = PEER_NAME_1 if i % 2 == 0 else PEER_NAME_2 + + conn.execute( + text( + f'INSERT INTO "{schema}"."messages" ' + + '("public_id", "workspace_name", "session_name", "peer_name", "content", "seq_in_session") ' + + "VALUES (:msg_id, :ws_name, :session_name, :peer_name, :content, :seq)" + ), + { + "msg_id": message_id, + "ws_name": workspace, + "session_name": session, + "peer_name": peer, + "content": f"test message {i}", + "seq": i, + }, + ) + + # Get internal message IDs for queue references + message_db_ids: list[int] = [] + for msg_id in message_ids: + result = conn.execute( + text(f'SELECT "id" FROM "{schema}"."messages" WHERE "public_id" = :msg_id'), + {"msg_id": msg_id}, + ).one() + message_db_ids.append(result.id) + + # Bulk insert 100k queue items with workspace_name and message_id in payload + # Use efficient batch insert with generate_series + conn.execute(text("SET LOCAL synchronous_commit = OFF")) + + # Insert queue items in three categories: + # 1. Items with both workspace_name and message_id (60k) + # 2. Items with workspace_name but NO message_id (30k) + # 3. Items with workspace_name and NULL message_id value (10k) + + # Category 1: Both workspace_name and message_id (60k items) + conn.execute( + text( + f'INSERT INTO "{schema}"."queue" ' + + '("work_unit_key", "task_type", "payload", "processed") ' + + "SELECT " + + " 'work-unit-' || gs::text, " + + " 'representation', " + + " jsonb_build_object(" + + " 'workspace_name', CASE WHEN gs % 2 = 0 THEN :ws_name_1 ELSE :ws_name_2 END, " + + " 'message_id', :msg_db_id_0 + (gs % 10), " + + " 'other_field', 'value-' || gs::text" + + " ), " + + " false " + + "FROM generate_series(1, :n) AS gs" + ), + { + "ws_name_1": WORKSPACE_NAME_1, + "ws_name_2": WORKSPACE_NAME_2, + "msg_db_id_0": message_db_ids[0], + "n": 60_000, + }, + ) + + # Category 2: Only workspace_name, no message_id key (30k items) + conn.execute( + text( + f'INSERT INTO "{schema}"."queue" ' + + '("work_unit_key", "task_type", "payload", "processed") ' + + "SELECT " + + " 'work-unit-no-msg-' || gs::text, " + + " 'summary', " + + " jsonb_build_object(" + + " 'workspace_name', CASE WHEN gs % 2 = 0 THEN :ws_name_1 ELSE :ws_name_2 END, " + + " 'other_field', 'value-' || gs::text" + + " ), " + + " false " + + "FROM generate_series(60001, :n) AS gs" + ), + { + "ws_name_1": WORKSPACE_NAME_1, + "ws_name_2": WORKSPACE_NAME_2, + "n": 90_000, + }, + ) + + # Category 3: workspace_name with explicit NULL message_id (10k items) + conn.execute( + text( + f'INSERT INTO "{schema}"."queue" ' + + '("work_unit_key", "task_type", "payload", "processed") ' + + "SELECT " + + " 'work-unit-null-msg-' || gs::text, " + + " 'representation', " + + " jsonb_build_object(" + + " 'workspace_name', CASE WHEN gs % 2 = 0 THEN :ws_name_1 ELSE :ws_name_2 END, " + + " 'message_id', NULL::bigint, " + + " 'other_field', 'value-' || gs::text" + + " ), " + + " false " + + "FROM generate_series(90001, :n) AS gs" + ), + { + "ws_name_1": WORKSPACE_NAME_1, + "ws_name_2": WORKSPACE_NAME_2, + "n": 100_000, + }, + ) + + # Verify we have exactly 100k queue items + count = conn.execute(text(f'SELECT COUNT(*) FROM "{schema}"."queue"')).scalar() + assert count == 100_000, f"Expected 100k queue items but found {count}" + + +@register_after_upgrade("ec8f94139b02") +def verify_codify_workspace_name_and_message_id_in(verifier: MigrationVerifier) -> None: + """Add assertions validating the effects of ec8f94139b02.""" + # Verify columns were added with correct nullability + verifier.assert_column_exists("queue", "workspace_name", nullable=False) + verifier.assert_column_exists("queue", "message_id", nullable=True) + + # Verify all indexes were created + verifier.assert_indexes_exist(_INDEXES) + + # Verify foreign key constraints exist + verifier.assert_constraint_exists("queue", "fk_queue_workspace_name", "foreign_key") + verifier.assert_constraint_exists("queue", "fk_queue_message_id", "foreign_key") + + conn = verifier.conn + schema = verifier.schema + + # Verify all rows have non-null workspace_name after migration + verifier.assert_no_nulls("queue", "workspace_name") + + # Verify data transformation: workspace_name extracted from payload + ws1_count = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."queue" ' + + 'WHERE "workspace_name" = :ws_name' + ), + {"ws_name": WORKSPACE_NAME_1}, + ).scalar() + ws2_count = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."queue" ' + + 'WHERE "workspace_name" = :ws_name' + ), + {"ws_name": WORKSPACE_NAME_2}, + ).scalar() + + # Should be roughly 50/50 split (we alternate in the insert) + assert ( + ws1_count == 50_000 + ), f"Expected 50k items with workspace_name_1, got {ws1_count}" + assert ( + ws2_count == 50_000 + ), f"Expected 50k items with workspace_name_2, got {ws2_count}" + + # Verify data transformation: message_id extracted from payload where it exists + msg_id_count = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."queue" ' + + 'WHERE "message_id" IS NOT NULL' + ) + ).scalar() + + # Should be 60k items with message_id (category 1 only) + assert ( + msg_id_count == 60_000 + ), f"Expected 60k items with message_id, got {msg_id_count}" + + # Verify items without message_id in payload have NULL in column + null_msg_id_count = conn.execute( + text(f'SELECT COUNT(*) FROM "{schema}"."queue" ' + 'WHERE "message_id" IS NULL') + ).scalar() + + # Should be 40k items (30k without key + 10k with NULL value) + assert ( + null_msg_id_count == 40_000 + ), f"Expected 40k items with NULL message_id, got {null_msg_id_count}" + + # Verify workspace_name was removed from payload + ws_in_payload_count = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."queue" ' + + "WHERE payload ? 'workspace_name'" + ) + ).scalar() + assert ( + ws_in_payload_count == 0 + ), f"Found {ws_in_payload_count} items still with workspace_name in payload" + + # Verify message_id was removed from payload + msg_in_payload_count = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."queue" ' + "WHERE payload ? 'message_id'" + ) + ).scalar() + assert ( + msg_in_payload_count == 0 + ), f"Found {msg_in_payload_count} items still with message_id in payload" + + # Verify other_field remains in payload (data preservation) + other_field_count = conn.execute( + text( + f'SELECT COUNT(*) FROM "{schema}"."queue" ' + + "WHERE payload ? 'other_field'" + ) + ).scalar() + assert ( + other_field_count == 100_000 + ), f"Expected all 100k items to retain other_field in payload, got {other_field_count}" + + # Spot check: verify a specific queue item was transformed correctly + sample_item = conn.execute( + text( + 'SELECT "workspace_name", "message_id", "payload" ' + + f'FROM "{schema}"."queue" ' + + "WHERE work_unit_key = 'work-unit-1' " + + "LIMIT 1" + ) + ).one() + + assert sample_item.workspace_name == WORKSPACE_NAME_2 # gs=1 is odd, so workspace 2 + assert sample_item.message_id is not None # Category 1 item + payload = ( + json.loads(sample_item.payload) + if isinstance(sample_item.payload, str) + else sample_item.payload + ) + assert "workspace_name" not in payload + assert "message_id" not in payload + assert payload.get("other_field") == "value-1" diff --git a/tests/bench/harness.py b/tests/bench/harness.py index 09bb7840..1e3f1a2c 100755 --- a/tests/bench/harness.py +++ b/tests/bench/harness.py @@ -11,6 +11,7 @@ This script: """ import argparse +import asyncio import os import shutil import subprocess @@ -22,6 +23,8 @@ from pathlib import Path import yaml +from src.cache.client import close_cache, init_cache + class HonchoHarness: """ @@ -65,6 +68,9 @@ class HonchoHarness: # Update the database port compose_data["services"]["database"]["ports"] = [f"{self.db_port}:5432"] + # Update the Redis port + compose_data["services"]["redis"]["ports"] = ["6379:6379"] + # Add a unique project name to avoid conflicts compose_data["name"] = f"honcho_harness_{self.db_port}" @@ -111,13 +117,15 @@ class HonchoHarness: def get_database_env_vars(self) -> dict[str, str]: """ - Get environment variables for database configuration and required API keys. + Get environment variables for database configuration, cache configuration, and required API keys. Returns: - Dictionary of environment variables for database connection and API keys + Dictionary of environment variables for database connection, cache, and API keys """ return { "DB_CONNECTION_URI": f"postgresql+psycopg://testuser:testpwd@localhost:{self.db_port}/honcho", + "CACHE_ENABLED": "true", + "CACHE_URL": "redis://localhost:6379/0", } def start_database(self) -> None: @@ -149,6 +157,99 @@ class HonchoHarness: print("Database started successfully") + def start_redis(self) -> None: + """ + Start the Redis cache server using Docker Compose. + """ + print("Starting Redis cache server on port 6379...") + + # Change to the temp directory and start the redis service + result = subprocess.run( + [ + "docker-compose", + "-f", + str(self.docker_compose_file), + "-p", + f"honcho_harness_{self.db_port}", + "up", + "-d", + "redis", + ], + cwd=self.temp_dir, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(f"Failed to start Redis: {result.stderr}") + sys.exit(1) + + print("Redis started successfully") + + def wait_for_redis(self, timeout: int = 30) -> bool: + """ + Wait for Redis to be ready. + + Args: + timeout: Maximum time to wait in seconds + + Returns: + True if Redis is ready, False otherwise + """ + print("Waiting for Redis to be ready...") + start_time = time.time() + redis_port = 6379 + + while time.time() - start_time < timeout: + try: + import redis + + # Test Redis connection + r = redis.Redis( + host="localhost", port=redis_port, decode_responses=True + ) + r.ping() # pyright: ignore[reportUnknownMemberType] + print("Redis is ready!") + return True + except Exception: + pass + + time.sleep(1) # Check every second + + print("Redis failed to become ready within timeout") + return False + + async def init_cache(self) -> None: + """ + Initialize the Redis cache connection. + """ + try: + # Add the project root to the path so we can import Honcho modules + sys.path.insert(0, str(self.project_root)) + + # Set environment variables for cache configuration + env = self.get_database_env_vars() + for key, value in env.items(): + os.environ[key] = value + + await init_cache() + print(f"[Instance {self.instance_id}] Cache initialized successfully") + except Exception as e: + print(f"[Instance {self.instance_id}] Failed to initialize cache: {e}") + + async def close_cache(self) -> None: + """ + Close the Redis cache connection. + """ + try: + # Add the project root to the path so we can import Honcho modules + sys.path.insert(0, str(self.project_root)) + + await close_cache() + print(f"[Instance {self.instance_id}] Cache closed successfully") + except Exception as e: + print(f"[Instance {self.instance_id}] Failed to close cache: {e}") + def wait_for_database(self, timeout: int = 60) -> bool: """ Wait for the database to be ready. @@ -576,10 +677,16 @@ except Exception as e: except Exception as e: print(f"Error removing temp directory: {e}") + # Close cache + try: + asyncio.run(self.close_cache()) + except Exception as e: + print(f"Error closing cache: {e}") + # Restore .env file self.restore_env_file() - def run(self) -> None: + async def run(self) -> None: """ Run the complete Honcho harness. """ @@ -606,11 +713,22 @@ except Exception as e: # Start database self.start_database() + # Start Redis + self.start_redis() + # Wait for database to be ready if not self.wait_for_database(): print("Database failed to start. Exiting.") sys.exit(1) + # Wait for Redis to be ready + if not self.wait_for_redis(): + print("Redis failed to start. Exiting.") + sys.exit(1) + + # Initialize cache + await self.init_cache() + # Provision database self.provision_database() @@ -700,7 +818,7 @@ class HonchoHarnessPool: ) self.harnesses.append(harness) - def run(self) -> None: + async def run(self) -> None: """ Run all Honcho harnesses in the pool. """ @@ -748,6 +866,9 @@ class HonchoHarnessPool: # Start database harness.start_database() + # Start Redis + harness.start_redis() + # Wait for database to be ready if not harness.wait_for_database(): print( @@ -755,6 +876,16 @@ class HonchoHarnessPool: ) sys.exit(1) + # Wait for Redis to be ready + if not harness.wait_for_redis(): + print( + f"Redis failed to start for instance {harness.instance_id}. Exiting." + ) + sys.exit(1) + + # Initialize cache + await harness.init_cache() + # Provision database harness.provision_database() @@ -903,7 +1034,7 @@ Examples: base_api_port=args.api_port, project_root=args.project_root, ) - pool.run() + asyncio.run(pool.run()) else: harness = HonchoHarness( db_port=args.port, @@ -911,7 +1042,7 @@ Examples: project_root=args.project_root, instance_id=0, ) - harness.run() + asyncio.run(harness.run()) if __name__ == "__main__": diff --git a/tests/bench/longmem.py b/tests/bench/longmem.py index 6557ad74..c7182be2 100644 --- a/tests/bench/longmem.py +++ b/tests/bench/longmem.py @@ -870,7 +870,7 @@ Evaluate whether the actual response correctly answers the question based on the return results async def run_all_questions( - self, test_file: Path, batch_size: int = 10 + self, test_file: Path, batch_size: int = 10, test_count: int | None = None ) -> tuple[list[TestResult], float]: """ Run all questions in a longmemeval test file. @@ -878,11 +878,20 @@ Evaluate whether the actual response correctly answers the question based on the Args: test_file: Path to the longmemeval JSON file batch_size: Number of questions to run concurrently in each batch + test_count: Optional number of tests to run (runs first N tests) Returns: Tuple of (list of test results, total duration) """ questions = self.load_test_file(test_file) + + # Limit to first N questions if test_count is specified + if test_count is not None and test_count > 0: + questions = questions[:test_count] + print( + f"limiting to first {len(questions)} {'question' if len(questions) == 1 else 'questions'} from {test_file}" + ) + print( f"found {len(questions)} {'question' if len(questions) == 1 else 'questions'} in {test_file}" ) @@ -1140,6 +1149,7 @@ Examples: %(prog)s --test-file tests/bench/longmemeval_data/longmemeval_s.json # Run longmemeval tests %(prog)s --test-file test.json --pool-size 4 # Use 4 Honcho instances %(prog)s --test-file test.json --base-api-port 8000 --pool-size 4 # Custom base port with pool + %(prog)s --test-file test.json --test-count 50 # Run only first 50 tests """, ) @@ -1208,6 +1218,12 @@ Examples: help="Use get_context + judge LLM instead of dialectic .chat endpoint (default: False)", ) + parser.add_argument( + "--test-count", + type=int, + help="Number of tests to run from the test file (default: all tests)", + ) + args = parser.parse_args() # Validate arguments @@ -1223,6 +1239,10 @@ Examples: print(f"Error: Pool size must be positive, got {args.pool_size}") return 1 + if args.test_count is not None and args.test_count <= 0: + print(f"Error: Test count must be positive, got {args.test_count}") + return 1 + # Create test runner runner = LongMemEvalRunner( base_api_port=args.base_api_port, @@ -1237,7 +1257,7 @@ Examples: try: # Run all questions results, total_elapsed = await runner.run_all_questions( - args.test_file, args.batch_size + args.test_file, args.batch_size, args.test_count ) runner.print_summary(results, total_elapsed_seconds=total_elapsed) diff --git a/tests/conftest.py b/tests/conftest.py index 4b7cb9c1..1278ce78 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,9 @@ from unittest.mock import AsyncMock, MagicMock, patch import jwt import pytest import pytest_asyncio +from cashews.backends.interface import ControlMixin +from cashews.picklers import PicklerType +from fakeredis import FakeAsyncRedis from fastapi import Request from fastapi.responses import JSONResponse from fastapi.testclient import TestClient @@ -26,6 +29,7 @@ from sqlalchemy_utils import ( ) from src import models +from src.cache.client import cache from src.config import settings from src.db import Base from src.dependencies import get_db @@ -164,8 +168,79 @@ async def db_session(db_engine: AsyncEngine): await session.rollback() +@pytest_asyncio.fixture(scope="session") +async def fake_cache_session(): + """Set up fakeredis for caching once per test session.""" + # Store original settings + original_enabled = settings.CACHE.ENABLED + original_url = settings.CACHE.URL + + # Create a fake redis instance that persists for the session + fake_redis = FakeAsyncRedis(decode_responses=True) + + # Patch redis creation to use fakeredis + # Cashews uses redis.asyncio.from_url to create connections + def fake_redis_from_url(*_args: Any, **_kwargs: Any): + return fake_redis + + # Patch the cashews backend's _disable property to avoid ContextVar issues + # This works around cashews' ContextVar not being properly initialized in TestClient context + + original_disable_property = ControlMixin._disable # pyright: ignore[reportPrivateUsage] + + @property # type: ignore + def patched_disable_property(self): # pyright: ignore + try: + return original_disable_property.fget(self) # pyright: ignore[reportOptionalCall] + except LookupError: + # Return empty set as default if ContextVar not set in current context + return set() # pyright: ignore + + # Start patching + redis_patch = patch("redis.asyncio.from_url", fake_redis_from_url) + redis_patch.start() + ControlMixin._disable = patched_disable_property # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] + + try: + # Enable caching and set URL for tests + settings.CACHE.ENABLED = True + settings.CACHE.URL = "redis://fake-redis:6379/0" + + # Setup cache for tests that don't use TestClient (direct CRUD tests) + # For TestClient tests, the app's lifespan handler will also call cache.setup() + # The ContextVar patch above handles any context issues + cache.setup( # pyright: ignore[reportUnknownMemberType] + "redis://fake-redis:6379/0", pickle_type=PicklerType.SQLALCHEMY, enable=True + ) + + yield fake_redis + finally: + # Stop the patches + redis_patch.stop() + ControlMixin._disable = original_disable_property # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] + + # Restore original settings + settings.CACHE.ENABLED = original_enabled + settings.CACHE.URL = original_url + + +@pytest_asyncio.fixture(scope="function", autouse=True) +async def fake_cache(fake_cache_session: FakeAsyncRedis): + """Clear cache between tests.""" + # Clear cache before each test + await fake_cache_session.flushall() # pyright: ignore[reportUnknownMemberType] + + yield cache + + # Clear cache after each test + await fake_cache_session.flushall() # pyright: ignore[reportUnknownMemberType] + + @pytest.fixture(scope="function") -async def client(db_session: AsyncSession): +async def client( + db_session: AsyncSession, + fake_cache_session: FakeAsyncRedis, # pyright: ignore[reportUnusedParameter] +) -> AsyncGenerator[TestClient, Any]: """Create a FastAPI TestClient for the scope of a single test function""" # Register exception handlers for tests @@ -491,6 +566,7 @@ def mock_tracked_db(db_session: AsyncSession): patch("src.dependencies.tracked_db", mock_tracked_db_context), patch("src.deriver.queue_manager.tracked_db", mock_tracked_db_context), patch("src.routers.sessions.tracked_db", mock_tracked_db_context), + patch("src.routers.peers.tracked_db", mock_tracked_db_context), patch("src.crud.representation.tracked_db", mock_tracked_db_context), patch("src.routers.peers.tracked_db", mock_tracked_db_context), patch("src.dreamer.dreamer.tracked_db", mock_tracked_db_context), diff --git a/tests/crud/test_workspace.py b/tests/crud/test_workspace.py index 18601128..d8ea00a5 100644 --- a/tests/crud/test_workspace.py +++ b/tests/crud/test_workspace.py @@ -318,6 +318,7 @@ class TestWorkspaceCRUD: work_unit_key=f"representation:{test_workspace.name}:{session.name}:{test_peer.name}:{test_peer.name}", task_type="representation", payload={"test": "data"}, + workspace_name=test_workspace.name, ) db_session.add(queue_item) await db_session.flush() diff --git a/tests/deriver/conftest.py b/tests/deriver/conftest.py index 89410656..0a1e561e 100644 --- a/tests/deriver/conftest.py +++ b/tests/deriver/conftest.py @@ -1,7 +1,7 @@ import asyncio -from collections.abc import Callable, Generator +from collections.abc import Awaitable, Callable, Generator, Sequence from datetime import datetime, timezone -from typing import Any, Literal +from typing import Any, Literal, TypeAlias, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -11,7 +11,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models, schemas from src.utils.queue_payload import create_payload -from src.utils.work_unit import get_work_unit_key +from src.utils.work_unit import construct_work_unit_key + +QueuePayload: TypeAlias = dict[str, Any] +QueuePayloadEntry: TypeAlias = QueuePayload | tuple[QueuePayload, int | None] @pytest.fixture @@ -155,18 +158,29 @@ def create_queue_payload() -> Callable[..., Any]: @pytest.fixture async def add_queue_items( db_session: AsyncSession, -) -> Callable[[list[dict[str, Any]], str], Any]: +) -> Callable[ + [Sequence[QueuePayloadEntry], str, str], Awaitable[list[models.QueueItem]] +]: """Helper function to add queue items to the database""" async def _add_items( - payloads: list[dict[str, Any]], session_id: str + payloads: Sequence[QueuePayloadEntry], + session_id: str, + workspace_name: str, ) -> list[models.QueueItem]: """Add queue items to the database and return them""" queue_items: list[models.QueueItem] = [] - for payload in payloads: + for payload_entry in payloads: + payload: QueuePayload + message_id: int | None + if isinstance(payload_entry, tuple): + payload, message_id = payload_entry + else: + payload = payload_entry + message_id = cast(int | None, payload.get("message_id")) # Generate work_unit_key from the payload - task_type = payload.get("task_type", "unknown") - work_unit_key = get_work_unit_key(payload) + task_type = cast(str, payload.get("task_type", "unknown")) + work_unit_key = construct_work_unit_key(workspace_name, payload) queue_item = models.QueueItem( session_id=session_id, @@ -174,6 +188,8 @@ async def add_queue_items( work_unit_key=work_unit_key, payload=payload, processed=False, + workspace_name=workspace_name, + message_id=message_id, ) db_session.add(queue_item) queue_items.append(queue_item) @@ -203,7 +219,7 @@ async def sample_queue_items( messages = sample_messages # Create various types of queue payloads - payloads: list[dict[str, Any]] = [] + payloads: list[tuple[dict[str, Any], int]] = [] # Create representation payloads for each message for message in messages: @@ -214,7 +230,7 @@ async def sample_queue_items( observer=message.peer_name, observed=message.peer_name, ) - payloads.append(payload1) + payloads.append((payload1, message.id)) # Representation for observer peer payload2 = create_queue_payload( @@ -223,7 +239,7 @@ async def sample_queue_items( observer=peer2.name, # peer2 observes others observed=message.peer_name, ) - payloads.append(payload2) + payloads.append((payload2, message.id)) # Create summary payloads for session for i, message in enumerate(messages): @@ -232,10 +248,10 @@ async def sample_queue_items( task_type="summary", message_seq_in_session=i + 1, ) - payloads.append(payload) + payloads.append((payload, message.id)) # Add all payloads as queue items - queue_items = await add_queue_items(payloads, session.id) + queue_items = await add_queue_items(payloads, session.id, session.workspace_name) return queue_items @@ -294,6 +310,5 @@ def mock_representation_manager(monkeypatch: pytest.MonkeyPatch) -> AsyncMock: mock_manager = AsyncMock(spec=RepresentationManager) mock_manager.save_representation.return_value = 0 - mock_manager.get_relevant_observations = AsyncMock(return_value=MagicMock()) return mock_manager diff --git a/tests/deriver/test_deriver_processing.py b/tests/deriver/test_deriver_processing.py index 002377df..c47ac954 100644 --- a/tests/deriver/test_deriver_processing.py +++ b/tests/deriver/test_deriver_processing.py @@ -9,6 +9,7 @@ import pytest from src import models from src.deriver.deriver import process_representation_tasks_batch from src.utils.representation import Representation +from src.utils.work_unit import construct_work_unit_key @pytest.mark.asyncio @@ -32,14 +33,12 @@ class TestDeriverProcessing: sample_session_with_peers: tuple[models.Session, list[models.Peer]], ): """Test that work unit keys are generated correctly""" - from src.utils.work_unit import get_work_unit_key session, peers = sample_session_with_peers peer1, peer2, _ = peers # Create a payload for representation task representation_payload = { - "workspace_name": "workspace1", "session_name": session.name, "observer": peer2.name, "observed": peer1.name, @@ -47,22 +46,25 @@ class TestDeriverProcessing: } # Generate work unit key for representation - work_unit_key = get_work_unit_key(representation_payload) - expected_key = ( - f"representation:workspace1:{session.name}:{peer2.name}:{peer1.name}" + work_unit_key = construct_work_unit_key( + session.workspace_name, representation_payload ) + expected_key = f"representation:{session.workspace_name}:{session.name}:{peer2.name}:{peer1.name}" assert work_unit_key == expected_key # Create a payload for summary task summary_payload = { - "workspace_name": "workspace1", "session_name": session.name, "task_type": "summary", } # Generate work unit key for summary - summary_work_unit_key = get_work_unit_key(summary_payload) - expected_summary_key = f"summary:workspace1:{session.name}:None:None" + summary_work_unit_key = construct_work_unit_key( + session.workspace_name, summary_payload + ) + expected_summary_key = ( + f"summary:{session.workspace_name}:{session.name}:None:None" + ) assert summary_work_unit_key == expected_summary_key async def test_mock_queue_manager( @@ -100,7 +102,6 @@ class TestDeriverProcessing: await mock_representation_manager.save_representation( Representation(explicit=[], deductive=[]) ) - mock_representation_manager.get_relevant_observations.return_value = [] # type: ignore[attr-defined] # Verify the methods were called assert mock_representation_manager.save_representation.called # type: ignore[attr-defined] diff --git a/tests/deriver/test_queue_processing.py b/tests/deriver/test_queue_processing.py index d620392a..dfb85605 100644 --- a/tests/deriver/test_queue_processing.py +++ b/tests/deriver/test_queue_processing.py @@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from src import models from src.config import settings from src.deriver.queue_manager import QueueManager, WorkerOwnership -from src.utils.work_unit import get_work_unit_key +from src.utils.work_unit import construct_work_unit_key @pytest.mark.asyncio @@ -139,7 +139,7 @@ class TestQueueProcessing: for message in messages: await db_session.refresh(message) - payloads: list[Any] = [] + payloads: list[tuple[dict[str, Any], int]] = [] for message in messages: payload = create_queue_payload( # type: ignore[reportUnknownArgumentType] message=message, @@ -147,9 +147,9 @@ class TestQueueProcessing: observed=peer.name, observer=peer.name, ) - payloads.append(payload) + payloads.append((payload, message.id)) - items = await add_queue_items(payloads, session.id) + items = await add_queue_items(payloads, session.id, session.workspace_name) # Determine ascending order by DB id ordered = ( ( @@ -304,20 +304,23 @@ class TestQueueProcessing: await db_session.refresh(message) # Create queue items with token counts - payloads = [ - create_queue_payload( # type: ignore[reportUnknownArgumentType] - message=msg, - task_type="representation", - observed=peer.name, - observer=peer.name, + payload_entries = [ + ( + create_queue_payload( # type: ignore[reportUnknownArgumentType] + message=msg, + task_type="representation", + observed=peer.name, + observer=peer.name, + ), + msg, ) for msg in messages ] queue_items: list[models.QueueItem] = [] - for payload in payloads: + for payload, message in payload_entries: task_type = payload.get("task_type", "unknown") - work_unit_key = get_work_unit_key(payload) + work_unit_key = construct_work_unit_key(session.workspace_name, payload) queue_item = models.QueueItem( session_id=session.id, @@ -325,6 +328,8 @@ class TestQueueProcessing: work_unit_key=work_unit_key, payload=payload, processed=False, + workspace_name=session.workspace_name, + message_id=message.id, ) db_session.add(queue_item) queue_items.append(queue_item) @@ -430,7 +435,7 @@ class TestQueueProcessing: observed=peer.name, observer=target.name, ) - work_unit_key = get_work_unit_key(payload) + work_unit_key = construct_work_unit_key(session.workspace_name, payload) queue_item = models.QueueItem( session_id=session.id, @@ -438,6 +443,8 @@ class TestQueueProcessing: work_unit_key=work_unit_key, payload=payload, processed=False, + workspace_name=session.workspace_name, + message_id=message.id, ) db_session.add(queue_item) @@ -596,7 +603,7 @@ class TestQueueProcessing: observed=peer.name, observer=target.name, ) - work_unit_key = get_work_unit_key(payload) + work_unit_key = construct_work_unit_key(session.workspace_name, payload) queue_item = models.QueueItem( session_id=session.id, @@ -604,6 +611,8 @@ class TestQueueProcessing: work_unit_key=work_unit_key, payload=payload, processed=False, + workspace_name=session.workspace_name, + message_id=message.id, ) db_session.add(queue_item) @@ -702,7 +711,6 @@ class TestQueueProcessing: token_counts = [500, 600] messages = [ models.Message( - id=999, session_name=session.name, workspace_name=session.workspace_name, peer_name=peer.name, @@ -711,7 +719,6 @@ class TestQueueProcessing: seq_in_session=1, ), models.Message( - id=1000, session_name=session.name, workspace_name=session.workspace_name, peer_name=peer.name, @@ -721,6 +728,15 @@ class TestQueueProcessing: ), ] + # Save messages to database first + for message in messages: + db_session.add(message) + await db_session.commit() + + # Refresh to get the actual IDs + for message in messages: + await db_session.refresh(message) + # Create payloads and queue items queue_items: list[models.QueueItem] = [] for i, message in enumerate(messages): @@ -729,7 +745,7 @@ class TestQueueProcessing: ) payload["token_count"] = token_counts[i] - work_unit_key = get_work_unit_key(payload) + work_unit_key = construct_work_unit_key(session.workspace_name, payload) queue_item = models.QueueItem( session_id=session.id, @@ -737,6 +753,8 @@ class TestQueueProcessing: work_unit_key=work_unit_key, payload=payload, processed=False, + workspace_name=session.workspace_name, + message_id=message.id, ) db_session.add(queue_item) queue_items.append(queue_item) @@ -747,10 +765,11 @@ class TestQueueProcessing: processed_batches: list[dict[str, Any]] = [] async def mock_process_item( - task_type: str, - queue_payload: dict[str, Any], # pyright: ignore[reportUnusedParameter] + queue_item: models.QueueItem, ) -> None: - processed_batches.append({"task_type": task_type, "payload_count": 1}) + processed_batches.append( + {"task_type": queue_item.task_type, "payload_count": 1} + ) qm = QueueManager() work_unit_key = queue_items[0].work_unit_key @@ -838,21 +857,24 @@ class TestQueueProcessing: await db_session.refresh(message) # Create queue items - payloads = [ - create_queue_payload( # type: ignore[reportUnknownArgumentType] - message=msg, - task_type="representation", - observed=peer.name, - observer=peer.name, + payload_entries = [ + ( + create_queue_payload( # type: ignore[reportUnknownArgumentType] + message=msg, + task_type="representation", + observed=peer.name, + observer=peer.name, + ), + msg, ) for msg in messages ] # Add items to queue queue_items: list[models.QueueItem] = [] - for payload in payloads: + for payload, message in payload_entries: task_type = payload.get("task_type", "unknown") - work_unit_key = get_work_unit_key(payload) + work_unit_key = construct_work_unit_key(session.workspace_name, payload) queue_item = models.QueueItem( session_id=session.id, @@ -860,6 +882,8 @@ class TestQueueProcessing: work_unit_key=work_unit_key, payload=payload, processed=False, + workspace_name=session.workspace_name, + message_id=message.id, ) db_session.add(queue_item) queue_items.append(queue_item) @@ -949,21 +973,24 @@ class TestQueueProcessing: await db_session.refresh(message) # Create queue items - payloads = [ - create_queue_payload( # type: ignore[reportUnknownArgumentType] - message=msg, - task_type="representation", - observed=peer.name, - observer=peer.name, + payload_entries = [ + ( + create_queue_payload( # type: ignore[reportUnknownArgumentType] + message=msg, + task_type="representation", + observed=peer.name, + observer=peer.name, + ), + msg, ) for msg in messages ] # Add items to queue queue_items: list[models.QueueItem] = [] - for payload in payloads: + for payload, message in payload_entries: task_type = payload.get("task_type", "unknown") - work_unit_key = get_work_unit_key(payload) + work_unit_key = construct_work_unit_key(session.workspace_name, payload) queue_item = models.QueueItem( session_id=session.id, @@ -971,6 +998,8 @@ class TestQueueProcessing: work_unit_key=work_unit_key, payload=payload, processed=False, + workspace_name=session.workspace_name, + message_id=message.id, ) db_session.add(queue_item) queue_items.append(queue_item) diff --git a/tests/integration/test_enqueue.py b/tests/integration/test_enqueue.py index 5b1870e8..6e5fda30 100644 --- a/tests/integration/test_enqueue.py +++ b/tests/integration/test_enqueue.py @@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models, schemas from src.deriver import enqueue +from src.deriver.enqueue import generate_queue_records from src.models import Peer, QueueItem, Workspace @@ -1379,3 +1380,146 @@ class TestAdvancedEnqueueEdgeCases: assert len(actual_payloads) == len(expected_payloads) for expected in expected_payloads: assert expected in actual_payloads + + +@pytest.mark.asyncio +class TestGenerateQueueRecordsSeqInSession: + """Unit tests for generate_queue_records function focusing on seq_in_session handling""" + + async def test_generate_queue_records_uses_seq_from_payload_not_crud( + self, + db_session: AsyncSession, + sample_data: tuple[Workspace, Peer], + ): + """ + Test that generate_queue_records uses seq_in_session from payload + instead of making a CRUD call to get_message_seq_in_session. + """ + + test_workspace, test_peer = sample_data + + # Create a test session + test_session = models.Session( + workspace_name=test_workspace.name, name=str(generate_nanoid()) + ) + db_session.add(test_session) + await db_session.commit() + + # Create a message payload with seq_in_session included + message_payload = { + "message_id": 12345, + "peer_name": test_peer.name, + "workspace_name": test_workspace.name, + "session_name": test_session.name, + "content": "Test message", + "seq_in_session": 20, # Multiple of MESSAGES_PER_SHORT_SUMMARY to trigger summary creation + "created_at": datetime.now(timezone.utc), # Required by create_payload + } + + # Mock the CRUD function to track if it's called + # Also enable summary generation in settings + with ( + patch("src.deriver.enqueue.crud.get_message_seq_in_session") as mock_crud, + patch("src.deriver.enqueue.settings.SUMMARY.ENABLED", new=True), + ): + mock_crud.return_value = 200 + mock_db_session = AsyncMock() + + peers_config: dict[str, list[Any]] = { + test_peer.name: [ + {"observe_me": True}, + {"observe_others": True}, + ] + } + records = await generate_queue_records( + db_session=mock_db_session, + message=message_payload, + peers_with_configuration=peers_config, + session_id=test_session.id, + deriver_disabled=False, + ) + + mock_crud.assert_not_called() + + assert len(records) > 0 + + summary_records = [r for r in records if r["task_type"] == "summary"] + assert len(summary_records) > 0, "Expected summary records to be created" + for record in summary_records: + assert ( + record["payload"]["message_seq_in_session"] + != mock_crud.return_value + ) + assert record["payload"]["message_seq_in_session"] == 20 + + async def test_generate_queue_records_falls_back_to_crud_when_seq_missing( + self, + db_session: AsyncSession, + sample_data: tuple[Workspace, Peer], + ): + """ + Test that generate_queue_records falls back to CRUD call + when seq_in_session is missing from payload. + + This is the fallback behavior for backward compatibility. + """ + + test_workspace, test_peer = sample_data + + # Create a test session + test_session = models.Session( + workspace_name=test_workspace.name, name=str(generate_nanoid()) + ) + db_session.add(test_session) + await db_session.commit() + + # Create a message payload WITHOUT seq_in_session + message_payload = { + "message_id": 12345, + "peer_name": test_peer.name, + "workspace_name": test_workspace.name, + "session_name": test_session.name, + "content": "Test message", + "created_at": datetime.now(timezone.utc), + # seq_in_session is MISSING + } + + # Mock the CRUD function and enable summary generation in settings + with ( + patch("src.deriver.enqueue.crud.get_message_seq_in_session") as mock_crud, + patch("src.deriver.enqueue.settings.SUMMARY.ENABLED", True), + ): + mock_crud.return_value = ( + 60 # Multiple of MESSAGES_PER_LONG_SUMMARY to trigger summary creation + ) + + mock_db_session = AsyncMock() + + peers_config: dict[str, list[Any]] = { + test_peer.name: [ + {"observe_me": True}, + {"observe_others": True}, + ] + } + records = await generate_queue_records( + db_session=mock_db_session, + message=message_payload, + peers_with_configuration=peers_config, + session_id=test_session.id, + deriver_disabled=False, + ) + + # The CRUD function SHOULD have been called as fallback + mock_crud.assert_called_once_with( + mock_db_session, + workspace_name=test_workspace.name, + session_name=test_session.name, + message_id=12345, + ) + + # Verify that records were created with the fallback value + summary_records = [r for r in records if r["task_type"] == "summary"] + assert len(summary_records) > 0, "Expected summary records to be created" + for record in summary_records: + # Should use the value from CRUD fallback (60) + assert record["payload"]["message_seq_in_session"] == 60 diff --git a/tests/integration/test_representation.py b/tests/integration/test_representation.py index 15d8dd37..f3e112b5 100644 --- a/tests/integration/test_representation.py +++ b/tests/integration/test_representation.py @@ -222,8 +222,8 @@ class TestDocumentCreationWorkflow: observed=observed_peer.name, content="User likes dogs", session_name="test_session", + level="explicit", internal_metadata={ - "level": "explicit", "message_ids": [(1, 1)], "session_name": "test_session", }, @@ -274,7 +274,9 @@ class TestDocumentCreationWorkflow: observed=observed_peer.name, content="Highly derived observation", session_name=session.name, - internal_metadata={"level": "explicit", "times_derived": 5}, + level="explicit", + times_derived=5, + internal_metadata={}, embedding=[0.1] * 1536, ) db_session.add(highly_derived_doc) @@ -286,7 +288,9 @@ class TestDocumentCreationWorkflow: observed=observed_peer.name, content="Less derived observation", session_name=session.name, - internal_metadata={"level": "explicit", "times_derived": 2}, + level="explicit", + times_derived=2, + internal_metadata={}, embedding=[0.2] * 1536, ) db_session.add(less_derived_doc) @@ -315,8 +319,8 @@ class TestDocumentCreationWorkflow: observer="test_peer", observed="test_peer", content="User said they like programming", + level="explicit", internal_metadata={ - "level": "explicit", "message_ids": [(1, 1)], }, session_name="test_session", @@ -329,8 +333,8 @@ class TestDocumentCreationWorkflow: observer="test_peer", observed="test_peer", content="User is likely a software developer", + level="deductive", internal_metadata={ - "level": "deductive", "message_ids": [(1, 1)], "premises": ["User said they like programming"], }, diff --git a/tests/routes/test_queue_status.py b/tests/routes/test_queue_status.py index 85fd048f..bc50dcc7 100644 --- a/tests/routes/test_queue_status.py +++ b/tests/routes/test_queue_status.py @@ -3,7 +3,7 @@ from fastapi.testclient import TestClient from sqlalchemy.ext.asyncio import AsyncSession from src import models -from src.utils.work_unit import get_work_unit_key +from src.utils.work_unit import construct_work_unit_key @pytest.mark.asyncio @@ -157,9 +157,10 @@ class TestDeriverStatusEndpoint: queue_item = models.QueueItem( session_id=session.id, task_type="representation", - work_unit_key=get_work_unit_key(payload), + work_unit_key=construct_work_unit_key(workspace.name, payload), payload=payload, processed=False, + workspace_name=workspace.name, ) queue_items.append(queue_item) db_session.add_all(queue_items) @@ -231,9 +232,10 @@ class TestDeriverStatusEndpoint: queue_item = models.QueueItem( session_id=session.id, task_type="representation", - work_unit_key=get_work_unit_key(payload), + work_unit_key=construct_work_unit_key(workspace.name, payload), payload=payload, processed=False, + workspace_name=workspace.name, ) queue_items.append(queue_item) db_session.add_all(queue_items) @@ -292,9 +294,10 @@ class TestDeriverStatusEndpoint: queue_item = models.QueueItem( session_id=session.id, task_type="representation", - work_unit_key=get_work_unit_key(payload), + work_unit_key=construct_work_unit_key(workspace.name, payload), payload=payload, processed=False, + workspace_name=workspace.name, ) db_session.add(queue_item) await db_session.commit() diff --git a/tests/routes/test_webhooks.py b/tests/routes/test_webhooks.py index ed176473..8272dc9f 100644 --- a/tests/routes/test_webhooks.py +++ b/tests/routes/test_webhooks.py @@ -95,13 +95,6 @@ async def test_list_webhook_endpoints_with_data( assert "http://example2.com/webhook" in endpoint_urls -@pytest.mark.asyncio -async def test_list_webhook_endpoints_missing_workspace(client: TestClient): - response = client.get("/v2/workspaces/nonexistent-workspace/webhooks") - assert response.status_code == 404 - assert response.json() == {"detail": "Workspace nonexistent-workspace not found"} - - @pytest.mark.asyncio async def test_delete_webhook_endpoint( client: TestClient, sample_data: tuple[Workspace, Peer] diff --git a/tests/test_schema_validations.py b/tests/test_schema_validations.py index 86ad1b37..775c32b8 100644 --- a/tests/test_schema_validations.py +++ b/tests/test_schema_validations.py @@ -95,17 +95,18 @@ class TestDocumentValidations: def test_valid_document_create(self): metadata = DocumentMetadata( message_ids=[(1, 1)], - level="explicit", premises=[], message_created_at="2021-01-01T00:00:00Z", ) doc = DocumentCreate( content="test content", session_name="test", + level="explicit", metadata=metadata, embedding=[0.1, 0.2, 0.3], ) assert doc.content == "test content" + assert doc.level == "explicit" assert doc.metadata == metadata def test_document_content_too_short(self): @@ -113,9 +114,9 @@ class TestDocumentValidations: DocumentCreate( content="", session_name="test", + level="explicit", metadata=DocumentMetadata( message_ids=[(1, 1)], - level="explicit", premises=[], message_created_at="2021-01-01T00:00:00Z", ), @@ -129,9 +130,9 @@ class TestDocumentValidations: DocumentCreate( content="a" * 100001, session_name="test", + level="explicit", metadata=DocumentMetadata( message_ids=[(1, 1)], - level="explicit", premises=[], message_created_at="2021-01-01T00:00:00Z", ), diff --git a/uv.lock b/uv.lock index dfe736ee..01ae81c3 100644 --- a/uv.lock +++ b/uv.lock @@ -70,6 +70,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" }, ] +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + [[package]] name = "attrs" version = "25.3.0" @@ -118,6 +127,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, ] +[[package]] +name = "cashews" +version = "7.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/02/6c63550c84263219367e027038ccdac9bff900775262027ac35b6de91973/cashews-7.4.1.tar.gz", hash = "sha256:9d4ac7b0d0e20ec96680af60ae15dc26c19ccc267baa84a472c54bef86a93a8a", size = 91757, upload-time = "2025-07-14T22:39:48.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/e6/e77b27292b560725c35e478e90bdc9fe84c6ec849daac4360b4150083b4f/cashews-7.4.1-py3-none-any.whl", hash = "sha256:868019e9c8b0a75f345ea58b71197640b20dc2fe0892eb5ef6537f5652299ba4", size = 79356, upload-time = "2025-07-14T22:39:46.719Z" }, +] + +[package.optional-dependencies] +redis = [ + { name = "redis" }, +] + [[package]] name = "certifi" version = "2025.8.3" @@ -464,6 +487,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] +[[package]] +name = "fakeredis" +version = "2.32.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "redis" }, + { name = "sortedcontainers" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/2e/94ca3f2ff35f086d7d3eeb924054e328b2ac851f0a20302d942c8d29726c/fakeredis-2.32.0.tar.gz", hash = "sha256:63d745b40eb6c8be4899cf2a53187c097ccca3afbca04fdbc5edc8b936cd1d59", size = 171097, upload-time = "2025-10-07T10:46:58.876Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/1b/84ab7fd197eba5243b6625c78fbcffaa4cf6ac7dda42f95d22165f52187e/fakeredis-2.32.0-py3-none-any.whl", hash = "sha256:c9da8228de84060cfdb72c3cf4555c18c59ba7a5ae4d273f75e4822d6f01ecf8", size = 118422, upload-time = "2025-10-07T10:46:57.643Z" }, +] + [[package]] name = "fastapi" version = "0.116.1" @@ -673,10 +710,11 @@ wheels = [ [[package]] name = "honcho" -version = "2.4.1" +version = "2.4.2" source = { virtual = "." } dependencies = [ { name = "alembic" }, + { name = "cashews", extra = ["redis"] }, { name = "fastapi", extra = ["standard"] }, { name = "fastapi-pagination" }, { name = "google-genai" }, @@ -695,6 +733,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pyjwt" }, { name = "python-dotenv" }, + { name = "redis" }, { name = "rich" }, { name = "sentry-sdk", extra = ["anthropic", "fastapi", "sqlalchemy"] }, { name = "sqlalchemy" }, @@ -707,6 +746,7 @@ dependencies = [ dev = [ { name = "basedpyright" }, { name = "coverage" }, + { name = "fakeredis" }, { name = "honcho-ai" }, { name = "interrogate" }, { name = "pre-commit" }, @@ -721,6 +761,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "alembic", specifier = ">=1.14.0" }, + { name = "cashews", extras = ["redis"], specifier = "==7.4.1" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.111.0" }, { name = "fastapi-pagination", specifier = ">=0.12.24" }, { name = "google-genai", specifier = ">=1.32.0" }, @@ -739,6 +780,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.10.1" }, { name = "pyjwt", specifier = ">=2.10.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "redis", specifier = ">=6.0.0" }, { name = "rich", specifier = ">=13.7.1" }, { name = "sentry-sdk", extras = ["anthropic", "fastapi", "sqlalchemy"], specifier = ">=2.3.1" }, { name = "sqlalchemy", specifier = ">=2.0.30" }, @@ -751,6 +793,7 @@ requires-dist = [ dev = [ { name = "basedpyright", specifier = ">=1.29.4" }, { name = "coverage", specifier = ">=7.6.0" }, + { name = "fakeredis", specifier = ">=2.32.0" }, { name = "honcho-ai", editable = "sdks/python" }, { name = "interrogate", specifier = ">=1.7.0" }, { name = "pre-commit", specifier = ">=4.2.0" }, @@ -1967,6 +2010,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, ] +[[package]] +name = "redis" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/d6/e8b92798a5bd67d659d51a18170e91c16ac3b59738d91894651ee255ed49/redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010", size = 4647399, upload-time = "2025-08-07T08:10:11.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/02/89e2ed7e85db6c93dfa9e8f691c5087df4e3551ab39081a4d7c6d1f90e05/redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f", size = 279847, upload-time = "2025-08-07T08:10:09.84Z" }, +] + [[package]] name = "regex" version = "2025.7.34" @@ -2261,6 +2316,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.42"