diff --git a/.claude/skills/migrate-honcho-py/MIGRATION-CHECKLIST.md b/.claude/skills/migrate-honcho-py/MIGRATION-CHECKLIST.md index 7f0b9750..5a2d01ff 100644 --- a/.claude/skills/migrate-honcho-py/MIGRATION-CHECKLIST.md +++ b/.claude/skills/migrate-honcho-py/MIGRATION-CHECKLIST.md @@ -85,6 +85,7 @@ Use this checklist to track migration progress. Copy into your working notes and - [ ] `include_most_derived=` → `include_most_frequent=` - [ ] `max_observations=` → `max_conclusions=` +- [ ] `last_user_message=` → `search_query=` ## Return Type Changes diff --git a/.claude/skills/migrate-honcho-py/SKILL.md b/.claude/skills/migrate-honcho-py/SKILL.md index 1ed11c9c..57c59fa7 100644 --- a/.claude/skills/migrate-honcho-py/SKILL.md +++ b/.claude/skills/migrate-honcho-py/SKILL.md @@ -225,6 +225,7 @@ if card: | `chat(stream=True)` | `chat_stream()` | | `include_most_derived=` | `include_most_frequent=` | | `max_observations=` | `max_conclusions=` | +| `last_user_message=` | `search_query=` | | `config=` | `configuration=` | | `PeerContext` | `PeerContextResponse` | | `DeriverStatus` | `QueueStatusResponse` | diff --git a/.claude/skills/migrate-honcho-ts/DETAILED-CHANGES.md b/.claude/skills/migrate-honcho-ts/DETAILED-CHANGES.md index 776b341b..76273d34 100644 --- a/.claude/skills/migrate-honcho-ts/DETAILED-CHANGES.md +++ b/.claude/skills/migrate-honcho-ts/DETAILED-CHANGES.md @@ -186,6 +186,7 @@ const ctx = await session.getContext({ summary: true, peerTarget: user, peerPerspective: assistant, + lastUserMessage: "What are my preferences?", representationOptions: { maxObservations: 50, includeMostDerived: true @@ -197,6 +198,7 @@ const ctx = await session.context({ summary: true, peerTarget: user, peerPerspective: assistant, + searchQuery: "What are my preferences?", representationOptions: { maxConclusions: 50, includeMostFrequent: true diff --git a/.claude/skills/migrate-honcho-ts/MIGRATION-CHECKLIST.md b/.claude/skills/migrate-honcho-ts/MIGRATION-CHECKLIST.md index 78fcc6bd..6adcf94b 100644 --- a/.claude/skills/migrate-honcho-ts/MIGRATION-CHECKLIST.md +++ b/.claude/skills/migrate-honcho-ts/MIGRATION-CHECKLIST.md @@ -49,6 +49,7 @@ Use this checklist to track migration progress. Copy into your working notes and - [ ] Rename `maxObservations` → `maxConclusions` - [ ] Rename `includeMostDerived` → `includeMostFrequent` +- [ ] Rename `lastUserMessage` → `searchQuery` - [ ] Rename `Observation` type → `Conclusion` - [ ] Rename `ObservationScope` type → `ConclusionScope` diff --git a/.claude/skills/migrate-honcho-ts/SKILL.md b/.claude/skills/migrate-honcho-ts/SKILL.md index 5a45e14f..876cc5f9 100644 --- a/.claude/skills/migrate-honcho-ts/SKILL.md +++ b/.claude/skills/migrate-honcho-ts/SKILL.md @@ -175,6 +175,7 @@ await session.updateMessage(message, metadata) | `{ timeoutMs: 60000 }` | `{ timeout: 60 }` | | `{ maxObservations: 50 }` | `{ maxConclusions: 50 }` | | `{ includeMostDerived }` | `{ includeMostFrequent }` | +| `{ lastUserMessage }` | `{ searchQuery }` | | `{ config: ... }` | `{ configuration: ... }` | | `message.peer_id` | `message.peerId` | | `message.created_at` | `message.createdAt` | diff --git a/.env.template b/.env.template index 8593b237..1aa0f586 100644 --- a/.env.template +++ b/.env.template @@ -229,14 +229,10 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # SENTRY_PROFILES_SAMPLE_RATE=0.1 # ============================================================================= -# OpenTelemetry Settings (Push-based metrics via OTLP) +# Prometheus Metrics Settings (Pull-based metrics) # ============================================================================= -# OTEL_ENABLED=false -# OTEL_ENDPOINT=https://mimir.example.com/otlp/v1/metrics -# OTEL_HEADERS={"X-Scope-OrgID": "honcho"} # JSON string for auth headers -# OTEL_EXPORT_INTERVAL_MILLIS=60000 -# OTEL_SERVICE_NAME=honcho -# OTEL_SERVICE_NAMESPACE=honcho # Inherits from NAMESPACE if not set +# METRICS_ENABLED=false +# METRICS_NAMESPACE=honcho # Inherits from NAMESPACE if not set # ============================================================================= # CloudEvents Telemetry Settings (Analytics events) diff --git a/.gitignore b/.gitignore index e7aae15e..9a31fc38 100644 --- a/.gitignore +++ b/.gitignore @@ -190,6 +190,4 @@ CRUSH.md metrics.jsonl AGENTS.md lancedb_data/ - -mimir-data/ grafana-data/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bb1c9c3..5cb77854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [3.0.1] - 2026-01-27 + +### Fixed + +- Token counting in Explicit Agent Loop +- Backwards compatibility of queue items + ## [3.0.0] - 2026-01-19 ### Added @@ -14,18 +21,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Reasoning levels configuration for dialectic (`minimal`, `low`, `medium`, `high`, `max`) - Prometheus token tracking for deriver and dialectic operations - n8n integration +- Cloud Events for auditable telemetry +- External Vector Store support for turbopuffer and lancedb with reconciliation flow ### Changed - API route renaming for consistency - Dreamer and dialectic now respect peer card configuration settings - Observations renamed to Conclusions across API and SDKs +- Deriver to buffer representation tasks to normalize workloads +- Local Representation tasks to create singular QueueItems +- getContext endpoint to use `search_query` rather than force `last_user_message` ### Fixed - Dream scheduling bugs - Summary creation when start_message_id > end_message_id - Cashews upgrade to prevent NoScriptError +- Memory leak in `accumulate_metric` call ### Removed diff --git a/README.md b/README.md index d9684333..bc357667 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ --- -![Static Badge](https://img.shields.io/badge/Version-3.0.0-blue) +![Static Badge](https://img.shields.io/badge/Version-3.0.1-blue) [![PyPI version](https://img.shields.io/pypi/v/honcho-ai.svg)](https://pypi.org/project/honcho-ai/) [![NPM version](https://img.shields.io/npm/v/@honcho-ai/sdk.svg)](https://npmjs.org/package/@honcho-ai/sdk) [![Discord](https://img.shields.io/discord/1016845111637839922?style=flat&logo=discord&logoColor=23ffffff&label=Plastic%20Labs&labelColor=235865F2)](https://discord.gg/plasticlabs) @@ -44,26 +44,23 @@ poetry add honcho-ai ```python from honcho import Honcho -####### Storing Data in Honcho - # 1. Initialize your Honcho client honcho = Honcho(workspace_id="my-app-testing") -# 2.. Initialize Peers +# 2. Initialize peers alice = honcho.peer("alice") tutor = honcho.peer("tutor") -# 3. Make a Session and send messages +# 3. Create a session and add messages session = honcho.session("session-1") - -session.add_messages([ - alice.message("Hey there can you help me with my math homework"), - tutor.message("Absolutely send me your first problem!"), - . - . - . -]) +# Adding messages from a peer will automatically add them to the session +session.add_messages( + [ + alice.message("Hey there — can you help me with my math homework?"), + tutor.message("Absolutely. Send me your first problem!"), + ] +) ``` 3. Leverage reasoning from Honcho to inform your agent's behavior @@ -73,14 +70,14 @@ session.add_messages([ ### 1. Use the chat endpoint to ask questions about your users in natural language response = alice.chat("What learning styles does the user respond to best?") -### 2. Use Get context to get most recent messages and summaries to continue a conversation -context = session.get_context(summary=True, tokens=10000) +### 2. Use session context to continue a conversation with an LLM +context = session.context(summary=True, tokens=10_000) # Convert to a format to send to OpenAI and get the next message -openai_messages = context.to_openai_messages(assistant=tutor) +openai_messages = context.to_openai(assistant=tutor) from openai import OpenAI -client = Openai() +client = OpenAI() response = client.chat.completions.create( model="gpt-4", messages=openai_messages @@ -89,8 +86,8 @@ response = client.chat.completions.create( ### 3. Search for similar messages results = alice.search("Math Homework") -### 4. Get a cached representation of a Peer for the Session -alice_representation = session.working_rep("alice") +### 4. Get a session-scoped representation of a peer +alice_representation = session.representation(alice) ``` @@ -414,7 +411,7 @@ Then modify the values as needed. The TOML file is organized into sections: - `[summary]` - Session summarization settings - `[dream]` - Dream processing configuration (including specialist models and surprisal settings) - `[webhook]` - Webhook configuration -- `[otel]` - OpenTelemetry push-based metrics via OTLP +- `[metrics]` - Prometheus pull-based metrics - `[telemetry]` - CloudEvents telemetry for analytics - `[vector_store]` - Vector store configuration (pgvector, turbopuffer, or lancedb) - `[sentry]` - Error tracking and monitoring settings @@ -434,7 +431,7 @@ Examples: - `DERIVER_PROVIDER` - Provider for background deriver - `SUMMARY_PROVIDER` - Summary generation provider - `LOG_LEVEL` - Application log level -- `OTEL_ENABLED` - Enable OpenTelemetry metrics +- `METRICS_ENABLED` - Enable Prometheus metrics - `TELEMETRY_ENABLED` - Enable CloudEvents telemetry ### Configuration Priority @@ -593,7 +590,7 @@ serve the needs of any given application. #### Get Context In long-running conversations with an LLM, the context window can fill up -quickly. To address this, Honcho provides a `get_context` +quickly. To address this, Honcho provides a `context` endpoint that returns a combination of messages, conclusions, summaries from a session up to a provided token limit. @@ -607,10 +604,10 @@ There are several search endpoints that let developers query messages at the Requests can include advanced filters to further refine the results. -#### Dialectic API +#### Chat API The flagship interface for using these insights is through -the [Dialectic Endpoint](https://blog.plasticlabs.ai/archive/ARCHIVED;-Introducing-Honcho's-Dialectic-API). +the [`Chat` Endpoint](https://blog.plasticlabs.ai/archive/ARCHIVED;-Introducing-Honcho's-Dialectic-API). This is a regular API endpoint (`/peers/{peer_id}/chat`) that takes natural language requests to get data about the `Peer`. This robust design lets us use this single endpoint for all @@ -625,10 +622,10 @@ API include: - Asking Honcho for a 2nd opinion or approach about how to respond to the Peer - Getting personalized responses that incorporate long-term facts and context -#### Working Representations +#### Representations For low-latency use cases, -Honcho provides access to a `get_representation` endpoint that +Honcho provides access to a `representation` endpoint that returns a static document with insights about a `Peer` in the context of a particular session. diff --git a/config.toml.example b/config.toml.example index b54985b9..d8de12c0 100644 --- a/config.toml.example +++ b/config.toml.example @@ -187,14 +187,10 @@ INCLUDE_LEVELS = ["explicit", "deductive"] SECRET = "" MAX_WORKSPACE_LIMIT = 10 -# OpenTelemetry settings (push-based metrics via OTLP) -[otel] +# Prometheus metrics settings (pull-based metrics) +[metrics] ENABLED = false -# ENDPOINT = "https://mimir.example.com/otlp/v1/metrics" -# HEADERS = '{"X-Scope-OrgID": "honcho"}' # JSON string for auth headers -EXPORT_INTERVAL_MILLIS = 60000 -SERVICE_NAME = "honcho" -# SERVICE_NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set +# NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set # CloudEvents telemetry settings (analytics events) [telemetry] diff --git a/docker-compose.yml.example b/docker-compose.yml.example index c72840f0..39b5d8c3 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -59,17 +59,6 @@ services: interval: 5s timeout: 5s retries: 5 - mimir: - image: grafana/mimir:2.14.0 - command: - - -config.file=/etc/mimir/mimir.yaml - ports: - - 9009:9009 - volumes: - - ./mimir-data:/data - configs: - - source: mimir_config - target: /etc/mimir/mimir.yaml grafana: image: grafana/grafana:11.4.0 ports: @@ -81,66 +70,6 @@ services: - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer volumes: - ./grafana-data:/var/lib/grafana - depends_on: - - mimir - volumes: pgdata: venv: - -configs: - mimir_config: - content: | - multitenancy_enabled: false - - server: - http_listen_port: 9009 - log_level: warn - - common: - storage: - backend: filesystem - filesystem: - dir: /data - - blocks_storage: - backend: filesystem - filesystem: - dir: /data/blocks - bucket_store: - sync_dir: /data/tsdb-sync - tsdb: - dir: /data/tsdb - - compactor: - data_dir: /data/compactor - sharding_ring: - kvstore: - store: memberlist - - distributor: - ring: - instance_addr: 127.0.0.1 - kvstore: - store: memberlist - - ingester: - ring: - instance_addr: 127.0.0.1 - kvstore: - store: memberlist - replication_factor: 1 - - ruler_storage: - backend: filesystem - filesystem: - dir: /data/rules - - alertmanager_storage: - backend: filesystem - filesystem: - dir: /data/alertmanager - - store_gateway: - sharding_ring: - replication_factor: 1 diff --git a/docs/bun.lock b/docs/bun.lock index a9b4e1c5..182c58c0 100644 --- a/docs/bun.lock +++ b/docs/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "honcho-docs", diff --git a/docs/changelog/compatibility-guide.mdx b/docs/changelog/compatibility-guide.mdx index 93349ccc..d4939d7b 100644 --- a/docs/changelog/compatibility-guide.mdx +++ b/docs/changelog/compatibility-guide.mdx @@ -8,7 +8,7 @@ This guide helps you understand which versions of Honcho's API are compatible wi ## Version Compatibility -### Honcho API v3.0.0 (Current) +### Honcho API v3.0.1 (Current) @@ -34,7 +34,8 @@ This guide helps you understand which versions of Honcho's API are compatible wi | Honcho API Version | TypeScript SDK | Python SDK | |-------------------|---------------|------------| -| v3.0.0 (Current) | v2.0.0 | v2.0.0 | +| v3.0.1 (Current) | v2.0.0 | v2.0.0 | +| v3.0.0 | v2.0.0 | v2.0.0 | | v2.5.1 | v1.6.0 | v1.6.0 | | v2.5.0 | v1.6.0 | v1.6.0 | | v2.4.3 | v1.5.0 | v1.5.0 | diff --git a/docs/changelog/introduction.mdx b/docs/changelog/introduction.mdx index 99414bd7..0ac5cc01 100644 --- a/docs/changelog/introduction.mdx +++ b/docs/changelog/introduction.mdx @@ -27,10 +27,43 @@ Welcome to the Honcho changelog! This section documents all notable changes to t ### Honcho API and SDK Changelogs - + + ### Fixed + + - Token counting in Explicit Agent Loop + - Backwards compatibility of queue items + + + + ### Added + + - Agentic Dreamer for intelligent memory consolidation using LLM agents + - Agentic Dialectic for query answering using LLM agents with tool use + - Reasoning levels configuration for dialectic (`minimal`, `low`, `medium`, `high`, `max`) + - Prometheus token tracking for deriver and dialectic operations + - n8n integration + - Cloud Events for auditable telemetry + - External Vector Store support for turbopuffer and lancedb with reconciliation flow + ### Changed - - Major version release + - API route renaming for consistency + - Dreamer and dialectic now respect peer card configuration settings + - Observations renamed to Conclusions across API and SDKs + - Deriver to buffer representation tasks to normalize workloads + - Local Representation tasks to create singular QueueItems + - getContext endpoint to use `search_query` rather than force `last_user_message` + + ### Fixed + + - Dream scheduling bugs + - Summary creation when start_message_id > end_message_id + - Cashews upgrade to prevent NoScriptError + - Memory leak in `accumulate_metric` call + + ### Removed + + - Peer card configuration from message configuration; peer cards no longer created/updated in deriver process @@ -148,7 +181,7 @@ Welcome to the Honcho changelog! This section documents all notable changes to t ### Added - - Get peer cards endpoint (`GET /v3/peers/{peer_id}/peer-card`) for retrieving targeted peer context information + - Get peer cards endpoint (`GET /v2/peers/{peer_id}/card`) for retrieving targeted peer context information ### Changed diff --git a/docs/docs.json b/docs/docs.json index a0135eeb..f0ce1ef3 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -24,11 +24,226 @@ }, "navigation": { "versions": [ + { + "version": "v3.0.1", + "api": { + "openapi": [ + "v3/openapi.json" + ] + }, + "tabs": [ + { + "tab": "Documentation", + "groups": [ + { + "group": "Introduction", + "pages": [ + "v3/documentation/introduction/overview", + "v3/documentation/introduction/quickstart", + "v3/documentation/introduction/vibecoding" + ] + }, + { + "group": "Core Concepts", + "pages": [ + "v3/documentation/core-concepts/architecture", + "v3/documentation/core-concepts/reasoning", + "v3/documentation/core-concepts/representation" + ] + }, + { + "group": "Features", + "pages": [ + "v3/documentation/features/get-context", + "v3/documentation/features/chat", + { + "group": "Advanced", + "pages": [ + "v3/documentation/features/advanced/overview", + "v3/documentation/features/advanced/queue-status", + "v3/documentation/features/advanced/reasoning-configuration", + "v3/documentation/features/advanced/representation-scopes", + "v3/documentation/features/advanced/summarizer", + "v3/documentation/features/advanced/search", + "v3/documentation/features/advanced/using-filters", + "v3/documentation/features/advanced/streaming-response" + ] + } + ] + }, + { + "group": "Reference", + "pages": [ + "v3/documentation/reference/platform", + "v3/documentation/reference/sdk" + ] + } + ] + }, + { + "tab": "Guides", + "groups": [ + { + "group": "Overview", + "pages": [ + "v3/guides/overview", + "v3/guides/file-uploads", + "v3/guides/storing-data" + ] + }, + { + "group": "Integrations", + "pages": [ + "v3/guides/integrations/crewai", + "v3/guides/integrations/langgraph", + "v3/guides/integrations/mcp", + "v3/guides/integrations/n8n" + ] + }, + { + "group": "Migrations", + "pages": [ + "v3/guides/migrations/mem0" + ] + }, + { + "group": "Chatbots", + "pages": [ + "v3/guides/discord", + "v3/guides/telegram" + ] + } + ] + }, + { + "tab": "Open Source", + "groups": [ + { + "group": "Self-Hosting", + "pages": [ + "v3/contributing/self-hosting", + "v3/contributing/configuration" + ] + }, + { + "group": "Contributing", + "pages": [ + "v3/contributing/guidelines", + "v3/contributing/license" + ] + } + ] + }, + { + "tab": "API Reference", + "groups": [ + { + "group": "API Documentation", + "pages": [ + "v3/api-reference/introduction" + ] + }, + { + "group": "workspaces", + "pages": [ + "v3/api-reference/endpoint/workspaces/get-or-create-workspace", + "v3/api-reference/endpoint/workspaces/get-all-workspaces", + "v3/api-reference/endpoint/workspaces/update-workspace", + "v3/api-reference/endpoint/workspaces/delete-workspace", + "v3/api-reference/endpoint/workspaces/search-workspace", + "v3/api-reference/endpoint/workspaces/get-queue-status", + "v3/api-reference/endpoint/workspaces/schedule-dream" + ] + }, + { + "group": "peers", + "pages": [ + "v3/api-reference/endpoint/peers/get-peers", + "v3/api-reference/endpoint/peers/get-or-create-peer", + "v3/api-reference/endpoint/peers/update-peer", + "v3/api-reference/endpoint/peers/get-sessions-for-peer", + "v3/api-reference/endpoint/peers/chat", + "v3/api-reference/endpoint/peers/get-representation", + "v3/api-reference/endpoint/peers/get-peer-card", + "v3/api-reference/endpoint/peers/set-peer-card", + "v3/api-reference/endpoint/peers/get-peer-context", + "v3/api-reference/endpoint/peers/search-peer" + ] + }, + { + "group": "sessions", + "pages": [ + "v3/api-reference/endpoint/sessions/get-or-create-session", + "v3/api-reference/endpoint/sessions/get-sessions", + "v3/api-reference/endpoint/sessions/update-session", + "v3/api-reference/endpoint/sessions/delete-session", + "v3/api-reference/endpoint/sessions/clone-session", + "v3/api-reference/endpoint/sessions/get-session-peers", + "v3/api-reference/endpoint/sessions/set-session-peers", + "v3/api-reference/endpoint/sessions/add-peers-to-session", + "v3/api-reference/endpoint/sessions/remove-peers-from-session", + "v3/api-reference/endpoint/sessions/get-peer-config", + "v3/api-reference/endpoint/sessions/set-peer-config", + "v3/api-reference/endpoint/sessions/get-session-context", + "v3/api-reference/endpoint/sessions/get-session-summaries", + "v3/api-reference/endpoint/sessions/search-session" + ] + }, + { + "group": "messages", + "pages": [ + "v3/api-reference/endpoint/messages/create-messages-for-session", + "v3/api-reference/endpoint/messages/get-messages", + "v3/api-reference/endpoint/messages/get-message", + "v3/api-reference/endpoint/messages/update-message", + "v3/api-reference/endpoint/messages/create-messages-with-file" + ] + }, + { + "group": "conclusions", + "pages": [ + "v3/api-reference/endpoint/conclusions/create-conclusions", + "v3/api-reference/endpoint/conclusions/list-conclusions", + "v3/api-reference/endpoint/conclusions/query-conclusions", + "v3/api-reference/endpoint/conclusions/delete-conclusion" + ] + }, + { + "group": "webhooks", + "pages": [ + "v3/api-reference/endpoint/webhooks/list-webhook-endpoints", + "v3/api-reference/endpoint/webhooks/get-or-create-webhook-endpoint", + "v3/api-reference/endpoint/webhooks/delete-webhook-endpoint", + "v3/api-reference/endpoint/webhooks/test-emit" + ] + }, + { + "group": "miscellaneous", + "pages": [ + "v3/api-reference/endpoint/keys/create-key" + ] + } + ] + }, + { + "tab": "Changelog", + "groups": [ + { + "group": "Overview", + "pages": [ + "changelog/introduction", + "changelog/compatibility-guide" + ] + } + ] + } + ] + }, { "version": "v2.5.1", "api": { "openapi": [ - "openapi.json" + "v2/openapi.json" ] }, "tabs": [ @@ -224,221 +439,6 @@ } ] }, - { - "version": "v3.0.0", - "api": { - "openapi": [ - "openapi.json" - ] - }, - "tabs": [ - { - "tab": "Documentation", - "groups": [ - { - "group": "Introduction", - "pages": [ - "v3/documentation/introduction/overview", - "v3/documentation/introduction/quickstart", - "v3/documentation/introduction/vibecoding" - ] - }, - { - "group": "Core Concepts", - "pages": [ - "v3/documentation/core-concepts/architecture", - "v3/documentation/core-concepts/reasoning", - "v3/documentation/core-concepts/representation" - ] - }, - { - "group": "Features", - "pages": [ - "v3/documentation/features/get-context", - "v3/documentation/features/chat", - { - "group": "Advanced", - "pages": [ - "v3/documentation/features/advanced/overview", - "v3/documentation/features/advanced/queue-status", - "v3/documentation/features/advanced/reasoning-configuration", - "v3/documentation/features/advanced/representation-scopes", - "v3/documentation/features/advanced/summarizer", - "v3/documentation/features/advanced/search", - "v3/documentation/features/advanced/using-filters", - "v3/documentation/features/advanced/streaming-response" - ] - } - ] - }, - { - "group": "Reference", - "pages": [ - "v3/documentation/reference/platform", - "v3/documentation/reference/sdk" - ] - } - ] - }, - { - "tab": "Guides", - "groups": [ - { - "group": "Overview", - "pages": [ - "v3/guides/overview", - "v3/guides/file-uploads", - "v3/guides/storing-data" - ] - }, - { - "group": "Integrations", - "pages": [ - "v3/guides/integrations/crewai", - "v3/guides/integrations/langgraph", - "v3/guides/integrations/mcp" - ] - }, - { - "group": "Migrations", - "pages": [ - "v3/guides/migrations/mem0" - ] - }, - { - "group": "Chatbots", - "pages": [ - "v3/guides/discord", - "v3/guides/telegram" - ] - } - ] - }, - { - "tab": "Open Source", - "groups": [ - { - "group": "Self-Hosting", - "pages": [ - "v3/contributing/self-hosting", - "v3/contributing/configuration" - ] - }, - { - "group": "Contributing", - "pages": [ - "v3/contributing/guidelines", - "v3/contributing/license" - ] - } - ] - }, - { - "tab": "API Reference", - "groups": [ - { - "group": "API Documentation", - "pages": [ - "v3/api-reference/introduction" - ] - }, - { - "group": "workspaces", - "pages": [ - "v3/api-reference/endpoint/workspaces/get-or-create-workspace", - "v3/api-reference/endpoint/workspaces/get-all-workspaces", - "v3/api-reference/endpoint/workspaces/update-workspace", - "v3/api-reference/endpoint/workspaces/delete-workspace", - "v3/api-reference/endpoint/workspaces/search-workspace", - "v3/api-reference/endpoint/workspaces/get-deriver-status", - "v3/api-reference/endpoint/workspaces/trigger-dream" - ] - }, - { - "group": "peers", - "pages": [ - "v3/api-reference/endpoint/peers/get-peers", - "v3/api-reference/endpoint/peers/get-or-create-peer", - "v3/api-reference/endpoint/peers/update-peer", - "v3/api-reference/endpoint/peers/get-sessions-for-peer", - "v3/api-reference/endpoint/peers/chat", - "v3/api-reference/endpoint/peers/get-working-representation", - "v3/api-reference/endpoint/peers/get-peer-card", - "v3/api-reference/endpoint/peers/set-peer-card", - "v3/api-reference/endpoint/peers/get-peer-context", - "v3/api-reference/endpoint/peers/search-peer" - ] - }, - { - "group": "sessions", - "pages": [ - "v3/api-reference/endpoint/sessions/get-or-create-session", - "v3/api-reference/endpoint/sessions/get-sessions", - "v3/api-reference/endpoint/sessions/update-session", - "v3/api-reference/endpoint/sessions/delete-session", - "v3/api-reference/endpoint/sessions/clone-session", - "v3/api-reference/endpoint/sessions/get-session-peers", - "v3/api-reference/endpoint/sessions/set-session-peers", - "v3/api-reference/endpoint/sessions/add-peers-to-session", - "v3/api-reference/endpoint/sessions/remove-peers-from-session", - "v3/api-reference/endpoint/sessions/get-peer-config", - "v3/api-reference/endpoint/sessions/set-peer-config", - "v3/api-reference/endpoint/sessions/get-session-context", - "v3/api-reference/endpoint/sessions/get-session-summaries", - "v3/api-reference/endpoint/sessions/search-session" - ] - }, - { - "group": "messages", - "pages": [ - "v3/api-reference/endpoint/messages/create-messages-for-session", - "v3/api-reference/endpoint/messages/get-messages", - "v3/api-reference/endpoint/messages/get-message", - "v3/api-reference/endpoint/messages/update-message", - "v3/api-reference/endpoint/messages/create-messages-with-file" - ] - }, - { - "group": "observations", - "pages": [ - "v3/api-reference/endpoint/observations/create-observations", - "v3/api-reference/endpoint/observations/list-observations", - "v3/api-reference/endpoint/observations/query-observations", - "v3/api-reference/endpoint/observations/delete-observation" - ] - }, - { - "group": "webhooks", - "pages": [ - "v3/api-reference/endpoint/webhooks/list-webhook-endpoints", - "v3/api-reference/endpoint/webhooks/get-or-create-webhook-endpoint", - "v3/api-reference/endpoint/webhooks/delete-webhook-endpoint", - "v3/api-reference/endpoint/webhooks/test-emit" - ] - }, - { - "group": "miscellaneous", - "pages": [ - "v3/api-reference/endpoint/keys/create-key", - "v3/api-reference/endpoint/metrics" - ] - } - ] - }, - { - "tab": "Changelog", - "groups": [ - { - "group": "Overview", - "pages": [ - "changelog/introduction", - "changelog/compatibility-guide" - ] - } - ] - } - ] - }, { "version": "v1.1.0", "api": { diff --git a/docs/v2/documentation/core-concepts/features/get-context.mdx b/docs/v2/documentation/core-concepts/features/get-context.mdx index 566f5f1a..20f8d10f 100644 --- a/docs/v2/documentation/core-concepts/features/get-context.mdx +++ b/docs/v2/documentation/core-concepts/features/get-context.mdx @@ -139,17 +139,17 @@ context = session.get_context( ``` -### Semantic Search with Last Message +### Semantic Search -Use `last_user_message` to fetch semantically relevant observations based on the most recent message: +Use `search_query` to fetch semantically relevant observations based on a query string: ```python Python -# Get context with semantic search based on last message +# Get context with semantic search based on query context = session.get_context( tokens=2000, peer_target="user-123", - last_user_message="What are my account preferences?", + search_query="What are my account preferences?", search_top_k=10, # Number of relevant observations search_max_distance=0.8, # Max semantic distance (0.0-1.0) include_most_derived=True, # Include most recent observations @@ -159,11 +159,11 @@ context = session.get_context( ```typescript TypeScript (async () => { - // Get context with semantic search based on last message + // Get context with semantic search based on query const context = await session.getContext({ tokens: 2000, peerTarget: "user-123", - lastUserMessage: "What are my account preferences?", + searchQuery: "What are my account preferences?", searchTopK: 10, // Number of relevant observations searchMaxDistance: 0.8, // Max semantic distance (0.0-1.0) includeMostDerived: true, // Include most recent observations @@ -207,7 +207,7 @@ context = session.get_context( | `tokens` | `int` | Maximum tokens to include | | `peer_target` | `str` | Peer ID to include representation for | | `peer_perspective` | `str` | Peer ID for perspective (requires peer_target) | -| `last_user_message` | `str` | Message for semantic search (requires peer_target) | +| `search_query` | `str` | Query for semantic search (requires peer_target) | | `limit_to_session` | `bool` | Limit to session observations only | | `search_top_k` | `int` | Semantic search results to include (1-100) | | `search_max_distance` | `float` | Max semantic distance (0.0-1.0) | diff --git a/docs/v2/documentation/reference/sdk.mdx b/docs/v2/documentation/reference/sdk.mdx index 87c574e4..fbed91fd 100644 --- a/docs/v2/documentation/reference/sdk.mdx +++ b/docs/v2/documentation/reference/sdk.mdx @@ -533,7 +533,7 @@ context = session.get_context( tokens=2000, peer_target="user", peer_perspective="assistant", - last_user_message="What are my preferences?", + search_query="What are my preferences?", limit_to_session=True, search_top_k=10, search_max_distance=0.8, @@ -612,7 +612,7 @@ const richContext = await session.getContext({ tokens: 2000, peerTarget: "user", peerPerspective: "assistant", - lastUserMessage: "What are my preferences?", + searchQuery: "What are my preferences?", limitToSession: true, searchTopK: 10, searchMaxDistance: 0.8, @@ -752,7 +752,7 @@ The SessionContext object has the following structure: | `tokens` | `int` | Maximum tokens to include | | `peer_target` | `str` | Peer ID to get representation for | | `peer_perspective` | `str` | Peer ID for perspective (requires peer_target) | -| `last_user_message` | `str` | Most recent message for semantic search | +| `search_query` | `str` | Query string for semantic search | | `limit_to_session` | `bool` | Limit representation to session only | | `search_top_k` | `int` | Number of semantic search results (1-100) | | `search_max_distance` | `float` | Max semantic distance (0.0-1.0) | diff --git a/docs/v3/api-reference/endpoint/conclusions/create-conclusions.mdx b/docs/v3/api-reference/endpoint/conclusions/create-conclusions.mdx new file mode 100644 index 00000000..b35c98b8 --- /dev/null +++ b/docs/v3/api-reference/endpoint/conclusions/create-conclusions.mdx @@ -0,0 +1,3 @@ +--- +openapi: post /v3/workspaces/{workspace_id}/conclusions +--- diff --git a/docs/v3/api-reference/endpoint/conclusions/delete-conclusion.mdx b/docs/v3/api-reference/endpoint/conclusions/delete-conclusion.mdx new file mode 100644 index 00000000..e55086f7 --- /dev/null +++ b/docs/v3/api-reference/endpoint/conclusions/delete-conclusion.mdx @@ -0,0 +1,3 @@ +--- +openapi: delete /v3/workspaces/{workspace_id}/conclusions/{conclusion_id} +--- diff --git a/docs/v3/api-reference/endpoint/conclusions/list-conclusions.mdx b/docs/v3/api-reference/endpoint/conclusions/list-conclusions.mdx new file mode 100644 index 00000000..bf47ffb5 --- /dev/null +++ b/docs/v3/api-reference/endpoint/conclusions/list-conclusions.mdx @@ -0,0 +1,3 @@ +--- +openapi: post /v3/workspaces/{workspace_id}/conclusions/list +--- diff --git a/docs/v3/api-reference/endpoint/conclusions/query-conclusions.mdx b/docs/v3/api-reference/endpoint/conclusions/query-conclusions.mdx new file mode 100644 index 00000000..810b2bf1 --- /dev/null +++ b/docs/v3/api-reference/endpoint/conclusions/query-conclusions.mdx @@ -0,0 +1,3 @@ +--- +openapi: post /v3/workspaces/{workspace_id}/conclusions/query +--- diff --git a/docs/v3/api-reference/endpoint/messages/create-messages-for-session.mdx b/docs/v3/api-reference/endpoint/messages/create-messages-for-session.mdx index 9a579dc4..a7c0bd90 100644 --- a/docs/v3/api-reference/endpoint/messages/create-messages-for-session.mdx +++ b/docs/v3/api-reference/endpoint/messages/create-messages-for-session.mdx @@ -1,3 +1,3 @@ --- -openapi: post /v3/workspaces/{workspace_id}/sessions/{session_id}/messages/ +openapi: post /v3/workspaces/{workspace_id}/sessions/{session_id}/messages --- diff --git a/docs/v3/api-reference/endpoint/metrics.mdx b/docs/v3/api-reference/endpoint/metrics.mdx deleted file mode 100644 index 00ca0fca..00000000 --- a/docs/v3/api-reference/endpoint/metrics.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -openapi: get /metrics ---- diff --git a/docs/v3/api-reference/endpoint/observations/create-observations.mdx b/docs/v3/api-reference/endpoint/observations/create-observations.mdx deleted file mode 100644 index 408eddbd..00000000 --- a/docs/v3/api-reference/endpoint/observations/create-observations.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -openapi: post /v3/workspaces/{workspace_id}/observations ---- diff --git a/docs/v3/api-reference/endpoint/observations/delete-observation.mdx b/docs/v3/api-reference/endpoint/observations/delete-observation.mdx deleted file mode 100644 index 653f73cb..00000000 --- a/docs/v3/api-reference/endpoint/observations/delete-observation.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -openapi: delete /v3/workspaces/{workspace_id}/observations/{observation_id} ---- diff --git a/docs/v3/api-reference/endpoint/observations/list-observations.mdx b/docs/v3/api-reference/endpoint/observations/list-observations.mdx deleted file mode 100644 index 6d1ffd0e..00000000 --- a/docs/v3/api-reference/endpoint/observations/list-observations.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -openapi: post /v3/workspaces/{workspace_id}/observations/list ---- diff --git a/docs/v3/api-reference/endpoint/observations/query-observations.mdx b/docs/v3/api-reference/endpoint/observations/query-observations.mdx deleted file mode 100644 index a9625753..00000000 --- a/docs/v3/api-reference/endpoint/observations/query-observations.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -openapi: post /v3/workspaces/{workspace_id}/observations/query ---- diff --git a/docs/v3/api-reference/endpoint/peers/get-working-representation.mdx b/docs/v3/api-reference/endpoint/peers/get-representation.mdx similarity index 100% rename from docs/v3/api-reference/endpoint/peers/get-working-representation.mdx rename to docs/v3/api-reference/endpoint/peers/get-representation.mdx diff --git a/docs/v3/api-reference/endpoint/sessions/clone-session.mdx b/docs/v3/api-reference/endpoint/sessions/clone-session.mdx index fc20d81c..2985ff10 100644 --- a/docs/v3/api-reference/endpoint/sessions/clone-session.mdx +++ b/docs/v3/api-reference/endpoint/sessions/clone-session.mdx @@ -1,3 +1,3 @@ --- -openapi: get /v3/workspaces/{workspace_id}/sessions/{session_id}/clone +openapi: post /v3/workspaces/{workspace_id}/sessions/{session_id}/clone --- diff --git a/docs/v3/api-reference/endpoint/sessions/set-peer-config.mdx b/docs/v3/api-reference/endpoint/sessions/set-peer-config.mdx index 7591bf85..ab888364 100644 --- a/docs/v3/api-reference/endpoint/sessions/set-peer-config.mdx +++ b/docs/v3/api-reference/endpoint/sessions/set-peer-config.mdx @@ -1,3 +1,3 @@ --- -openapi: post /v3/workspaces/{workspace_id}/sessions/{session_id}/peers/{peer_id}/config +openapi: put /v3/workspaces/{workspace_id}/sessions/{session_id}/peers/{peer_id}/config --- diff --git a/docs/v3/api-reference/endpoint/workspaces/get-deriver-status.mdx b/docs/v3/api-reference/endpoint/workspaces/get-deriver-status.mdx deleted file mode 100644 index b8364804..00000000 --- a/docs/v3/api-reference/endpoint/workspaces/get-deriver-status.mdx +++ /dev/null @@ -1,3 +0,0 @@ ---- -openapi: get /v3/workspaces/{workspace_id}/deriver/status ---- diff --git a/docs/v3/api-reference/endpoint/workspaces/get-queue-status.mdx b/docs/v3/api-reference/endpoint/workspaces/get-queue-status.mdx new file mode 100644 index 00000000..0c468af0 --- /dev/null +++ b/docs/v3/api-reference/endpoint/workspaces/get-queue-status.mdx @@ -0,0 +1,3 @@ +--- +openapi: get /v3/workspaces/{workspace_id}/queue/status +--- diff --git a/docs/v3/api-reference/endpoint/workspaces/trigger-dream.mdx b/docs/v3/api-reference/endpoint/workspaces/schedule-dream.mdx similarity index 100% rename from docs/v3/api-reference/endpoint/workspaces/trigger-dream.mdx rename to docs/v3/api-reference/endpoint/workspaces/schedule-dream.mdx diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index bac2a73e..57b77e40 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -57,7 +57,7 @@ Then modify the values as needed. The TOML file is organized into sections: - `[summary]` - Session summarization settings (frequency thresholds, provider, model, token limits for short and long summaries) - `[dream]` - Dream processing configuration (enable/disable, thresholds, idle timeouts, dream types, LLM settings, surprisal sampling) - `[webhook]` - Webhook configuration (webhook secret, workspace limits) -- `[otel]` - OpenTelemetry settings for push-based metrics via OTLP +- `[metrics]` - Prometheus pull-based metrics settings - `[telemetry]` - CloudEvents telemetry settings for analytics - `[vector_store]` - Vector store configuration (pgvector, Turbopuffer, LanceDB) - `[sentry]` - Error tracking and monitoring settings (enable/disable, DSN, environment, sample rates) @@ -540,29 +540,19 @@ VECTOR_STORE_LANCEDB_PATH=./lancedb_data ## Monitoring Configuration -### OpenTelemetry (Push-based Metrics) +### Prometheus Metrics (Pull-based) -Honcho supports push-based metrics via OpenTelemetry Protocol (OTLP) to any compatible backend (Mimir, Grafana Cloud, etc.). +Honcho exposes Prometheus metrics via `/metrics` endpoints for scraping: +- **API process**: Port 8000 at `/metrics` +- **Deriver process**: Port 9090 at `/metrics` -**OpenTelemetry Settings:** +**Metrics Settings:** ```bash -# Enable/disable OTel metrics -OTEL_ENABLED=false +# Enable/disable Prometheus metrics +METRICS_ENABLED=false -# OTLP HTTP endpoint for metrics -# For Mimir: /otlp/v1/metrics -# For Grafana Cloud: https://otlp-gateway-.grafana.net/otlp/v1/metrics -OTEL_ENDPOINT=https://mimir.example.com/otlp/v1/metrics - -# Optional auth headers (JSON format in env var) -OTEL_HEADERS='{"X-Scope-OrgID": "honcho"}' - -# Export interval in milliseconds (default: 60 seconds) -OTEL_EXPORT_INTERVAL_MILLIS=60000 - -# Service identification -OTEL_SERVICE_NAME=honcho -OTEL_SERVICE_NAMESPACE=honcho # Inherits from app.NAMESPACE if not set +# Namespace label for all metrics (inherits from app.NAMESPACE if not set) +METRICS_NAMESPACE=honcho ``` ### CloudEvents Telemetry (Analytics) @@ -691,7 +681,7 @@ MODEL = "claude-sonnet-4-20250514" [webhook] MAX_WORKSPACE_LIMIT = 10 -[otel] +[metrics] ENABLED = false [telemetry] @@ -798,7 +788,7 @@ MODEL = "claude-sonnet-4-20250514" [webhook] MAX_WORKSPACE_LIMIT = 10 -[otel] +[metrics] ENABLED = true [telemetry] @@ -838,7 +828,7 @@ LLM_GROQ_API_KEY=your-prod-groq-key WEBHOOK_SECRET=your-webhook-signing-secret # Monitoring -OTEL_ENDPOINT=https://mimir.example.com/otlp/v1/metrics +METRICS_ENABLED=true TELEMETRY_ENDPOINT=https://telemetry.honcho.dev/v1/events SENTRY_DSN=https://your-sentry-dsn@sentry.io/project-id SENTRY_ENVIRONMENT=production diff --git a/docs/v3/documentation/core-concepts/representation.mdx b/docs/v3/documentation/core-concepts/representation.mdx index 08425029..0c440239 100644 --- a/docs/v3/documentation/core-concepts/representation.mdx +++ b/docs/v3/documentation/core-concepts/representation.mdx @@ -61,6 +61,6 @@ Humans reconstruct the past from imperfect recollections, then act on those reco Understand how representations fit into Honcho's architecture - Learn how to query representations with natural language + Chat with Honcho about your users diff --git a/docs/v3/documentation/features/advanced/overview.mdx b/docs/v3/documentation/features/advanced/overview.mdx index 00a92cc2..77f17a7b 100644 --- a/docs/v3/documentation/features/advanced/overview.mdx +++ b/docs/v3/documentation/features/advanced/overview.mdx @@ -10,7 +10,7 @@ Advanced features give you fine-grained control over Honcho's behavior and imple ## Configuration & Monitoring - [Queue Status](/v3/documentation/features/advanced/queue-status) - Monitor background processing and reasoning tasks -- [Configuration](/v3/documentation/features/advanced/toggle-reasoning) - Configure reasoning models and behavior +- [Configuration](/v3/documentation/features/advanced/reasoning-configuration) - Configure reasoning models and behavior - [Summarizer](/v3/documentation/features/advanced/summarizer) - Automatic session summarization ## Querying & Filtering diff --git a/docs/v3/documentation/features/get-context.mdx b/docs/v3/documentation/features/get-context.mdx index 2e27a1e3..31c7e50f 100644 --- a/docs/v3/documentation/features/get-context.mdx +++ b/docs/v3/documentation/features/get-context.mdx @@ -143,16 +143,16 @@ context = session.get_context( ``` -### Semantic Search with Last Message +### Semantic Search -Use `last_user_message` to fetch semantically relevant conclusions based on the most recent message (requires `peer_target`): +Use `search_query` to fetch semantically relevant conclusions based on a query string (requires `peer_target`): ```python Python context = session.get_context( tokens=2000, peer_target="user-123", - last_user_message="What are my coding preferences?", + search_query="What are my coding preferences?", search_top_k=10, # Number of relevant conclusions to fetch search_max_distance=0.8, # Max semantic distance (0.0-1.0) include_most_frequent=True, # Include most frequent conclusions @@ -165,7 +165,7 @@ context = session.get_context( const context = await session.getContext({ tokens: 2000, peerTarget: "user-123", - lastUserMessage: "What are my coding preferences?", + searchQuery: "What are my coding preferences?", representationOptions: { searchTopK: 10, // Number of relevant conclusions to fetch searchMaxDistance: 0.8, // Max semantic distance (0.0-1.0) @@ -211,7 +211,7 @@ context = session.get_context( | `tokens` | `int` | Maximum tokens to include | | `peer_target` | `str` | Peer ID to include representation for | | `peer_perspective` | `str` | Peer ID for perspective (requires peer_target) | -| `last_user_message` | `str` | Message for semantic search (requires peer_target) | +| `search_query` | `str` | Query for semantic search (requires peer_target) | | `limit_to_session` | `bool` | Limit to session conclusions only | | `search_top_k` | `int` | Semantic search results to include (1-100) | | `search_max_distance` | `float` | Max semantic distance (0.0-1.0) | diff --git a/docs/v3/documentation/introduction/overview.mdx b/docs/v3/documentation/introduction/overview.mdx index f3b631b7..10ce0775 100644 --- a/docs/v3/documentation/introduction/overview.mdx +++ b/docs/v3/documentation/introduction/overview.mdx @@ -7,7 +7,7 @@ sidebarTitle: "Overview" Honcho is an open source memory library with a managed service for building stateful agents. Use it with any model, framework, or architecture. It enables agents to build and maintain state about any entity--users, agents, groups, ideas, and more. And because it's a continual learning system, it understands entities that change over time. Using Honcho as your memory system will earn your agents higher retention, more trust, and help you build data moats to out-compete incumbents. -Honcho has defined the Pareto Frontier of Agent Memory. Watch the [video](https://x.com/honchodotdev/status/2002090546521911703?s=20), check out our [evals page](https://evals.honcho.dev/), and read the [blog post](https://blog.plasticlabs.ai/research/Benchmarking-Honcho) for more detail. + Honcho has defined the Pareto Frontier of Agent Memory. Watch the [video](https://x.com/honchodotdev/status/2002090546521911703?s=20), check out our [evals page](https://evals.honcho.dev/), and read the [blog post](https://blog.plasticlabs.ai/research/Benchmarking-Honcho) for more detail. @@ -42,7 +42,7 @@ Break free from this cycle. Honcho is a general solution to context engineering, ## How Honcho Works -Honcho is a memory system that reasons. Read more on the approach [here](https://blog.plasticlabs.ai/blog/Memory-as-Reasoning). + Honcho is a memory system that reasons. Read more on the approach [here](https://blog.plasticlabs.ai/blog/Memory-as-Reasoning). Honcho has four storage primitives that work together: @@ -65,9 +65,9 @@ Honcho has four storage primitives that work together: - **Workspaces** - Top-level containers that isolate different applications or environments - **Peers** - Any entity that persists but changes over time (users, agents, objects, and more) - **Sessions** - Interaction threads between peers with temporal boundaries -- **Messages** - Units of data that trigger reasoning (conversations, events, activity, documents, and more) +- **Messages** - Units of data that trigger reasoning (conversations, events, activity, documents, and more) -When you write messages to Honcho, they're stored and processed in the background. Custom reasoning models perform formal logical [*reasoning*](/v3/documentation/core-concepts/reasoning) to generate conclusions about each peer. These conclusions are stored as [*representations*](/v3/documentation/core-concepts/representation) that you can query to provide rich context for your agents. +When you write messages to Honcho, they're stored and processed in the background. Custom reasoning models perform formal logical [_reasoning_](/v3/documentation/core-concepts/reasoning) to generate conclusions about each peer. These conclusions are stored as [_representations_](/v3/documentation/core-concepts/representation) that you can query to provide rich context for your agents. ![Honcho Architecture](/images/architecture.png) @@ -75,7 +75,7 @@ The diagram above shows the flow: agents write messages to Honcho, which trigger ## Why Reasoning? -Traditional RAG systems retrieve what was explicitly said, but they miss what matters most—the insights only accessible by *rigorously thinking* about your data. Without reasoning, you're leaving latent information on the table. Static retrieval can't surface implicit connections, struggles when new information contradicts old data, and fails when you need to make predictions under uncertainty. +Traditional RAG systems retrieve what was explicitly said, but they miss what matters most—the insights only accessible by _rigorously thinking_ about your data. Without reasoning, you're leaving latent information on the table. Static retrieval can't surface implicit connections, struggles when new information contradicts old data, and fails when you need to make predictions under uncertainty. Honcho uses formal logic to extract all that latent information. This reasoning is AI-native—it performs the rigorous, compute-intensive thinking that humans struggle with, instantly and consistently. The result is memory that goes beyond simple RAG recall to provide exhaustive context for statefulness. @@ -100,4 +100,4 @@ Welcome to Honcho. We're excited to have you at the frontier of AI with us 🫡. Learn how Honcho reasons about data to build memory - + \ No newline at end of file diff --git a/docs/v3/documentation/introduction/quickstart.mdx b/docs/v3/documentation/introduction/quickstart.mdx index 96eb6acd..f0c5cdc6 100644 --- a/docs/v3/documentation/introduction/quickstart.mdx +++ b/docs/v3/documentation/introduction/quickstart.mdx @@ -369,7 +369,7 @@ From here, you can explore how to use Honcho's features in your own applications Deep dive into how Honcho's primitives fit together - Query representations with natural language + Chat with Honcho about your users Integration patterns and advanced use cases diff --git a/docs/v3/documentation/introduction/vibecoding.mdx b/docs/v3/documentation/introduction/vibecoding.mdx index 40a2d0c2..43679935 100644 --- a/docs/v3/documentation/introduction/vibecoding.mdx +++ b/docs/v3/documentation/introduction/vibecoding.mdx @@ -1,7 +1,7 @@ --- title: "AI-Powered Honcho Setup" icon: "wand-magic-sparkles" -description: "Universal starter prompt and Claude Code skill for building with Honcho" +description: "Agent skills and starter prompt for building with Honcho" sidebarTitle: 'Vibecoding Setup' --- @@ -14,38 +14,44 @@ We follow the llms.txt standard. There are both an llms.txt and llms-full.txt av --- -## Claude Code Skill +## Agent Skills -If you're using [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview), you can install the Honcho integration skill for a guided, interactive setup experience. The skill will explore your codebase, ask targeted questions about your integration needs, and implement Honcho step by step. - -### Installation +We provide agent skills for coding assistants like Claude Code, Cursor, Windsurf, and others. -```bash Global Installation (all projects) -# Add to your global skills directory -curl -o ~/.claude/skills/honcho-integration.md https://raw.githubusercontent.com/plastic-labs/honcho/main/docs/SKILL.md +```bash Install via npx (Recommended) +npx skills add plastic-labs/honcho ``` -```bash Project-specific Installation -# Add to your project's .claude directory -mkdir -p .claude/skills -curl -o .claude/skills/honcho-integration.md https://raw.githubusercontent.com/plastic-labs/honcho/main/docs/SKILL.md +```bash Install as Claude Skill Manually +curl -o ~/.claude/skills/honcho-integration.md https://raw.githubusercontent.com/plastic-labs/honcho/main/docs/SKILL.md ``` -### Usage +### Available Skills -Once installed, invoke the skill in Claude Code: +#### honcho-integration -``` -/honcho-integration -``` +**For new integrations.** This skill helps you add Honcho to an existing Python or TypeScript codebase. It provides a guided, interactive experience: -The skill will: -1. **Explore your codebase** to understand your language, framework, and existing AI/LLM integrations -2. **Interview you** about which entities should be peers, your preferred integration pattern, and session structure -3. **Implement the integration** based on your answers -4. **Verify the setup** to ensure everything is configured correctly +1. **Explores your codebase** to understand your language, framework, and existing AI/LLM integrations +2. **Interviews you** about which entities should be peers, your preferred integration pattern, and session structure +3. **Implements the integration** based on your answers—installing the SDK, creating peers, configuring sessions, and wiring up the chat endpoint +4. **Verifies the setup** to ensure everything is configured correctly + +Invoke with `/honcho-integration` in your coding agent. + +#### migrate-honcho-py / migrate-honcho-ts + +**For SDK upgrades.** Migrates code from v1.6.0 to v2.0.0 (required for Honcho 3.0.0). Use when upgrading the SDK or seeing errors about removed APIs like `observations`, `Representation`, `.core`, or `get_config`. + +Both skills handle: terminology changes (`Observation` → `Conclusion`), `Representation` class removal, method renames, and streaming API updates. + +| Python | TypeScript | +|--------|------------| +| `/migrate-honcho-py` | `/migrate-honcho-ts` | +| `AsyncHoncho` → `.aio` accessor | `@honcho-ai/core` removal | +| | `snake_case` → `camelCase` | --- diff --git a/docs/v3/documentation/reference/sdk.mdx b/docs/v3/documentation/reference/sdk.mdx index a520fc0a..9e0ce05d 100644 --- a/docs/v3/documentation/reference/sdk.mdx +++ b/docs/v3/documentation/reference/sdk.mdx @@ -58,7 +58,7 @@ session.add_messages([ assistant.message("It's sunny and 75°F outside!") ]) -# Query peer representations in natural language +# Chat with Honcho about a peer response = alice.chat("What did the assistant tell this user about the weather?") # Get conversation context for LLM completions @@ -85,7 +85,7 @@ await session.addMessages([ assistant.message("It's sunny and 75°F outside!") ]); -// Query peer representations in natural language +// Chat with Honcho about a peer const response = await alice.chat("What did the assistant tell this user about the weather?"); // Get conversation context for LLM completions @@ -533,7 +533,7 @@ context = session.get_context( tokens=2000, peer_target="user", peer_perspective="assistant", - last_user_message="What are my preferences?", + search_query="What are my preferences?", limit_to_session=True, search_top_k=10, search_max_distance=0.8, @@ -612,7 +612,7 @@ const richContext = await session.getContext({ tokens: 2000, peerTarget: "user", peerPerspective: "assistant", - lastUserMessage: "What are my preferences?", + searchQuery: "What are my preferences?", limitToSession: true, searchTopK: 10, searchMaxDistance: 0.8, @@ -752,7 +752,7 @@ The SessionContext object has the following structure: | `tokens` | `int` | Maximum tokens to include | | `peer_target` | `str` | Peer ID to get representation for | | `peer_perspective` | `str` | Peer ID for perspective (requires peer_target) | -| `last_user_message` | `str` | Most recent message for semantic search | +| `search_query` | `str` or `Message` | Query string or Message object for semantic search | | `limit_to_session` | `bool` | Limit representation to session only | | `search_top_k` | `int` | Number of semantic search results (1-100) | | `search_max_distance` | `float` | Max semantic distance (0.0-1.0) | @@ -879,7 +879,7 @@ If `created_at` is not provided, messages will use the server's current timestam ### Metadata and Filtering -See [Using Filters](/v3/guides/using-filters) for more examples on how to use filters. +See [Using Filters](/v3/documentation/features/advanced/using-filters) for more examples on how to use filters. ```python Python diff --git a/docs/v3/documentation/scratch/honcho-memory/quickstart.mdx b/docs/v3/documentation/scratch/honcho-memory/quickstart.mdx index a8953fdc..df7fabae 100644 --- a/docs/v3/documentation/scratch/honcho-memory/quickstart.mdx +++ b/docs/v3/documentation/scratch/honcho-memory/quickstart.mdx @@ -237,7 +237,7 @@ and Bob. We: As soon as you save a message in Honcho, it will start to reason about it to pull out insights and develop a profile of the user. This is the default -behavior and can be toggled off via [the configuration](/v3/documentation/core-concepts/configuration). +behavior and can be toggled off via [the configuration](/v3/documentation/features/advanced/reasoning-configuration). ## Next Steps diff --git a/docs/v3/documentation/scratch/local-vs-global.mdx b/docs/v3/documentation/scratch/local-vs-global.mdx index 29674d57..e46f2b80 100644 --- a/docs/v3/documentation/scratch/local-vs-global.mdx +++ b/docs/v3/documentation/scratch/local-vs-global.mdx @@ -53,7 +53,7 @@ This feature is illustrated in the graphic below: We can enable local representation for a `Peer` by setting `observe_others=True`. This is shown in the [Configure -Reasoning](/v3/documentation/core-concepts/configuration) page. +Reasoning](/v3/documentation/features/advanced/reasoning-configuration) page. Now if we used Bob's local representation of Alice then Bob would only get insights on what they've seen Alice say to them. diff --git a/docs/v3/guides/discord.mdx b/docs/v3/guides/discord.mdx index 5b24dca8..9d60d3d2 100644 --- a/docs/v3/guides/discord.mdx +++ b/docs/v3/guides/discord.mdx @@ -198,7 +198,7 @@ Discord bots also offer slash command functionality. Here's an example using Hon ```python @bot.slash_command( name="chat", - description="Query the peer's representation in natural language.", + description="Chat with Honcho about a peer.", ) async def chat(ctx, query: str): await ctx.defer() diff --git a/docs/v3/guides/integrations/crewai.mdx b/docs/v3/guides/integrations/crewai.mdx index 57fda576..4c9185c1 100644 --- a/docs/v3/guides/integrations/crewai.mdx +++ b/docs/v3/guides/integrations/crewai.mdx @@ -97,7 +97,7 @@ results = storage.search("query", filters={ }) ``` -For the full filter syntax including logical operators (AND, OR, NOT), comparison operators, and metadata filtering, see the [Using Filters](https://docs.honcho.dev/v3/documentation/core-concepts/features/using-filters) documentation. +For the full filter syntax including logical operators (AND, OR, NOT), comparison operators, and metadata filtering, see the [Using Filters](https://docs.honcho.dev/v3/documentation/features/advanced/using-filters) documentation. For comprehensive details about CrewAI's memory system, see the [official CrewAI Memory documentation](https://docs.crewai.com/en/concepts/memory). @@ -282,13 +282,13 @@ Now that you have a working CrewAI integration with Honcho, you can: Understand Honcho's peer-based model and core primitives - + Learn about retrieving and formatting conversation context - + Query `peer` representations for deeper understanding - + Build stateful agents with LangGraph and Honcho diff --git a/docs/v3/guides/integrations/langgraph.mdx b/docs/v3/guides/integrations/langgraph.mdx index 5427e24f..59781c98 100644 --- a/docs/v3/guides/integrations/langgraph.mdx +++ b/docs/v3/guides/integrations/langgraph.mdx @@ -227,7 +227,7 @@ const graph = new StateGraph(StateAnnotation) ### Understanding get_context() -The [`get_context()`](/v3/documentation/core-concepts/features/get-context) method retrieves comprehensive conversation context and formats it for your LLM. It automatically: +The [`get_context()`](/v3/documentation/features/get-context) method retrieves comprehensive conversation context and formats it for your LLM. It automatically: - **Manages conversation history** - Tracks all messages and determines what's relevant - **Respects token limits** - Stays within context window constraints without manual counting @@ -248,7 +248,7 @@ That's it. Call `session.get_context().to_openai(assistant)` and you get properl -For more details on all available parameters, see [`get_context() documentation`](/v3/documentation/core-concepts/features/get-context) +For more details on all available parameters, see [`get_context() documentation`](/v3/documentation/features/get-context) ## Chat Loop @@ -354,10 +354,10 @@ Now that you have a working LangGraph integration with Honcho, you can: ## Related Resources - + Learn more about retrieving and formatting conversation context - + Use Honcho in Claude Desktop with MCP diff --git a/docs/v3/guides/integrations/n8n.mdx b/docs/v3/guides/integrations/n8n.mdx new file mode 100644 index 00000000..3a8ccb42 --- /dev/null +++ b/docs/v3/guides/integrations/n8n.mdx @@ -0,0 +1,761 @@ +--- +title: "n8n" +icon: 'share-nodes' +description: "Connect Honcho to your n8n workflows to build intelligent automation workflows and agents that leverage persistent memory across sessions." +sidebarTitle: 'n8n' +--- + +## Quick Start + +### Prerequisites + +- n8n instance (self-hosted or cloud) +- Honcho API key ([get one here](https://app.honcho.dev)) +- Basic understanding of n8n workflows +- Basic understanding of [Honcho architecture](/v3/documentation/core-concepts/architecture). Specifically **workspaces**, **sessions**, **peers**, and **messages**. + +### Before You Start + +**This integration uses HTTP Request nodes.** There's no native Honcho node for n8n yet. While this requires more setup, it gives you full control over the API and works with any n8n version. + +**This tutorial is instructional, not production-ready.** We load a single Gmail message with hardcoded IDs to demonstrate the concepts clearly. See [Next Steps](#next-steps) for handling multiple messages and dynamic configurations. + +**Why Honcho over n8n's built-in memory?** n8n's "memory" nodes are vector databases for RAG-style retrieval. Honcho offers richer context and reasoning—it builds understanding of users over time, not just similarity search. [Learn more](https://blog.plasticlabs.ai/blog/Memory-as-Reasoning). + +### Setting Up the HTTP Request Node + +The Honcho integration in n8n uses the HTTP Request node to interact with the Honcho API. Here's how to configure it: + +1. Add an **HTTP Request** node to your workflow (Core > HTTP Request) + +
+ + Adding HTTP Request node from Core nodes + +
+ +2. Set the **Method** based on your operation (typically `POST` for creating resources, `GET` for retrieving) + +3. Set the **URL** to the appropriate Honcho API endpoint. For example, create workspace is `https://api.honcho.dev/v3/workspaces` + +4. For authentication, select **Generic Credential Type** and then **Bearer Auth** + +5. Click **Create New Credential** and paste your Honcho API key in the Bearer Token field + +
+ + Bearer Auth credential setup in n8n + +
+ +6. Check **Send Body** and select **JSON** as the Body Content Type when creating resources + +## Step-by-Step Tutorial + +We'll build a workflow that ingests Gmail emails into Honcho, then uses that memory to power a conversational AI chatbot. + +The workflow has two parts (separated by sticky notes in the canvas): + +1. **Data Ingestion**: Manual trigger → Workspace → Session → Gmail → Extract Peers → Create Peers → Add to Session → Create Messages + +2. **AI Chat Interface**: Chat Trigger → Agent (with LLM and Honcho tools) + +The Agent uses Honcho's `get_context()` endpoint to retrieve relevant information about email conversations, enabling contextual conversations about your email data. + +![Complete workflow overview showing both data ingestion and chat sections](/images/integrations/n8n/complete_workflow.png) + +### Part 1: Loading Email Data into Honcho + +These nodes handle the initial setup and data ingestion: + + +**Pro Tip: Copy from API Playground** + +The fastest way to configure any Honcho endpoint is to copy the curl command directly from the [app.honcho.dev](https://app.honcho.dev) API playground: + +1. Navigate to the endpoint you want to use in the API playground +2. Fill in your parameters, verify the results and click **Copy as cURL** +3. Then in n8n use the **import cURL** button to directly import the request (be sure to verify the bearer token imported correctly) + + + +#### Step 1: Manual Trigger + +Start with a **Manual Trigger** node to execute the workflow on demand. This is useful for initial setup and testing before automating with a Gmail trigger. + +#### Step 2: Get or Create Workspace + +1. Add an **HTTP Request** node +2. **Method**: `POST` +3. **URL**: `https://api.honcho.dev/v3/workspaces` +4. **Body** (JSON): `{ "id": "email-test", "metadata": {} }` + + +**Verify Your Data in Honcho** +As you build the data ingestion workflow, verify everything is created correctly in your [Honcho instance](https://app.honcho.dev/). + + +#### Step 3: Get or Create Session + +1. Add another **HTTP Request** node +2. **Method**: `POST` +3. **URL**: `https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/sessions` +4. **Body** (JSON): `{ "id": "new_session" }` + +#### Step 4: Get Gmail Message + +1. Add a **Gmail** node +2. **Operation**: Get +3. **Message ID**: Your target message ID (a string of letters & numbers) +4. Configure your Gmail OAuth2 credentials + + +**Finding the Gmail Message ID** +The easiest way to find a Gmail message ID is to use n8n's Gmail Get Many operation. Temporarily add it, set the limit to 1, and execute. Use the message ID in the output for the message ID field. + +In this tutorial, we load in only a single message to demonstrate the workflow. + + +#### Step 5: Extract Peers from Email + +Use native n8n nodes to extract email participants as peers: + +**5a. Add a Set node ("Combine Email Fields")** +- Combines From, To, Cc, Bcc into an array of individual emails +- **Field name**: `allEmails` +- **Type**: Array +- **Value**: `{{ [$json.From, $json.To, $json.Cc, $json.Bcc].filter(Boolean).flatMap(field => field.split(',').map(e => e.trim())).filter(Boolean) }}` + +**5b. Add a Split Out node** +- Splits the array into individual items (one per email address) +- **Field to Split Out**: `allEmails` + +**5c. Add a Set node ("Clean Names")** +- Extracts the display name from each email and formats it +- **Field name**: `name` +- **Value**: `{{ $json.allEmails.split('<')[0].trim().replace(/ /g, '_') }}` + +#### Step 6: Get or Create Peer + +1. Add an **HTTP Request** node +2. **Method**: `POST` +3. **URL**: `https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/peers` +4. **Body**: `{ "id": "{{ $json.name }}" }` + +This creates a peer for each email participant, allowing Honcho to build understanding of each person. + +#### Step 7: Add Peers to Session + +1. Add an **HTTP Request** node +2. **Method**: `POST` +3. **URL**: `https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/sessions/{{ $json.id }}/peers` +4. **Body**: `{ "{{ $json.id }}": {} }` + +#### Step 8: Limit Node + +Add a **Limit** node to control the flow so the message is only added once to the session. + +#### Step 9: Create Message for Session + +1. Add an **HTTP Request** node +2. **Method**: `POST` +3. **URL**: `https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/sessions/{{ $('Get or Create Session').item.json.id }}/messages/` +4. **Body** (JSON): `{ "messages": [{ "content": "{{ $('Get a message').item.json.snippet }}", "peer_id": "{{ $('Get a message').item.json.From.split('<')[0].trim().replace(/ /g, '_') }}" }] }` + +The `peer_id` must exactly match a peer created in Step 6. The expression above uses the same cleaning logic as the Clean Names node (`split('<')[0].trim().replace(/ /g, '_')`). + +### Part 2: Building a Stateful AI Chatbot + +Now that data is loaded into Honcho, create a chat interface that leverages this memory: + +#### Step 1: Chat Trigger + +Add a **When chat message received** node (from LangChain nodes) to create an interactive chat interface. + +#### Step 2: AI Agent + +1. Add an **Agent** node (LangChain) +2. Configure the system message: +``` +You are a helpful assistant that retrieves context about email conversations. + +Use the Get_Context tool to retrieve session context. + +Today's date: {{ $now }} +``` + +#### Step 3: Connect LLM + +Add an **OpenAI Chat Model** node (or your preferred LLM) and connect it to the Agent. + +#### Step 4: Add Honcho Tools + +Create an HTTP Request Tool node for Honcho's context retrieval: + +**Get Context Tool:** +- **Method**: `GET` +- **URL**: `https://api.honcho.dev/v3/workspaces/email-test/sessions/new_session/context` +- Returns formatted context for the entire session including all messages and peer interactions + +Connect the tool to the Agent node. The URL uses the same workspace (`email-test`) and session (`new_session`) IDs created during data ingestion. + +--- + +## Import the Workflow + +Want to skip the manual setup? Import this workflow directly into n8n. In n8n, go to **Workflows** → **Import from URL** (use the raw JSON link below) or **Import from File**. + +[Import from URL (raw JSON)](https://raw.githubusercontent.com/plastic-labs/honcho/main/examples/n8n/n8n.json) or expand below to copy: + + +```json +{ + "name": "Honcho Empowered Email AI Agent", + "nodes": [ + { + "parameters": { + "content": "## Data Ingestion\nLoads Gmail email into Honcho.\n\n**Run this section first** by clicking 'Execute workflow'.", + "height": 356, + "width": 2008 + }, + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [ + -16, + -64 + ], + "id": "53f767a8-8df9-4ad7-8a3d-37c145495627", + "name": "Sticky Note - Data Ingestion" + }, + { + "parameters": { + "content": "## AI Chat With Honcho get_context()\nQuery your email data using natural language.\n\n**Run after data ingestion** to chat with the agent.", + "height": 480, + "width": 752 + }, + "type": "n8n-nodes-base.stickyNote", + "typeVersion": 1, + "position": [ + 32, + 464 + ], + "id": "34066961-d29e-4fe8-93bf-8f7043e142b0", + "name": "Sticky Note - AI Chat" + }, + { + "parameters": { + "model": "gpt-4o", + "options": {} + }, + "id": "16da7a98-5622-427a-bbab-1be39f828d0b", + "name": "OpenAI Chat Model", + "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", + "position": [ + 240, + 800 + ], + "typeVersion": 1, + "credentials": { + "openAiApi": { + "id": "qrvGphL3ydUODxQZ", + "name": "OpenAi account" + } + } + }, + { + "parameters": { + "options": { + "systemMessage": "You are a helpful assistant that retrieves context about email conversations.\n\nUse the Get_Context tool to retrieve session context.\n\nToday's date: {{ $now }}" + } + }, + "id": "049c3c19-756c-4755-88d9-94312857d9bb", + "name": "AI Agent", + "type": "@n8n/n8n-nodes-langchain.agent", + "position": [ + 368, + 576 + ], + "typeVersion": 1.7 + }, + { + "parameters": { + "options": {} + }, + "id": "b9a2ef6a-0e81-45c9-a5ea-16e74a9aa77d", + "name": "When chat message received", + "type": "@n8n/n8n-nodes-langchain.chatTrigger", + "position": [ + 80, + 576 + ], + "webhookId": "c91764c2-0b51-4025-ad74-d5f44127aa5a", + "typeVersion": 1.1 + }, + { + "parameters": { + "method": "POST", + "url": "=https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/peers", + "authentication": "predefinedCredentialType", + "nodeCredentialType": "httpBearerAuth", + "sendBody": true, + "bodyParameters": { + "parameters": [ + { + "name": "id", + "value": "={{ $json.name }}" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.3, + "position": [ + 1296, + 96 + ], + "id": "f2e81445-a866-4d9f-9a8a-b2dc8cbaea8b", + "name": "Get or Create Peer", + "credentials": { + "httpBearerAuth": { + "id": "NbrkGo1GdYWQY3OX", + "name": "Bearer Auth account" + } + } + }, + { + "parameters": { + "operation": "get", + "messageId": "19b8fee837985953" + }, + "type": "n8n-nodes-base.gmail", + "typeVersion": 2.2, + "position": [ + 608, + 96 + ], + "id": "ee5d8cc7-876b-4096-b6e5-14f1b92084a6", + "name": "Get a message", + "webhookId": "4ab02540-af03-405d-a6eb-dfcaa76477fc", + "credentials": { + "gmailOAuth2": { + "id": "a2RvA5NMNjfOeHtd", + "name": "Gmail account 2" + } + } + }, + { + "parameters": { + "method": "POST", + "url": "=https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/sessions", + "authentication": "predefinedCredentialType", + "nodeCredentialType": "httpBearerAuth", + "sendBody": true, + "bodyParameters": { + "parameters": [ + { + "name": "id", + "value": "=new_session" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.3, + "position": [ + 448, + 96 + ], + "id": "eef01db2-3355-484d-97b8-34480c40fcf8", + "name": "Get or Create Session", + "credentials": { + "httpBearerAuth": { + "id": "NbrkGo1GdYWQY3OX", + "name": "Bearer Auth account" + } + } + }, + { + "parameters": {}, + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [ + 48, + 96 + ], + "id": "a9de9d8f-7911-444f-99bd-7d287f587eff", + "name": "When clicking 'Execute workflow'" + }, + { + "parameters": { + "method": "POST", + "url": "https://api.honcho.dev/v3/workspaces", + "authentication": "predefinedCredentialType", + "nodeCredentialType": "httpBearerAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "{\n \"id\": \"email-test\",\n \"metadata\": {}\n}", + "options": {} + }, + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.3, + "position": [ + 240, + 96 + ], + "id": "49b91f82-1cd8-49aa-85a5-57a8ad7b5128", + "name": "Get or Create Workspace", + "credentials": { + "httpBearerAuth": { + "id": "NbrkGo1GdYWQY3OX", + "name": "Bearer Auth account" + } + } + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "allEmails", + "name": "allEmails", + "type": "array", + "value": "={{ [$json.From, $json.To, $json.Cc, $json.Bcc].filter(Boolean).flatMap(field => field.split(',').map(e => e.trim())).filter(Boolean) }}" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 784, + 96 + ], + "id": "3eb4ce63-d95c-42d7-8a16-f35e2419d8c0", + "name": "Combine Email Fields" + }, + { + "parameters": { + "fieldToSplitOut": "allEmails", + "options": {} + }, + "type": "n8n-nodes-base.splitOut", + "typeVersion": 1, + "position": [ + 960, + 96 + ], + "id": "4702001d-40f5-4b44-b443-6fb0b94e58a9", + "name": "Split Out" + }, + { + "parameters": { + "assignments": { + "assignments": [ + { + "id": "name", + "name": "name", + "type": "string", + "value": "={{ $json.allEmails.split('<')[0].trim().replace(/ /g, '_') }}" + } + ] + }, + "options": {} + }, + "type": "n8n-nodes-base.set", + "typeVersion": 3.4, + "position": [ + 1136, + 96 + ], + "id": "93327201-bfbe-4385-bfcd-7646f0513a62", + "name": "Clean Names" + }, + { + "parameters": { + "method": "POST", + "url": "=https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/sessions/{{ $('Get or Create Session').item.json.id }}/messages/", + "authentication": "predefinedCredentialType", + "nodeCredentialType": "httpBearerAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\"messages\": [{\"content\": \"{{ $('Get a message').item.json.snippet }}\", \"peer_id\": \"{{ $('Get a message').item.json.From.split('<')[0].trim().replace(/ /g, '_') }}\"}]}", + "options": {} + }, + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.3, + "position": [ + 1808, + 96 + ], + "id": "cb6732d0-e5a9-4d6e-98ef-8fe1c290740b", + "name": "Create Message for Session", + "credentials": { + "httpBearerAuth": { + "id": "NbrkGo1GdYWQY3OX", + "name": "Bearer Auth account" + } + } + }, + { + "parameters": { + "method": "POST", + "url": "=https://api.honcho.dev/v3/workspaces/{{ $('Get or Create Workspace').item.json.id }}/sessions/{{ $('Get or Create Session').item.json.id }}/peers", + "authentication": "predefinedCredentialType", + "nodeCredentialType": "httpBearerAuth", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\"{{ $json.id }}\": {}}", + "options": {} + }, + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.3, + "position": [ + 1472, + 96 + ], + "id": "3c2065b6-bd68-4698-a740-db3cd52f2267", + "name": "Add Peers to Session", + "credentials": { + "httpBearerAuth": { + "id": "NbrkGo1GdYWQY3OX", + "name": "Bearer Auth account" + } + } + }, + { + "parameters": {}, + "type": "n8n-nodes-base.limit", + "typeVersion": 1, + "position": [ + 1632, + 96 + ], + "id": "d7c38f49-1a76-4a67-a540-4bec7aae1d9f", + "name": "Limit" + }, + { + "parameters": { + "url": "https://api.honcho.dev/v3/workspaces/email-test/sessions/new_session/context", + "authentication": "predefinedCredentialType", + "nodeCredentialType": "httpBearerAuth", + "options": {} + }, + "type": "n8n-nodes-base.httpRequestTool", + "typeVersion": 4.3, + "position": [ + 656, + 784 + ], + "id": "095e62d6-aae3-4eb8-9b9c-c473bfe2716d", + "name": "Get_Context", + "credentials": { + "httpBearerAuth": { + "id": "NbrkGo1GdYWQY3OX", + "name": "Bearer Auth account" + } + } + } + ], + "pinData": {}, + "connections": { + "OpenAI Chat Model": { + "ai_languageModel": [ + [ + { + "node": "AI Agent", + "type": "ai_languageModel", + "index": 0 + } + ] + ] + }, + "When chat message received": { + "main": [ + [ + { + "node": "AI Agent", + "type": "main", + "index": 0 + } + ] + ] + }, + "When clicking 'Execute workflow'": { + "main": [ + [ + { + "node": "Get or Create Workspace", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get or Create Workspace": { + "main": [ + [ + { + "node": "Get or Create Session", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get a message": { + "main": [ + [ + { + "node": "Combine Email Fields", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get or Create Session": { + "main": [ + [ + { + "node": "Get a message", + "type": "main", + "index": 0 + } + ] + ] + }, + "Combine Email Fields": { + "main": [ + [ + { + "node": "Split Out", + "type": "main", + "index": 0 + } + ] + ] + }, + "Split Out": { + "main": [ + [ + { + "node": "Clean Names", + "type": "main", + "index": 0 + } + ] + ] + }, + "Clean Names": { + "main": [ + [ + { + "node": "Get or Create Peer", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get or Create Peer": { + "main": [ + [ + { + "node": "Add Peers to Session", + "type": "main", + "index": 0 + } + ] + ] + }, + "Add Peers to Session": { + "main": [ + [ + { + "node": "Limit", + "type": "main", + "index": 0 + } + ] + ] + }, + "Limit": { + "main": [ + [ + { + "node": "Create Message for Session", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get_Context": { + "ai_tool": [ + [ + { + "node": "AI Agent", + "type": "ai_tool", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1", + "availableInMCP": false + }, + "versionId": "ba0a3b77-cc19-49fd-9189-aaee65b35f99", + "meta": { + "templateCredsSetupCompleted": true, + "instanceId": "4e34c96e55eb26be21fa69ca62c4851a5d09b678190481f5d47c084b6b327003" + }, + "id": "dKOYeEOdrZOetmFRmIAUJ", + "tags": [] +} +``` + + + + +**Important:** After importing, you'll need to: +- Add your Honcho API key to the Bearer Auth credential +- Connect your Gmail OAuth2 credential +- Add your OpenAI API key (or swap for your preferred LLM) +- Update the Gmail Message ID in "Get a message" node + +**Running the workflow:** +1. First, execute the data ingestion section (click "Execute workflow") +2. Then use the chat interface to query your email data + + +--- + +## Next Steps + +Once you have the basic workflow running, consider these enhancements: + +- **Dynamic IDs**: Use n8n variables instead of hardcoding `email-test` and `new_session` +- **Chat with Peers**: Add an HTTP Request Tool for natural language queries about peer representations. Read more in the [docs](/v3/documentation/features/chat). +- **Load more messages**: Use Gmail's "Get All" operation to load entire conversation threads +- **Make it real-time**: Add a **Gmail Trigger** node to automatically ingest new emails as they arrive +- **Add error handling**: Connect an **Error Trigger** node with notifications (Email, Slack) and retry logic +- **Expand to other data sources**: Honcho works with Slack messages, CRM interactions, support tickets, and more + +--- + +## Related Resources + + + + Understand workspaces, sessions, peers, and messages + + + Learn about retrieving formatted conversation context + + diff --git a/docs/v3/guides/migrations/mem0.mdx b/docs/v3/guides/migrations/mem0.mdx index 77022854..79893bd4 100644 --- a/docs/v3/guides/migrations/mem0.mdx +++ b/docs/v3/guides/migrations/mem0.mdx @@ -259,14 +259,14 @@ Reference the [API Comparison](#api-comparison) to replace your Mem0 API calls w Mem0 requires manual assembly of context from `search()` results. Honcho's `session.get_context()` returns a ready-to-use `SessionContext` object with built-in token limits, auto-included summaries, and format helpers (`.to_openai()`, `.to_anthropic()`). - + Learn more about token-optimized context retrieval Mem0's `search()` returns basic vector, semantic, or raw memory matches. Honcho's `peer.chat()` enables your agent to *reason* about what it knows—returning synthesized natural language insights with streaming support and scoped queries. - + Learn more about inference-powered queries @@ -285,7 +285,7 @@ Additional features with **no Mem0 equivalent**: Understand peers and sessions - + Inference responses diff --git a/docs/v3/guides/overview.mdx b/docs/v3/guides/overview.mdx index 9b39510f..a74d66c3 100644 --- a/docs/v3/guides/overview.mdx +++ b/docs/v3/guides/overview.mdx @@ -16,10 +16,10 @@ Each guide focuses on a specific use case with practical examples. The goal is t Quick integration guides to get up and running: - + Get Honcho running with a single prompt in Claude Code - + Add persistent memory and theory of mind to your LangGraph agents diff --git a/docs/v3/guides/storing-data.mdx b/docs/v3/guides/storing-data.mdx index 961b9e1f..a1085668 100644 --- a/docs/v3/guides/storing-data.mdx +++ b/docs/v3/guides/storing-data.mdx @@ -42,7 +42,7 @@ Once a `Message` is saved in Honcho, it will kick off a background task that looks at the new data to generate insights about the `Peer` that sent the `Message` This is the default behavior of Honcho and can be turned off by [configuring the -Peer or Session](/v3/documentation/core-concepts/configuration) +Peer or Session](/v3/documentation/features/advanced/reasoning-configuration) This pattern of having a Peer, Session, and Messages is highly flexible and works for many different use cases and agent setups. Some use cases may only diff --git a/docs/v3/migrations/from-mem0.mdx b/docs/v3/migrations/from-mem0.mdx index 77022854..79893bd4 100644 --- a/docs/v3/migrations/from-mem0.mdx +++ b/docs/v3/migrations/from-mem0.mdx @@ -259,14 +259,14 @@ Reference the [API Comparison](#api-comparison) to replace your Mem0 API calls w Mem0 requires manual assembly of context from `search()` results. Honcho's `session.get_context()` returns a ready-to-use `SessionContext` object with built-in token limits, auto-included summaries, and format helpers (`.to_openai()`, `.to_anthropic()`). - + Learn more about token-optimized context retrieval Mem0's `search()` returns basic vector, semantic, or raw memory matches. Honcho's `peer.chat()` enables your agent to *reason* about what it knows—returning synthesized natural language insights with streaming support and scoped queries. - + Learn more about inference-powered queries @@ -285,7 +285,7 @@ Additional features with **no Mem0 equivalent**: Understand peers and sessions - + Inference responses diff --git a/docs/v3/openapi.json b/docs/v3/openapi.json index 355b6b42..09a8168f 100644 --- a/docs/v3/openapi.json +++ b/docs/v3/openapi.json @@ -11,7 +11,6 @@ }, "license": { "name": "GNU Affero General Public License v3.0", - "identifier": "AGPL-3.0-only", "url": "https://github.com/plastic-labs/honcho/blob/main/LICENSE" }, "version": "3.0.0" @@ -82,7 +81,7 @@ "workspaces" ], "summary": "Get All Workspaces", - "description": "Get all Workspaces, paginated with optional filters.", + "description": "Get all Workspaces", "operationId": "get_all_workspaces_v3_workspaces_list_post", "security": [ { @@ -434,7 +433,7 @@ "workspaces" ], "summary": "Schedule Dream", - "description": "Manually schedule a dream task for a specific collection.\n\nThis endpoint bypasses all automatic dream conditions (document threshold,\nminimum hours between dreams) and schedules the dream task for a future execution.\n\nCurrently this endpoint only supports scheduling immediate dreams. In the future,\nusers may pass a cron-style expression to schedule dreams at specific times.", + "description": "", "operationId": "schedule_dream_v3_workspaces__workspace_id__schedule_dream_post", "security": [ { @@ -487,7 +486,7 @@ "peers" ], "summary": "Get Peers", - "description": "Get all Peers for a Workspace, paginated with optional filters.", + "description": "Get all Peers for a Workspace", "operationId": "get_peers_v3_workspaces__workspace_id__peers_list_post", "security": [ { @@ -710,7 +709,7 @@ "peers" ], "summary": "Get Sessions For Peer", - "description": "Get all Sessions for a Peer, paginated with optional filters.", + "description": "Get all Sessions for a Peer", "operationId": "get_sessions_for_peer_v3_workspaces__workspace_id__peers__peer_id__sessions_post", "security": [ { @@ -812,8 +811,8 @@ "tags": [ "peers" ], - "summary": "Query a Peer's representation using natural language", - "description": "Query a Peer's representation using natural language. Performs agentic search and reasoning to comprehensively\nanswer the query based on all latent knowledge gathered about the peer from their messages and conclusions.", + "summary": "Chat", + "description": "", "operationId": "chat_v3_workspaces__workspace_id__peers__peer_id__chat_post", "security": [ { @@ -1374,7 +1373,7 @@ "sessions" ], "summary": "Get Sessions", - "description": "Get all Sessions for a Workspace, paginated with optional filters.", + "description": "Get all Sessions for a Workspace", "operationId": "get_sessions_v3_workspaces__workspace_id__sessions_list_post", "security": [ { @@ -1940,7 +1939,7 @@ "sessions" ], "summary": "Get Session Peers", - "description": "Get all Peers in a Session. Results are paginated.", + "description": "Get all Peers in a Session", "operationId": "get_session_peers_v3_workspaces__workspace_id__sessions__session_id__peers_get", "security": [ { @@ -2208,7 +2207,7 @@ "description": "Number of tokens to use for the context. Includes summary if set to true. Includes representation and peer card if they are included in the response. If not provided, the context will be exhaustive (within 100000 tokens)" }, { - "name": "last_message", + "name": "search_query", "in": "query", "required": false, "schema": { @@ -2220,10 +2219,10 @@ "type": "null" } ], - "description": "The most recent message, used to fetch semantically relevant conclusions", - "title": "Last Message" + "description": "A query string used to fetch semantically relevant conclusions", + "title": "Search Query" }, - "description": "The most recent message, used to fetch semantically relevant conclusions" + "description": "A query string used to fetch semantically relevant conclusions" }, { "name": "summary", @@ -2279,11 +2278,11 @@ "required": false, "schema": { "type": "boolean", - "description": "Only used if `last_message` is provided. Whether to limit the representation to the session (as opposed to everything known about the target peer)", + "description": "Only used if `search_query` is provided. Whether to limit the representation to the session (as opposed to everything known about the target peer)", "default": false, "title": "Limit To Session" }, - "description": "Only used if `last_message` is provided. Whether to limit the representation to the session (as opposed to everything known about the target peer)" + "description": "Only used if `search_query` is provided. Whether to limit the representation to the session (as opposed to everything known about the target peer)" }, { "name": "search_top_k", @@ -2300,10 +2299,10 @@ "type": "null" } ], - "description": "Only used if `last_message` is provided. The number of semantic-search-retrieved conclusions to include in the representation", + "description": "Only used if `search_query` is provided. The number of semantic-search-retrieved conclusions to include in the representation", "title": "Search Top K" }, - "description": "Only used if `last_message` is provided. The number of semantic-search-retrieved conclusions to include in the representation" + "description": "Only used if `search_query` is provided. The number of semantic-search-retrieved conclusions to include in the representation" }, { "name": "search_max_distance", @@ -2320,10 +2319,10 @@ "type": "null" } ], - "description": "Only used if `last_message` is provided. The maximum distance to search for semantically relevant conclusions", + "description": "Only used if `search_query` is provided. The maximum distance to search for semantically relevant conclusions", "title": "Search Max Distance" }, - "description": "Only used if `last_message` is provided. The maximum distance to search for semantically relevant conclusions" + "description": "Only used if `search_query` is provided. The maximum distance to search for semantically relevant conclusions" }, { "name": "include_most_frequent", @@ -2331,11 +2330,11 @@ "required": false, "schema": { "type": "boolean", - "description": "Only used if `last_message` is provided. Whether to include the most frequent conclusions in the representation", + "description": "Only used if `search_query` is provided. Whether to include the most frequent conclusions in the representation", "default": false, "title": "Include Most Frequent" }, - "description": "Only used if `last_message` is provided. Whether to include the most frequent conclusions in the representation" + "description": "Only used if `search_query` is provided. Whether to include the most frequent conclusions in the representation" }, { "name": "max_conclusions", @@ -2352,10 +2351,10 @@ "type": "null" } ], - "description": "Only used if `last_message` is provided. The maximum number of conclusions to include in the representation", + "description": "Only used if `search_query` is provided. The maximum number of conclusions to include in the representation", "title": "Max Conclusions" }, - "description": "Only used if `last_message` is provided. The maximum number of conclusions to include in the representation" + "description": "Only used if `search_query` is provided. The maximum number of conclusions to include in the representation" } ], "responses": { @@ -2663,7 +2662,7 @@ "messages" ], "summary": "Get Messages", - "description": "Get all messages for a Session with optional filters. Results are paginated.", + "description": "Get all messages for a Session with optional filters", "operationId": "get_messages_v3_workspaces__workspace_id__sessions__session_id__messages_list_post", "security": [ { @@ -2992,7 +2991,7 @@ "conclusions" ], "summary": "List Conclusions", - "description": "List Conclusions using optional filters, ordered by recency unless `reverse` is true. Results are paginated.", + "description": "List Conclusions using optional filters, ordered by recency unless `reverse` is true", "operationId": "list_conclusions_v3_workspaces__workspace_id__conclusions_list_post", "security": [ { @@ -3564,23 +3563,6 @@ } } } - }, - "/metrics": { - "get": { - "summary": "Metrics", - "description": "Prometheus metrics endpoint", - "operationId": "metrics_metrics_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } } }, "components": { @@ -3658,7 +3640,14 @@ "description": "The peer the conclusion is about" }, "session_id": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Session Id" }, "created_at": { @@ -3673,7 +3662,6 @@ "content", "observer_id", "observed_id", - "session_id", "created_at" ], "title": "Conclusion", @@ -3717,17 +3705,23 @@ "description": "The peer the conclusion is about" }, "session_id": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Session Id", - "description": "The session this conclusion relates to" + "description": "A session ID to store the conclusion in, if specified" } }, "type": "object", "required": [ "content", "observer_id", - "observed_id", - "session_id" + "observed_id" ], "title": "ConclusionCreate", "description": "Schema for creating a single conclusion." @@ -4787,16 +4781,22 @@ "description": "Type of dream to schedule" }, "session_id": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Session Id", - "description": "Session ID to scope the dream to" + "description": "Session ID to scope the dream to if specified" } }, "type": "object", "required": [ "observer", - "dream_type", - "session_id" + "dream_type" ], "title": "ScheduleDreamRequest" }, diff --git a/examples/crewai/python/src/honcho_crewai/tools.py b/examples/crewai/python/src/honcho_crewai/tools.py index f0fc0c75..9a5c2835 100644 --- a/examples/crewai/python/src/honcho_crewai/tools.py +++ b/examples/crewai/python/src/honcho_crewai/tools.py @@ -118,7 +118,7 @@ class HonchoGetContextTool(BaseTool): """ try: session = self._honcho.session(self._session_id) - context = session.get_context( + context = session.context( summary=summary, tokens=tokens, peer_target=peer_target, @@ -220,9 +220,8 @@ class HonchoDialecticTool(BaseTool): # Query the dialectic API (non-streaming) response = peer.chat( query=query, - stream=False, target=target, - session_id=scope_session_id, + session=scope_session_id, ) # Return the response or a default message diff --git a/examples/langgraph/python/main.py b/examples/langgraph/python/main.py index 3444d7c0..e7a34a86 100644 --- a/examples/langgraph/python/main.py +++ b/examples/langgraph/python/main.py @@ -36,7 +36,7 @@ def chatbot(state: State): # Get context in OpenAI format with token limit # tokens=2000 limits the context to 2000 tokens to manage costs and fit within model limits - messages = session.get_context(tokens=2000).to_openai(assistant=assistant) + messages = session.context(tokens=2000).to_openai(assistant=assistant) # Generate response response = llm.chat.completions.create( diff --git a/examples/langgraph/typescript/main.ts b/examples/langgraph/typescript/main.ts index 49a67c63..2bc7cfad 100644 --- a/examples/langgraph/typescript/main.ts +++ b/examples/langgraph/typescript/main.ts @@ -42,7 +42,7 @@ async function chatbot(state: State) { // Get context in OpenAI format with token limit // tokens: 2000 limits the context to 2000 tokens to manage costs and fit within model limits - const messages = (await session.getContext({ tokens: 2000 })).toOpenAI(assistant); + const messages = (await session.context({ tokens: 2000 })).toOpenAI(assistant); // Generate response const response = await llm.chat.completions.create({ diff --git a/fly.toml b/fly.toml index 26824fe3..d94bdba8 100644 --- a/fly.toml +++ b/fly.toml @@ -24,17 +24,17 @@ kill_timeout = '5s' soft_limit = 20 [[vm]] - memory = '512mb' + memory = '1gb' cpu_kind = 'shared' cpus = 1 processes = ['api', 'deriver'] [[metrics]] -port = 8000 -path = "/metrics" -processes = ["api"] + port = 8000 + path = "/metrics" + processes = ["api"] [[metrics]] -port = 9090 -path = "/metrics" -processes = ["deriver"] + port = 9090 + path = "/metrics" + processes = ["deriver"] diff --git a/mcp/package.json b/mcp/package.json index c13b93ba..d4014ce2 100644 --- a/mcp/package.json +++ b/mcp/package.json @@ -15,7 +15,7 @@ "deploy:staging": "wrangler deploy --env staging" }, "dependencies": { - "@honcho-ai/sdk": "^1.6.0" + "@honcho-ai/sdk": "^2.0.0" }, "devDependencies": { "@cloudflare/workers-types": "^4.20241002.0", diff --git a/mcp/worker.ts b/mcp/worker.ts index 2b558852..bf08f417 100644 --- a/mcp/worker.ts +++ b/mcp/worker.ts @@ -223,17 +223,20 @@ class HonchoWorker { * Get personalization insights about the user, based on the query and the accumulated knowledge of the user across all conversations. * @param sessionId - The ID of the session for context * @param query - The question about the user's preferences, habits, etc. + * @param reasoningLevel - Optional reasoning level: "minimal", "low", "medium", "high", or "max" * @returns A string with the personalization insights */ async getPersonalizationInsights( sessionId: string, query: string, + reasoningLevel?: string, ): Promise { const userPeer = await this.honcho.peer(this.config.userName); // Get the personalization insights (non-streaming returns string | null) const personalizationInsights = await userPeer.chat(query, { session: sessionId, + reasoningLevel, }); if (!personalizationInsights || typeof personalizationInsights !== 'string') { @@ -345,6 +348,7 @@ class HonchoWorker { * @param query - The natural language question to ask * @param targetPeerId - Optional target peer ID for local representation queries * @param sessionId - Optional session ID to scope the query to a specific session + * @param reasoningLevel - Optional reasoning level: "minimal", "low", "medium", "high", or "max" * @returns Response string containing the answer to the query, or "None" if no relevant information */ async chat( @@ -352,6 +356,7 @@ class HonchoWorker { query: string, targetPeerId?: string, sessionId?: string, + reasoningLevel?: string, ): Promise { const peer = await this.honcho.peer(peerId); let targetPeer; @@ -362,6 +367,7 @@ class HonchoWorker { const result = await peer.chat(query, { target: targetPeer, session: sessionId, + reasoningLevel, }); if (!result || typeof result !== 'string') { @@ -387,6 +393,247 @@ class HonchoWorker { return peers; } + /** + * Get the peer card for a peer. + * @param peerId - The ID of the observer peer + * @param targetPeerId - Optional target peer ID to get the card about + * @returns The peer card content or null if not found + */ + async getPeerCard( + peerId: string, + targetPeerId?: string, + ): Promise { + const peer = await this.honcho.peer(peerId); + let targetPeer; + if (targetPeerId) { + targetPeer = await this.honcho.peer(targetPeerId); + } + return await peer.card(targetPeer); + } + + /** + * Get the peer context (combined representation and peer card). + * @param peerId - The ID of the observer peer + * @param targetPeerId - Optional target peer ID + * @param sessionId - Optional session ID to scope the context + * @param searchQuery - Optional semantic search query + * @param maxConclusions - Optional maximum number of conclusions to include + * @returns Context object with representation and peer card + */ + async getPeerContext( + peerId: string, + targetPeerId?: string, + sessionId?: string, + searchQuery?: string, + maxConclusions?: number, + ): Promise> { + const peer = await this.honcho.peer(peerId); + let targetPeer; + if (targetPeerId) { + targetPeer = await this.honcho.peer(targetPeerId); + } + const context = await peer.context({ + target: targetPeer, + session: sessionId, + searchQuery, + maxConclusions, + }); + return { + peer_id: context.peerId, + target_id: context.targetId, + representation: context.representation, + peer_card: context.peerCard, + }; + } + + /** + * Get the formatted representation for a peer. + * @param peerId - The ID of the observer peer + * @param targetPeerId - Optional target peer ID + * @param sessionId - Optional session ID to scope the representation + * @param searchQuery - Optional semantic search query + * @param maxConclusions - Optional maximum number of conclusions + * @returns Formatted representation string + */ + async getRepresentation( + peerId: string, + targetPeerId?: string, + sessionId?: string, + searchQuery?: string, + maxConclusions?: number, + ): Promise { + const peer = await this.honcho.peer(peerId); + let targetPeer; + if (targetPeerId) { + targetPeer = await this.honcho.peer(targetPeerId); + } + return await peer.representation({ + target: targetPeer, + session: sessionId, + searchQuery, + maxConclusions, + }); + } + + ////////////////////////////////////////////////////// + /// /// + /// Conclusions operations /// + /// /// + ////////////////////////////////////////////////////// + + /** + * List conclusions for a peer. + * @param peerId - The ID of the observer peer + * @param targetPeerId - Optional target peer to get conclusions about + * @returns List of conclusions + */ + async listConclusions( + peerId: string, + targetPeerId?: string, + ): Promise { + const peer = await this.honcho.peer(peerId); + let conclusionScope; + if (targetPeerId) { + const targetPeer = await this.honcho.peer(targetPeerId); + conclusionScope = peer.conclusionsOf(targetPeer); + } else { + conclusionScope = peer.conclusions; + } + + const conclusionsPage = await conclusionScope.list(); + const conclusions = []; + for await (const conclusion of conclusionsPage) { + conclusions.push({ + id: conclusion.id, + content: conclusion.content, + observer_id: conclusion.observerId, + observed_id: conclusion.observedId, + session_id: conclusion.sessionId, + created_at: conclusion.createdAt, + }); + } + return conclusions; + } + + /** + * Query conclusions using semantic search. + * @param peerId - The ID of the observer peer + * @param query - The semantic search query + * @param targetPeerId - Optional target peer to search conclusions about + * @param topK - Maximum number of results to return + * @returns List of matching conclusions + */ + async queryConclusions( + peerId: string, + query: string, + targetPeerId?: string, + topK?: number, + ): Promise { + const peer = await this.honcho.peer(peerId); + let conclusionScope; + if (targetPeerId) { + const targetPeer = await this.honcho.peer(targetPeerId); + conclusionScope = peer.conclusionsOf(targetPeer); + } else { + conclusionScope = peer.conclusions; + } + + // SDK query returns Conclusion[] directly, not a Page + const conclusions = await conclusionScope.query(query, topK); + return conclusions.map((conclusion) => ({ + id: conclusion.id, + content: conclusion.content, + observer_id: conclusion.observerId, + observed_id: conclusion.observedId, + session_id: conclusion.sessionId, + created_at: conclusion.createdAt, + })); + } + + /** + * Create conclusions manually. + * @param peerId - The ID of the observer peer + * @param targetPeerId - The target peer the conclusions are about + * @param conclusions - List of conclusion content strings + * @param sessionId - Optional session ID to associate with conclusions + * @returns Number of conclusions created + */ + async createConclusions( + peerId: string, + targetPeerId: string, + conclusions: string[], + sessionId?: string, + ): Promise { + const peer = await this.honcho.peer(peerId); + const targetPeer = await this.honcho.peer(targetPeerId); + const conclusionScope = peer.conclusionsOf(targetPeer); + + // Convert string array to ConclusionCreateParams array + const conclusionParams = conclusions.map((content) => ({ + content, + sessionId, + })); + + await conclusionScope.create(conclusionParams); + return conclusions.length; + } + + /** + * Delete a conclusion. + * @param peerId - The ID of the observer peer + * @param targetPeerId - The target peer the conclusion is about + * @param conclusionId - The ID of the conclusion to delete + */ + async deleteConclusion( + peerId: string, + targetPeerId: string, + conclusionId: string, + ): Promise { + const peer = await this.honcho.peer(peerId); + const targetPeer = await this.honcho.peer(targetPeerId); + const conclusionScope = peer.conclusionsOf(targetPeer); + + await conclusionScope.delete(conclusionId); + } + + ////////////////////////////////////////////////////// + /// /// + /// System operations /// + /// /// + ////////////////////////////////////////////////////// + + /** + * Schedule a dream (memory consolidation) for a peer. + * @param peerId - The ID of the observer peer + * @param targetPeerId - Optional target peer to dream about (defaults to observer for self-reflection) + * @param sessionId - Optional session ID to scope the dream to + * @returns Confirmation message + */ + async scheduleDream( + peerId: string, + targetPeerId?: string, + sessionId?: string, + ): Promise { + const peer = await this.honcho.peer(peerId); + const session = sessionId ? await this.honcho.session(sessionId) : undefined; + const targetPeer = targetPeerId ? await this.honcho.peer(targetPeerId) : undefined; + + await this.honcho.scheduleDream({ + observer: peer, + session: session, + observed: targetPeer, + }); + return "Dream scheduled successfully"; + } + + /** + * Get the current queue status for the deriver. + * @returns Queue status information + */ + async getQueueStatus(): Promise> { + return await this.honcho.queueStatus(); + } + ////////////////////////////////////////////////////// /// /// /// Session operations /// @@ -739,6 +986,12 @@ const tools: Tool[] = [ description: "The question about the user's preferences, habits, etc.", }, + reasoning_level: { + type: "string", + enum: ["minimal", "low", "medium", "high", "max"], + description: + "The reasoning level for the response. Higher levels provide more detailed analysis.", + }, }, required: ["session_id", "query"], }, @@ -879,6 +1132,12 @@ const tools: Tool[] = [ description: "Optional session ID to scope the query to a specific session.", }, + reasoning_level: { + type: "string", + enum: ["minimal", "low", "medium", "high", "max"], + description: + "The reasoning level for the response. Higher levels provide more detailed analysis.", + }, }, required: ["peer_id", "query"], }, @@ -892,6 +1151,223 @@ const tools: Tool[] = [ required: [], }, }, + { + name: "get_peer_card", + description: + "Get the peer card for a peer. The peer card contains compact biographical facts about the peer.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + target_peer_id: { + type: "string", + description: + "Optional target peer ID to get the card about. If not provided, returns the peer's own card.", + }, + }, + required: ["peer_id"], + }, + }, + { + name: "get_peer_context", + description: + "Get the peer context, which combines the representation and peer card for comprehensive peer information.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + target_peer_id: { + type: "string", + description: "Optional target peer ID to get context about.", + }, + session_id: { + type: "string", + description: "Optional session ID to scope the context.", + }, + search_query: { + type: "string", + description: "Optional semantic search query to filter conclusions.", + }, + max_conclusions: { + type: "integer", + description: "Maximum number of conclusions to include.", + }, + }, + required: ["peer_id"], + }, + }, + { + name: "get_representation", + description: + "Get the formatted representation for a peer, containing their conclusions and observations.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + target_peer_id: { + type: "string", + description: "Optional target peer ID to get representation about.", + }, + session_id: { + type: "string", + description: "Optional session ID to scope the representation.", + }, + search_query: { + type: "string", + description: "Optional semantic search query to filter conclusions.", + }, + max_conclusions: { + type: "integer", + description: "Maximum number of conclusions to include.", + }, + }, + required: ["peer_id"], + }, + }, + + // Conclusions operations + { + name: "list_conclusions", + description: + "List conclusions for a peer. Conclusions are facts and observations derived from conversations.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + target_peer_id: { + type: "string", + description: + "Optional target peer ID to get conclusions about. If not provided, returns conclusions about self.", + }, + }, + required: ["peer_id"], + }, + }, + { + name: "query_conclusions", + description: "Query conclusions using semantic search.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + query: { + type: "string", + description: "The semantic search query.", + }, + target_peer_id: { + type: "string", + description: "Optional target peer ID to search conclusions about.", + }, + top_k: { + type: "integer", + description: "Maximum number of results to return.", + }, + }, + required: ["peer_id", "query"], + }, + }, + { + name: "create_conclusions", + description: "Create conclusions manually for a peer.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + target_peer_id: { + type: "string", + description: "The target peer the conclusions are about.", + }, + conclusions: { + type: "array", + items: { type: "string" }, + description: "List of conclusion content strings to create.", + }, + session_id: { + type: "string", + description: + "Optional session ID to associate with conclusions. If not provided, conclusions will be global.", + }, + }, + required: ["peer_id", "target_peer_id", "conclusions"], + }, + }, + { + name: "delete_conclusion", + description: "Delete a specific conclusion.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + target_peer_id: { + type: "string", + description: "The target peer the conclusion is about.", + }, + conclusion_id: { + type: "string", + description: "The ID of the conclusion to delete.", + }, + }, + required: ["peer_id", "target_peer_id", "conclusion_id"], + }, + }, + + // System operations + { + name: "schedule_dream", + description: + "Schedule a dream (memory consolidation) for a peer. Dreams help consolidate and improve memory quality.", + inputSchema: { + type: "object", + properties: { + peer_id: { + type: "string", + description: "The ID of the observer peer.", + }, + target_peer_id: { + type: "string", + description: + "Optional target peer to dream about. If not provided, defaults to the observer peer (self-reflection).", + }, + session_id: { + type: "string", + description: + "Optional session ID to scope the dream to. If not provided, the dream will be global.", + }, + }, + required: ["peer_id"], + }, + }, + { + name: "get_queue_status", + description: + "Get the current queue status for the deriver (background processing system).", + inputSchema: { + type: "object", + properties: {}, + required: [], + }, + }, // Session operations { @@ -1170,6 +1646,7 @@ async function executeToolCall( result = await honcho.getPersonalizationInsights( toolArguments.session_id, toolArguments.query, + toolArguments.reasoning_level, ); break; } @@ -1272,6 +1749,7 @@ async function executeToolCall( toolArguments.query, toolArguments.target_peer_id, toolArguments.session_id, + toolArguments.reasoning_level, ); break; } @@ -1280,6 +1758,146 @@ async function executeToolCall( result = await honcho.listPeers(); break; + case "get_peer_card": { + const validation = validateArguments( + toolArguments, + ["peer_id"], + requestId, + ); + if (validation) return validation; + + result = await honcho.getPeerCard( + toolArguments.peer_id, + toolArguments.target_peer_id, + ); + break; + } + + case "get_peer_context": { + const validation = validateArguments( + toolArguments, + ["peer_id"], + requestId, + ); + if (validation) return validation; + + result = await honcho.getPeerContext( + toolArguments.peer_id, + toolArguments.target_peer_id, + toolArguments.session_id, + toolArguments.search_query, + toolArguments.max_conclusions, + ); + break; + } + + case "get_representation": { + const validation = validateArguments( + toolArguments, + ["peer_id"], + requestId, + ); + if (validation) return validation; + + result = await honcho.getRepresentation( + toolArguments.peer_id, + toolArguments.target_peer_id, + toolArguments.session_id, + toolArguments.search_query, + toolArguments.max_conclusions, + ); + break; + } + + // Conclusions operations + case "list_conclusions": { + const validation = validateArguments( + toolArguments, + ["peer_id"], + requestId, + ); + if (validation) return validation; + + result = await honcho.listConclusions( + toolArguments.peer_id, + toolArguments.target_peer_id, + ); + break; + } + + case "query_conclusions": { + const validation = validateArguments( + toolArguments, + ["peer_id", "query"], + requestId, + ); + if (validation) return validation; + + result = await honcho.queryConclusions( + toolArguments.peer_id, + toolArguments.query, + toolArguments.target_peer_id, + toolArguments.top_k, + ); + break; + } + + case "create_conclusions": { + const validation = validateArguments( + toolArguments, + ["peer_id", "target_peer_id", "conclusions"], + requestId, + ); + if (validation) return validation; + + const count = await honcho.createConclusions( + toolArguments.peer_id, + toolArguments.target_peer_id, + toolArguments.conclusions, + toolArguments.session_id, + ); + result = `Created ${count} conclusions successfully`; + break; + } + + case "delete_conclusion": { + const validation = validateArguments( + toolArguments, + ["peer_id", "target_peer_id", "conclusion_id"], + requestId, + ); + if (validation) return validation; + + await honcho.deleteConclusion( + toolArguments.peer_id, + toolArguments.target_peer_id, + toolArguments.conclusion_id, + ); + result = "Conclusion deleted successfully"; + break; + } + + // System operations + case "schedule_dream": { + const validation = validateArguments( + toolArguments, + ["peer_id"], + requestId, + ); + if (validation) return validation; + + result = await honcho.scheduleDream( + toolArguments.peer_id, + toolArguments.target_peer_id, + toolArguments.session_id, + ); + break; + } + + case "get_queue_status": + result = await honcho.getQueueStatus(); + break; + // Session operations case "create_session": { const validation = validateArguments( @@ -1553,7 +2171,7 @@ export default { }, serverInfo: { name: "Honcho MCP Server", - version: "1.0.0", + version: "3.0.0", }, }), ), diff --git a/migrations/versions/e4eba9cfaa6f_make_document_session_name_nullable.py b/migrations/versions/e4eba9cfaa6f_make_document_session_name_nullable.py new file mode 100644 index 00000000..216b61c8 --- /dev/null +++ b/migrations/versions/e4eba9cfaa6f_make_document_session_name_nullable.py @@ -0,0 +1,84 @@ +"""make document session_name nullable for sessionless dreams + +Allow dreams to run without a session_id by making the session_name +column nullable on the documents table. + +Revision ID: e4eba9cfaa6f +Revises: 119a52b73c60 +Create Date: 2026-01-26 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from nanoid import generate as generate_nanoid + +from migrations.utils import get_schema + +# revision identifiers, used by Alembic. +revision: str = "e4eba9cfaa6f" +down_revision: str | None = "119a52b73c60" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +schema = get_schema() + + +def ensure_orphaned_sessions_exist(schema: str) -> None: + """Ensure a per-workspace `__orphaned__` session exists for downgrade. + + During downgrade we backfill NULL `documents.session_name` values to a + sentinel value (`__orphaned__`) and then make the column non-nullable. The + `documents` table has a composite foreign key (`session_name`, + `workspace_name`) that references `sessions` (`name`, `workspace_name`). + If we set `session_name` to a non-null sentinel without ensuring a matching + session row exists for each affected workspace, the UPDATE can violate the + foreign key. + + This helper inserts missing placeholder sessions scoped by workspace using + `ON CONFLICT DO NOTHING` to remain safe and idempotent. + """ + conn = op.get_bind() + orphaned_session_name = "__orphaned__" + + workspaces = conn.execute( + sa.text( + f""" + SELECT DISTINCT workspace_name + FROM "{schema}".documents + WHERE session_name IS NULL + """ + ) + ).fetchall() + + for (workspace_name,) in workspaces: + conn.execute( + sa.text( + f""" + INSERT INTO "{schema}".sessions (id, name, workspace_name, is_active) + VALUES (:id, :name, :workspace_name, true) + ON CONFLICT (name, workspace_name) DO NOTHING + """ + ), + { + "id": generate_nanoid(), + "name": orphaned_session_name, + "workspace_name": workspace_name, + }, + ) + + +def upgrade() -> None: + """Make session_name nullable on documents table.""" + op.alter_column("documents", "session_name", nullable=True, schema=schema) + + +def downgrade() -> None: + """Make session_name non-nullable again, backfilling NULLs first.""" + ensure_orphaned_sessions_exist(schema) + op.execute( + f"UPDATE \"{schema}\".documents SET session_name = '__orphaned__' WHERE session_name IS NULL" + ) + op.alter_column("documents", "session_name", nullable=False, schema=schema) diff --git a/pyproject.toml b/pyproject.toml index 6d7878a2..972d2998 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "honcho" -version = "3.0.0" +version = "3.0.1" description = "Honcho Server" authors = [ {name = "Plastic Labs", email = "hello@plasticlabs.ai"}, @@ -37,8 +37,7 @@ dependencies = [ "redis>=7.0.0,<8.0.0", "cashews[redis]==7.4.4", "scikit-learn>=1.6.0", - "opentelemetry-sdk>=1.36.0", - "opentelemetry-exporter-otlp-proto-http>=1.36.0", + "prometheus_client>=0.21.0", "cloudevents>=1.12.0", ] [dependency-groups] @@ -116,7 +115,7 @@ reportUnusedCallResult = false reportCallInDefaultInitializer = false reportAny = false reportExplicitAny = false -allowedUntypedLibraries = ["langfuse", "lancedb", "pyarrow", "opentelemetry"] +allowedUntypedLibraries = ["langfuse", "lancedb", "pyarrow"] reportImplicitOverride = false reportImportCycles = false diff --git a/sdks/python/src/honcho/aio.py b/sdks/python/src/honcho/aio.py index 20cb488b..1d72946a 100644 --- a/sdks/python/src/honcho/aio.py +++ b/sdks/python/src/honcho/aio.py @@ -372,7 +372,7 @@ class HonchoAio(AsyncMetadataConfigMixin): async def schedule_dream( self, observer: str | PeerBase, - session: str | SessionBase, + session: str | SessionBase | None = None, observed: str | PeerBase | None = None, ) -> None: """ @@ -385,7 +385,7 @@ class HonchoAio(AsyncMetadataConfigMixin): Args: observer: The observer peer (ID string or Peer object) whose perspective to use for the dream. - session: The session (ID string or Session object) to scope the dream to. + session: Optional session (ID string or Session object) to scope the dream to. observed: Optional observed peer (ID string or Peer object). If not provided, defaults to the observer (self-reflection). """ @@ -934,9 +934,9 @@ class SessionAio(AsyncMetadataConfigMixin): None, description="A peer ID to get context for.", ), - last_user_message: str | Message | None = Field( + search_query: str | Message | None = Field( None, - description="The most recent message text (string or Message object), used to fetch semantically relevant conclusions.", + description="A query string (or Message object) used to fetch semantically relevant conclusions.", ), peer_perspective: str | None = Field( None, @@ -976,15 +976,13 @@ class SessionAio(AsyncMetadataConfigMixin): "You must provide a `peer_target` when `peer_perspective` is provided" ) - if peer_target is None and last_user_message is not None: + if peer_target is None and search_query is not None: raise ValueError( - "You must provide a `peer_target` when `last_user_message` is provided" + "You must provide a `peer_target` when `search_query` is provided" ) - last_user_message_text = ( - last_user_message.content - if isinstance(last_user_message, Message) - else last_user_message + search_query_text = ( + search_query.content if isinstance(search_query, Message) else search_query ) query: dict[str, Any] = { @@ -993,8 +991,8 @@ class SessionAio(AsyncMetadataConfigMixin): } if tokens is not None: query["tokens"] = tokens - if last_user_message_text is not None: - query["last_message"] = last_user_message_text + if search_query_text is not None: + query["search_query"] = search_query_text if peer_target is not None: query["peer_target"] = peer_target if peer_perspective is not None: @@ -1319,19 +1317,28 @@ class ConclusionScopeAio: ) -> list[Conclusion]: """Create conclusions in this scope asynchronously.""" await self._scope._honcho._ensure_workspace_async() - conclusion_params = [ - { - "content": c.content - if isinstance(c, ConclusionCreateParams) - else c["content"], - "session_id": c.session_id - if isinstance(c, ConclusionCreateParams) - else c["session_id"], + + def build_conclusion_payload( + item: ConclusionCreateParams | dict[str, Any], + ) -> dict[str, Any]: + """Build a single conclusion create payload.""" + payload: dict[str, Any] = { "observer_id": self._scope.observer, "observed_id": self._scope.observed, } - for c in conclusions - ] + if isinstance(item, ConclusionCreateParams): + payload["content"] = item.content + if item.session_id is not None: + payload["session_id"] = item.session_id + return payload + + payload["content"] = item["content"] + session_id = item.get("session_id") + if session_id is not None: + payload["session_id"] = session_id + return payload + + conclusion_params = [build_conclusion_payload(c) for c in conclusions] data = await self._scope._honcho._async_http_client.post( routes.conclusions(self._scope.workspace_id), diff --git a/sdks/python/src/honcho/api_types.py b/sdks/python/src/honcho/api_types.py index c5858a40..47d8e4da 100644 --- a/sdks/python/src/honcho/api_types.py +++ b/sdks/python/src/honcho/api_types.py @@ -355,7 +355,7 @@ class ConclusionResponse(BaseModel): content: str observer_id: str observed_id: str - session_id: str + session_id: str | None = None created_at: datetime.datetime @@ -365,7 +365,7 @@ class ConclusionCreateParams(BaseModel): content: str = Field(min_length=1, max_length=65535) observer_id: str observed_id: str - session_id: str + session_id: str | None = None class ConclusionBatchCreateParams(BaseModel): diff --git a/sdks/python/src/honcho/client.py b/sdks/python/src/honcho/client.py index 35efc2c5..a40f12ee 100644 --- a/sdks/python/src/honcho/client.py +++ b/sdks/python/src/honcho/client.py @@ -561,7 +561,7 @@ class Honcho(BaseModel, MetadataConfigMixin): # pyright: ignore[reportUnsafeMul def schedule_dream( self, observer: str | PeerBase, - session: str | SessionBase, + session: str | SessionBase | None = None, observed: str | PeerBase | None = None, ) -> None: """ @@ -574,7 +574,7 @@ class Honcho(BaseModel, MetadataConfigMixin): # pyright: ignore[reportUnsafeMul Args: observer: The observer peer (ID string or Peer object) whose perspective to use for the dream. - session: The session (ID string or Session object) to scope the dream to. + session: Optional session (ID string or Session object) to scope the dream to. observed: Optional observed peer (ID string or Peer object). If not provided, defaults to the observer (self-reflection). """ diff --git a/sdks/python/src/honcho/conclusions.py b/sdks/python/src/honcho/conclusions.py index abba9ba8..56e8f4c6 100644 --- a/sdks/python/src/honcho/conclusions.py +++ b/sdks/python/src/honcho/conclusions.py @@ -27,7 +27,7 @@ __all__ = [ class ConclusionCreateParams(BaseModel): content: str - session_id: str + session_id: str | None = None class Conclusion: @@ -50,7 +50,7 @@ class Conclusion: content: str observer_id: str observed_id: str - session_id: str + session_id: str | None = None created_at: datetime.datetime def __init__( @@ -59,7 +59,7 @@ class Conclusion: content: str, observer_id: str, observed_id: str, - session_id: str, + session_id: str | None, created_at: datetime.datetime, ) -> None: self.id = id @@ -269,7 +269,8 @@ class ConclusionScope: Args: conclusions: List of conclusions to create. - Each conclusion can be a ConclusionCreateParams object or a dictionary with 'content' and 'session_id' keys. + Each conclusion can be a ConclusionCreateParams object or a dictionary + with a required 'content' key and an optional 'session_id' key. Returns: List of created Conclusion objects @@ -278,24 +279,51 @@ class ConclusionScope: ```python conclusions = peer.conclusions.create([ {"content": "User prefers dark mode", "session_id": "session1"}, - {"content": "User is interested in AI", "session_id": "session1"}, + {"content": "User is interested in AI"}, ]) ``` """ self._honcho._ensure_workspace() - conclusion_params = [ - { - "content": c.content - if isinstance(c, ConclusionCreateParams) - else c["content"], - "session_id": c.session_id - if isinstance(c, ConclusionCreateParams) - else c["session_id"], + + def build_conclusion_payload( + item: ConclusionCreateParams | dict[str, Any], + ) -> dict[str, Any]: + """ + Build a single conclusion create payload. + + This normalizes both `ConclusionCreateParams` instances and plain dictionaries + into the wire format expected by the Honcho API. + + Notes: + - `content` is required. + - `session_id` is optional; when not provided it is omitted from the payload. + - `observer_id` and `observed_id` are always injected from this scope. + + Args: + item: A `ConclusionCreateParams` instance or a dict with a required + `content` key and an optional `session_id` key. + + Returns: + A dictionary suitable for inclusion in the `conclusions` array for the + create conclusions endpoint. + """ + payload: dict[str, Any] = { "observer_id": self.observer, "observed_id": self.observed, } - for c in conclusions - ] + if isinstance(item, ConclusionCreateParams): + payload["content"] = item.content + if item.session_id is not None: + payload["session_id"] = item.session_id + return payload + + payload["content"] = item["content"] + session_id = item.get("session_id") + if session_id is not None: + payload["session_id"] = session_id + return payload + + conclusion_params = [build_conclusion_payload(c) for c in conclusions] data = self._honcho._http.post( routes.conclusions(self.workspace_id), diff --git a/sdks/python/src/honcho/session.py b/sdks/python/src/honcho/session.py index 6f1bb77b..8414cc60 100644 --- a/sdks/python/src/honcho/session.py +++ b/sdks/python/src/honcho/session.py @@ -517,9 +517,9 @@ class Session(SessionBase, MetadataConfigMixin): None, description="A peer ID to get context for. If given *without* `peer_perspective`, a representation and peer card will be included from the omniscient Honcho-level view of `peer_target`. If given *with* `peer_perspective`, will get the representation and card for `peer_target` *from the perspective of `peer_perspective`*.", ), - last_user_message: str | Message | None = Field( + search_query: str | Message | None = Field( None, - description="The most recent message text (string or Message object), used to fetch semantically relevant conclusions. Use this alongside `peer_target` to get a more focused context -- does nothing if `peer_target` is not provided.", + description="A query string (or Message object) used to fetch semantically relevant conclusions. Use this alongside `peer_target` to get a more focused context -- does nothing if `peer_target` is not provided.", ), peer_perspective: str | None = Field( None, @@ -533,13 +533,13 @@ class Session(SessionBase, MetadataConfigMixin): None, ge=1, le=100, - description="Number of semantically relevant facts to return when searching with `last_user_message`.", + description="Number of semantically relevant facts to return when searching with `search_query`.", ), search_max_distance: float | None = Field( None, ge=0.0, le=1.0, - description="Maximum semantic distance for search results (0.0-1.0) when searching with `last_user_message`.", + description="Maximum semantic distance for search results (0.0-1.0) when searching with `search_query`.", ), include_most_frequent: bool | None = Field( None, @@ -565,7 +565,7 @@ class Session(SessionBase, MetadataConfigMixin): tokens: Maximum number of tokens to include in the context. Will default to Honcho server configuration if not provided. peer_target: A peer ID to get context for. - last_user_message: The most recent message for semantic search. + search_query: A query string for semantic search. peer_perspective: A peer ID to get context from the perspective of. limit_to_session: Whether to limit the representation to this session only. search_top_k: Number of semantically relevant facts to return. @@ -589,15 +589,13 @@ class Session(SessionBase, MetadataConfigMixin): "You must provide a `peer_target` when `peer_perspective` is provided" ) - if peer_target is None and last_user_message is not None: + if peer_target is None and search_query is not None: raise ValueError( - "You must provide a `peer_target` when `last_user_message` is provided" + "You must provide a `peer_target` when `search_query` is provided" ) - last_user_message_text = ( - last_user_message.content - if isinstance(last_user_message, Message) - else last_user_message + search_query_text = ( + search_query.content if isinstance(search_query, Message) else search_query ) query: dict[str, Any] = { @@ -606,8 +604,8 @@ class Session(SessionBase, MetadataConfigMixin): } if tokens is not None: query["tokens"] = tokens - if last_user_message_text is not None: - query["last_message"] = last_user_message_text + if search_query_text is not None: + query["search_query"] = search_query_text if peer_target is not None: query["peer_target"] = peer_target if peer_perspective is not None: diff --git a/sdks/typescript/__tests__/conclusions.test.ts b/sdks/typescript/__tests__/conclusions.test.ts index 68051b13..8e7a6a09 100644 --- a/sdks/typescript/__tests__/conclusions.test.ts +++ b/sdks/typescript/__tests__/conclusions.test.ts @@ -423,4 +423,110 @@ describe('Conclusions', () => { expect(str).toContain(client.workspaceId) }) }) + + // =========================================================================== + // Sessionless Conclusions (Optional session_id) + // =========================================================================== + + describe('sessionless conclusions (optional session_id)', () => { + test('create conclusion without sessionId', async () => { + const peer = await client.peer('sessionless-create-peer', { metadata: {} }) + + // Create conclusion without sessionId + const conclusions = await peer.conclusions.create({ + content: 'Global conclusion without session', + // No sessionId - this is the key test + }) + + expect(conclusions.length).toBe(1) + expect(conclusions[0]).toBeInstanceOf(Conclusion) + expect(conclusions[0].content).toBe('Global conclusion without session') + expect(conclusions[0].sessionId).toBeNull() + }) + + test('create multiple conclusions without sessionId', async () => { + const peer = await client.peer('sessionless-multi-peer', { metadata: {} }) + + const conclusions = await peer.conclusions.create([ + { content: 'Global observation 1' }, + { content: 'Global observation 2' }, + { content: 'Global observation 3' }, + ]) + + expect(conclusions.length).toBe(3) + for (const c of conclusions) { + expect(c.sessionId).toBeNull() + } + }) + + test('create mixed session and sessionless conclusions', async () => { + const peer = await client.peer('sessionless-mixed-peer', { metadata: {} }) + const session = await client.session('sessionless-mixed-session', { + metadata: {}, + }) + + const conclusions = await peer.conclusions.create([ + { content: 'Session-scoped conclusion', sessionId: session }, + { content: 'Global conclusion without session' }, + ]) + + expect(conclusions.length).toBe(2) + + const sessionConclusion = conclusions.find( + (c) => c.content === 'Session-scoped conclusion' + ) + const globalConclusion = conclusions.find( + (c) => c.content === 'Global conclusion without session' + ) + + expect(sessionConclusion?.sessionId).toBe(session.id) + expect(globalConclusion?.sessionId).toBeNull() + }) + + test('list includes sessionless conclusions', async () => { + const peer = await client.peer('sessionless-list-peer', { metadata: {} }) + + // Create a sessionless conclusion + const [created] = await peer.conclusions.create({ + content: 'Sessionless conclusion for list test', + }) + + // List all conclusions (no session filter) + const page = await peer.conclusions.list() + + const found = page.items.find((c) => c.id === created.id) + expect(found).toBeDefined() + expect(found?.sessionId).toBeNull() + }) + + test('Conclusion.fromApiResponse handles null session_id', () => { + const response = { + id: 'test-id', + content: 'Test content', + observer_id: 'observer', + observed_id: 'observed', + session_id: null, + created_at: '2024-01-15T10:00:00Z', + } + + const conclusion = Conclusion.fromApiResponse(response) + + expect(conclusion.sessionId).toBeNull() + expect(conclusion.id).toBe('test-id') + expect(conclusion.content).toBe('Test content') + }) + + test('Conclusion constructor accepts null sessionId', () => { + const conclusion = new Conclusion( + 'id', + 'content', + 'observer', + 'observed', + null, + '2024-01-01' + ) + + expect(conclusion.sessionId).toBeNull() + }) + }) }) diff --git a/sdks/typescript/bun.lock b/sdks/typescript/bun.lock index 52d6a631..c67af981 100644 --- a/sdks/typescript/bun.lock +++ b/sdks/typescript/bun.lock @@ -5,90 +5,17 @@ "": { "name": "@honcho-ai/sdk", "dependencies": { - "@honcho-ai/core": "2.2.0", "@types/node": "^24.0.1", "zod": "4.0.0", }, "devDependencies": { "@biomejs/biome": "^2.1.2", - "@types/jest": "^29.5.14", - "jest": "^29.7.0", - "ts-jest": "^29.1.0", + "@types/bun": "latest", "typescript": "^5.0.0", }, }, }, "packages": { - "@babel/code-frame": ["@babel/code-frame@7.27.1", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg=="], - - "@babel/compat-data": ["@babel/compat-data@7.28.5", "", {}, "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA=="], - - "@babel/core": ["@babel/core@7.28.5", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", "@babel/helper-compilation-targets": "^7.27.2", "@babel/helper-module-transforms": "^7.28.3", "@babel/helpers": "^7.28.4", "@babel/parser": "^7.28.5", "@babel/template": "^7.27.2", "@babel/traverse": "^7.28.5", "@babel/types": "^7.28.5", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw=="], - - "@babel/generator": ["@babel/generator@7.28.5", "", { "dependencies": { "@babel/parser": "^7.28.5", "@babel/types": "^7.28.5", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ=="], - - "@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.27.2", "", { "dependencies": { "@babel/compat-data": "^7.27.2", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ=="], - - "@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="], - - "@babel/helper-module-imports": ["@babel/helper-module-imports@7.27.1", "", { "dependencies": { "@babel/traverse": "^7.27.1", "@babel/types": "^7.27.1" } }, "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w=="], - - "@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.28.3", "", { "dependencies": { "@babel/helper-module-imports": "^7.27.1", "@babel/helper-validator-identifier": "^7.27.1", "@babel/traverse": "^7.28.3" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw=="], - - "@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.27.1", "", {}, "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw=="], - - "@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="], - - "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="], - - "@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="], - - "@babel/helpers": ["@babel/helpers@7.28.4", "", { "dependencies": { "@babel/template": "^7.27.2", "@babel/types": "^7.28.4" } }, "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w=="], - - "@babel/parser": ["@babel/parser@7.28.5", "", { "dependencies": { "@babel/types": "^7.28.5" }, "bin": "./bin/babel-parser.js" }, "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ=="], - - "@babel/plugin-syntax-async-generators": ["@babel/plugin-syntax-async-generators@7.8.4", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw=="], - - "@babel/plugin-syntax-bigint": ["@babel/plugin-syntax-bigint@7.8.3", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg=="], - - "@babel/plugin-syntax-class-properties": ["@babel/plugin-syntax-class-properties@7.12.13", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.12.13" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA=="], - - "@babel/plugin-syntax-class-static-block": ["@babel/plugin-syntax-class-static-block@7.14.5", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw=="], - - "@babel/plugin-syntax-import-attributes": ["@babel/plugin-syntax-import-attributes@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww=="], - - "@babel/plugin-syntax-import-meta": ["@babel/plugin-syntax-import-meta@7.10.4", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g=="], - - "@babel/plugin-syntax-json-strings": ["@babel/plugin-syntax-json-strings@7.8.3", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA=="], - - "@babel/plugin-syntax-jsx": ["@babel/plugin-syntax-jsx@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w=="], - - "@babel/plugin-syntax-logical-assignment-operators": ["@babel/plugin-syntax-logical-assignment-operators@7.10.4", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig=="], - - "@babel/plugin-syntax-nullish-coalescing-operator": ["@babel/plugin-syntax-nullish-coalescing-operator@7.8.3", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ=="], - - "@babel/plugin-syntax-numeric-separator": ["@babel/plugin-syntax-numeric-separator@7.10.4", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug=="], - - "@babel/plugin-syntax-object-rest-spread": ["@babel/plugin-syntax-object-rest-spread@7.8.3", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA=="], - - "@babel/plugin-syntax-optional-catch-binding": ["@babel/plugin-syntax-optional-catch-binding@7.8.3", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q=="], - - "@babel/plugin-syntax-optional-chaining": ["@babel/plugin-syntax-optional-chaining@7.8.3", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg=="], - - "@babel/plugin-syntax-private-property-in-object": ["@babel/plugin-syntax-private-property-in-object@7.14.5", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg=="], - - "@babel/plugin-syntax-top-level-await": ["@babel/plugin-syntax-top-level-await@7.14.5", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw=="], - - "@babel/plugin-syntax-typescript": ["@babel/plugin-syntax-typescript@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ=="], - - "@babel/template": ["@babel/template@7.27.2", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/parser": "^7.27.2", "@babel/types": "^7.27.1" } }, "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw=="], - - "@babel/traverse": ["@babel/traverse@7.28.5", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.28.5", "@babel/template": "^7.27.2", "@babel/types": "^7.28.5", "debug": "^4.3.1" } }, "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ=="], - - "@babel/types": ["@babel/types@7.28.5", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA=="], - - "@bcoe/v8-coverage": ["@bcoe/v8-coverage@0.2.3", "", {}, "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw=="], - "@biomejs/biome": ["@biomejs/biome@2.3.8", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.8", "@biomejs/cli-darwin-x64": "2.3.8", "@biomejs/cli-linux-arm64": "2.3.8", "@biomejs/cli-linux-arm64-musl": "2.3.8", "@biomejs/cli-linux-x64": "2.3.8", "@biomejs/cli-linux-x64-musl": "2.3.8", "@biomejs/cli-win32-arm64": "2.3.8", "@biomejs/cli-win32-x64": "2.3.8" }, "bin": { "biome": "bin/biome" } }, "sha512-Qjsgoe6FEBxWAUzwFGFrB+1+M8y/y5kwmg5CHac+GSVOdmOIqsAiXM5QMVGZJ1eCUCLlPZtq4aFAQ0eawEUuUA=="], "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.3.8", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HM4Zg9CGQ3txTPflxD19n8MFPrmUAjaC7PQdLkugeeC0cQ+PiVrd7i09gaBS/11QKsTDBJhVg85CEIK9f50Qww=="], @@ -107,564 +34,16 @@ "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.3.8", "", { "os": "win32", "cpu": "x64" }, "sha512-RguzimPoZWtBapfKhKjcWXBVI91tiSprqdBYu7tWhgN8pKRZhw24rFeNZTNf6UiBfjCYCi9eFQs/JzJZIhuK4w=="], - "@honcho-ai/core": ["@honcho-ai/core@2.2.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-3lDBdruXlg2m8OEGe5mSD6pt4EgZulgvT0tbaM7k61Mi+m8H56aQNzlCv5yPLdw5G/FIEjyJJcFfHknqdxoyrg=="], - - "@istanbuljs/load-nyc-config": ["@istanbuljs/load-nyc-config@1.1.0", "", { "dependencies": { "camelcase": "^5.3.1", "find-up": "^4.1.0", "get-package-type": "^0.1.0", "js-yaml": "^3.13.1", "resolve-from": "^5.0.0" } }, "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ=="], - - "@istanbuljs/schema": ["@istanbuljs/schema@0.1.3", "", {}, "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA=="], - - "@jest/console": ["@jest/console@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0", "slash": "^3.0.0" } }, "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg=="], - - "@jest/core": ["@jest/core@29.7.0", "", { "dependencies": { "@jest/console": "^29.7.0", "@jest/reporters": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "ansi-escapes": "^4.2.1", "chalk": "^4.0.0", "ci-info": "^3.2.0", "exit": "^0.1.2", "graceful-fs": "^4.2.9", "jest-changed-files": "^29.7.0", "jest-config": "^29.7.0", "jest-haste-map": "^29.7.0", "jest-message-util": "^29.7.0", "jest-regex-util": "^29.6.3", "jest-resolve": "^29.7.0", "jest-resolve-dependencies": "^29.7.0", "jest-runner": "^29.7.0", "jest-runtime": "^29.7.0", "jest-snapshot": "^29.7.0", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "jest-watcher": "^29.7.0", "micromatch": "^4.0.4", "pretty-format": "^29.7.0", "slash": "^3.0.0", "strip-ansi": "^6.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "optionalPeers": ["node-notifier"] }, "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg=="], - - "@jest/environment": ["@jest/environment@29.7.0", "", { "dependencies": { "@jest/fake-timers": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "jest-mock": "^29.7.0" } }, "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw=="], - - "@jest/expect": ["@jest/expect@29.7.0", "", { "dependencies": { "expect": "^29.7.0", "jest-snapshot": "^29.7.0" } }, "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ=="], - - "@jest/expect-utils": ["@jest/expect-utils@29.7.0", "", { "dependencies": { "jest-get-type": "^29.6.3" } }, "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA=="], - - "@jest/fake-timers": ["@jest/fake-timers@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "@sinonjs/fake-timers": "^10.0.2", "@types/node": "*", "jest-message-util": "^29.7.0", "jest-mock": "^29.7.0", "jest-util": "^29.7.0" } }, "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ=="], - - "@jest/globals": ["@jest/globals@29.7.0", "", { "dependencies": { "@jest/environment": "^29.7.0", "@jest/expect": "^29.7.0", "@jest/types": "^29.6.3", "jest-mock": "^29.7.0" } }, "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ=="], - - "@jest/reporters": ["@jest/reporters@29.7.0", "", { "dependencies": { "@bcoe/v8-coverage": "^0.2.3", "@jest/console": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@jridgewell/trace-mapping": "^0.3.18", "@types/node": "*", "chalk": "^4.0.0", "collect-v8-coverage": "^1.0.0", "exit": "^0.1.2", "glob": "^7.1.3", "graceful-fs": "^4.2.9", "istanbul-lib-coverage": "^3.0.0", "istanbul-lib-instrument": "^6.0.0", "istanbul-lib-report": "^3.0.0", "istanbul-lib-source-maps": "^4.0.0", "istanbul-reports": "^3.1.3", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0", "jest-worker": "^29.7.0", "slash": "^3.0.0", "string-length": "^4.0.1", "strip-ansi": "^6.0.0", "v8-to-istanbul": "^9.0.1" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "optionalPeers": ["node-notifier"] }, "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg=="], - - "@jest/schemas": ["@jest/schemas@29.6.3", "", { "dependencies": { "@sinclair/typebox": "^0.27.8" } }, "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA=="], - - "@jest/source-map": ["@jest/source-map@29.6.3", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.18", "callsites": "^3.0.0", "graceful-fs": "^4.2.9" } }, "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw=="], - - "@jest/test-result": ["@jest/test-result@29.7.0", "", { "dependencies": { "@jest/console": "^29.7.0", "@jest/types": "^29.6.3", "@types/istanbul-lib-coverage": "^2.0.0", "collect-v8-coverage": "^1.0.0" } }, "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA=="], - - "@jest/test-sequencer": ["@jest/test-sequencer@29.7.0", "", { "dependencies": { "@jest/test-result": "^29.7.0", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "slash": "^3.0.0" } }, "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw=="], - - "@jest/transform": ["@jest/transform@29.7.0", "", { "dependencies": { "@babel/core": "^7.11.6", "@jest/types": "^29.6.3", "@jridgewell/trace-mapping": "^0.3.18", "babel-plugin-istanbul": "^6.1.1", "chalk": "^4.0.0", "convert-source-map": "^2.0.0", "fast-json-stable-stringify": "^2.1.0", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "jest-regex-util": "^29.6.3", "jest-util": "^29.7.0", "micromatch": "^4.0.4", "pirates": "^4.0.4", "slash": "^3.0.0", "write-file-atomic": "^4.0.2" } }, "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw=="], - - "@jest/types": ["@jest/types@29.6.3", "", { "dependencies": { "@jest/schemas": "^29.6.3", "@types/istanbul-lib-coverage": "^2.0.0", "@types/istanbul-reports": "^3.0.0", "@types/node": "*", "@types/yargs": "^17.0.8", "chalk": "^4.0.0" } }, "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw=="], - - "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], - - "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], - - "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], - - "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], - - "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], - - "@sinclair/typebox": ["@sinclair/typebox@0.27.8", "", {}, "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="], - - "@sinonjs/commons": ["@sinonjs/commons@3.0.1", "", { "dependencies": { "type-detect": "4.0.8" } }, "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ=="], - - "@sinonjs/fake-timers": ["@sinonjs/fake-timers@10.3.0", "", { "dependencies": { "@sinonjs/commons": "^3.0.0" } }, "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA=="], - - "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], - - "@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="], - - "@types/babel__template": ["@types/babel__template@7.4.4", "", { "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A=="], - - "@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="], - - "@types/graceful-fs": ["@types/graceful-fs@4.1.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ=="], - - "@types/istanbul-lib-coverage": ["@types/istanbul-lib-coverage@2.0.6", "", {}, "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w=="], - - "@types/istanbul-lib-report": ["@types/istanbul-lib-report@3.0.3", "", { "dependencies": { "@types/istanbul-lib-coverage": "*" } }, "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA=="], - - "@types/istanbul-reports": ["@types/istanbul-reports@3.0.4", "", { "dependencies": { "@types/istanbul-lib-report": "*" } }, "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ=="], - - "@types/jest": ["@types/jest@29.5.14", "", { "dependencies": { "expect": "^29.0.0", "pretty-format": "^29.0.0" } }, "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ=="], + "@types/bun": ["@types/bun@1.3.6", "", { "dependencies": { "bun-types": "1.3.6" } }, "sha512-uWCv6FO/8LcpREhenN1d1b6fcspAB+cefwD7uti8C8VffIv0Um08TKMn98FynpTiU38+y2dUO55T11NgDt8VAA=="], "@types/node": ["@types/node@24.10.1", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ=="], - "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], - - "@types/stack-utils": ["@types/stack-utils@2.0.3", "", {}, "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw=="], - - "@types/yargs": ["@types/yargs@17.0.35", "", { "dependencies": { "@types/yargs-parser": "*" } }, "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg=="], - - "@types/yargs-parser": ["@types/yargs-parser@21.0.3", "", {}, "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ=="], - - "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], - - "agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="], - - "ansi-escapes": ["ansi-escapes@4.3.2", "", { "dependencies": { "type-fest": "^0.21.3" } }, "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ=="], - - "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - - "ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="], - - "anymatch": ["anymatch@3.1.3", "", { "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" } }, "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw=="], - - "argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], - - "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], - - "babel-jest": ["babel-jest@29.7.0", "", { "dependencies": { "@jest/transform": "^29.7.0", "@types/babel__core": "^7.1.14", "babel-plugin-istanbul": "^6.1.1", "babel-preset-jest": "^29.6.3", "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "slash": "^3.0.0" }, "peerDependencies": { "@babel/core": "^7.8.0" } }, "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg=="], - - "babel-plugin-istanbul": ["babel-plugin-istanbul@6.1.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.0.0", "@istanbuljs/load-nyc-config": "^1.0.0", "@istanbuljs/schema": "^0.1.2", "istanbul-lib-instrument": "^5.0.4", "test-exclude": "^6.0.0" } }, "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA=="], - - "babel-plugin-jest-hoist": ["babel-plugin-jest-hoist@29.6.3", "", { "dependencies": { "@babel/template": "^7.3.3", "@babel/types": "^7.3.3", "@types/babel__core": "^7.1.14", "@types/babel__traverse": "^7.0.6" } }, "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg=="], - - "babel-preset-current-node-syntax": ["babel-preset-current-node-syntax@1.2.0", "", { "dependencies": { "@babel/plugin-syntax-async-generators": "^7.8.4", "@babel/plugin-syntax-bigint": "^7.8.3", "@babel/plugin-syntax-class-properties": "^7.12.13", "@babel/plugin-syntax-class-static-block": "^7.14.5", "@babel/plugin-syntax-import-attributes": "^7.24.7", "@babel/plugin-syntax-import-meta": "^7.10.4", "@babel/plugin-syntax-json-strings": "^7.8.3", "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", "@babel/plugin-syntax-numeric-separator": "^7.10.4", "@babel/plugin-syntax-object-rest-spread": "^7.8.3", "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", "@babel/plugin-syntax-optional-chaining": "^7.8.3", "@babel/plugin-syntax-private-property-in-object": "^7.14.5", "@babel/plugin-syntax-top-level-await": "^7.14.5" }, "peerDependencies": { "@babel/core": "^7.0.0 || ^8.0.0-0" } }, "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg=="], - - "babel-preset-jest": ["babel-preset-jest@29.6.3", "", { "dependencies": { "babel-plugin-jest-hoist": "^29.6.3", "babel-preset-current-node-syntax": "^1.0.0" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA=="], - - "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], - - "baseline-browser-mapping": ["baseline-browser-mapping@2.8.32", "", { "bin": { "baseline-browser-mapping": "dist/cli.js" } }, "sha512-OPz5aBThlyLFgxyhdwf/s2+8ab3OvT7AdTNvKHBwpXomIYeXqpUUuT8LrdtxZSsWJ4R4CU1un4XGh5Ez3nlTpw=="], - - "brace-expansion": ["brace-expansion@1.1.12", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg=="], - - "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - - "browserslist": ["browserslist@4.28.0", "", { "dependencies": { "baseline-browser-mapping": "^2.8.25", "caniuse-lite": "^1.0.30001754", "electron-to-chromium": "^1.5.249", "node-releases": "^2.0.27", "update-browserslist-db": "^1.1.4" }, "bin": { "browserslist": "cli.js" } }, "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ=="], - - "bs-logger": ["bs-logger@0.2.6", "", { "dependencies": { "fast-json-stable-stringify": "2.x" } }, "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog=="], - - "bser": ["bser@2.1.1", "", { "dependencies": { "node-int64": "^0.4.0" } }, "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ=="], - - "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], - - "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], - - "callsites": ["callsites@3.1.0", "", {}, "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="], - - "camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="], - - "caniuse-lite": ["caniuse-lite@1.0.30001757", "", {}, "sha512-r0nnL/I28Zi/yjk1el6ilj27tKcdjLsNqAOZr0yVjWPrSQyHgKI2INaEWw21bAQSv2LXRt1XuCS/GomNpWOxsQ=="], - - "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], - - "char-regex": ["char-regex@1.0.2", "", {}, "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw=="], - - "ci-info": ["ci-info@3.9.0", "", {}, "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ=="], - - "cjs-module-lexer": ["cjs-module-lexer@1.4.3", "", {}, "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q=="], - - "cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], - - "co": ["co@4.6.0", "", {}, "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ=="], - - "collect-v8-coverage": ["collect-v8-coverage@1.0.3", "", {}, "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw=="], - - "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], - - "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], - - "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], - - "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], - - "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], - - "create-jest": ["create-jest@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "chalk": "^4.0.0", "exit": "^0.1.2", "graceful-fs": "^4.2.9", "jest-config": "^29.7.0", "jest-util": "^29.7.0", "prompts": "^2.0.1" }, "bin": { "create-jest": "bin/create-jest.js" } }, "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q=="], - - "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], - - "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - - "dedent": ["dedent@1.7.0", "", { "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, "optionalPeers": ["babel-plugin-macros"] }, "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ=="], - - "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], - - "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], - - "detect-newline": ["detect-newline@3.1.0", "", {}, "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA=="], - - "diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="], - - "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], - - "electron-to-chromium": ["electron-to-chromium@1.5.263", "", {}, "sha512-DrqJ11Knd+lo+dv+lltvfMDLU27g14LMdH2b0O3Pio4uk0x+z7OR+JrmyacTPN2M8w3BrZ7/RTwG3R9B7irPlg=="], - - "emittery": ["emittery@0.13.1", "", {}, "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ=="], - - "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], - - "error-ex": ["error-ex@1.3.4", "", { "dependencies": { "is-arrayish": "^0.2.1" } }, "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ=="], - - "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], - - "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], - - "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], - - "es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="], - - "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], - - "escape-string-regexp": ["escape-string-regexp@2.0.0", "", {}, "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w=="], - - "esprima": ["esprima@4.0.1", "", { "bin": { "esparse": "./bin/esparse.js", "esvalidate": "./bin/esvalidate.js" } }, "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="], - - "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="], - - "execa": ["execa@5.1.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", "human-signals": "^2.1.0", "is-stream": "^2.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^4.0.1", "onetime": "^5.1.2", "signal-exit": "^3.0.3", "strip-final-newline": "^2.0.0" } }, "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg=="], - - "exit": ["exit@0.1.2", "", {}, "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ=="], - - "expect": ["expect@29.7.0", "", { "dependencies": { "@jest/expect-utils": "^29.7.0", "jest-get-type": "^29.6.3", "jest-matcher-utils": "^29.7.0", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0" } }, "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw=="], - - "fast-json-stable-stringify": ["fast-json-stable-stringify@2.1.0", "", {}, "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="], - - "fb-watchman": ["fb-watchman@2.0.2", "", { "dependencies": { "bser": "2.1.1" } }, "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA=="], - - "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], - - "find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="], - - "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], - - "form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="], - - "formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="], - - "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], - - "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], - - "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], - - "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], - - "get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="], - - "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], - - "get-package-type": ["get-package-type@0.1.0", "", {}, "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q=="], - - "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], - - "get-stream": ["get-stream@6.0.1", "", {}, "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg=="], - - "glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], - - "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], - - "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - - "handlebars": ["handlebars@4.7.8", "", { "dependencies": { "minimist": "^1.2.5", "neo-async": "^2.6.2", "source-map": "^0.6.1", "wordwrap": "^1.0.0" }, "optionalDependencies": { "uglify-js": "^3.1.4" }, "bin": { "handlebars": "bin/handlebars" } }, "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ=="], - - "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], - - "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], - - "has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="], - - "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], - - "html-escaper": ["html-escaper@2.0.2", "", {}, "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg=="], - - "human-signals": ["human-signals@2.1.0", "", {}, "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw=="], - - "humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="], - - "import-local": ["import-local@3.2.0", "", { "dependencies": { "pkg-dir": "^4.2.0", "resolve-cwd": "^3.0.0" }, "bin": { "import-local-fixture": "fixtures/cli.js" } }, "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA=="], - - "imurmurhash": ["imurmurhash@0.1.4", "", {}, "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA=="], - - "inflight": ["inflight@1.0.6", "", { "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA=="], - - "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], - - "is-arrayish": ["is-arrayish@0.2.1", "", {}, "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg=="], - - "is-core-module": ["is-core-module@2.16.1", "", { "dependencies": { "hasown": "^2.0.2" } }, "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w=="], - - "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], - - "is-generator-fn": ["is-generator-fn@2.1.0", "", {}, "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ=="], - - "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], - - "is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], - - "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - - "istanbul-lib-coverage": ["istanbul-lib-coverage@3.2.2", "", {}, "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg=="], - - "istanbul-lib-instrument": ["istanbul-lib-instrument@6.0.3", "", { "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", "@istanbuljs/schema": "^0.1.3", "istanbul-lib-coverage": "^3.2.0", "semver": "^7.5.4" } }, "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q=="], - - "istanbul-lib-report": ["istanbul-lib-report@3.0.1", "", { "dependencies": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^4.0.0", "supports-color": "^7.1.0" } }, "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw=="], - - "istanbul-lib-source-maps": ["istanbul-lib-source-maps@4.0.1", "", { "dependencies": { "debug": "^4.1.1", "istanbul-lib-coverage": "^3.0.0", "source-map": "^0.6.1" } }, "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw=="], - - "istanbul-reports": ["istanbul-reports@3.2.0", "", { "dependencies": { "html-escaper": "^2.0.0", "istanbul-lib-report": "^3.0.0" } }, "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA=="], - - "jest": ["jest@29.7.0", "", { "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", "import-local": "^3.0.2", "jest-cli": "^29.7.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "optionalPeers": ["node-notifier"], "bin": { "jest": "bin/jest.js" } }, "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw=="], - - "jest-changed-files": ["jest-changed-files@29.7.0", "", { "dependencies": { "execa": "^5.0.0", "jest-util": "^29.7.0", "p-limit": "^3.1.0" } }, "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w=="], - - "jest-circus": ["jest-circus@29.7.0", "", { "dependencies": { "@jest/environment": "^29.7.0", "@jest/expect": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "co": "^4.6.0", "dedent": "^1.0.0", "is-generator-fn": "^2.0.0", "jest-each": "^29.7.0", "jest-matcher-utils": "^29.7.0", "jest-message-util": "^29.7.0", "jest-runtime": "^29.7.0", "jest-snapshot": "^29.7.0", "jest-util": "^29.7.0", "p-limit": "^3.1.0", "pretty-format": "^29.7.0", "pure-rand": "^6.0.0", "slash": "^3.0.0", "stack-utils": "^2.0.3" } }, "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw=="], - - "jest-cli": ["jest-cli@29.7.0", "", { "dependencies": { "@jest/core": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/types": "^29.6.3", "chalk": "^4.0.0", "create-jest": "^29.7.0", "exit": "^0.1.2", "import-local": "^3.0.2", "jest-config": "^29.7.0", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "yargs": "^17.3.1" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "optionalPeers": ["node-notifier"], "bin": { "jest": "bin/jest.js" } }, "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg=="], - - "jest-config": ["jest-config@29.7.0", "", { "dependencies": { "@babel/core": "^7.11.6", "@jest/test-sequencer": "^29.7.0", "@jest/types": "^29.6.3", "babel-jest": "^29.7.0", "chalk": "^4.0.0", "ci-info": "^3.2.0", "deepmerge": "^4.2.2", "glob": "^7.1.3", "graceful-fs": "^4.2.9", "jest-circus": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-get-type": "^29.6.3", "jest-regex-util": "^29.6.3", "jest-resolve": "^29.7.0", "jest-runner": "^29.7.0", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "micromatch": "^4.0.4", "parse-json": "^5.2.0", "pretty-format": "^29.7.0", "slash": "^3.0.0", "strip-json-comments": "^3.1.1" }, "peerDependencies": { "@types/node": "*", "ts-node": ">=9.0.0" }, "optionalPeers": ["@types/node", "ts-node"] }, "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ=="], - - "jest-diff": ["jest-diff@29.7.0", "", { "dependencies": { "chalk": "^4.0.0", "diff-sequences": "^29.6.3", "jest-get-type": "^29.6.3", "pretty-format": "^29.7.0" } }, "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw=="], - - "jest-docblock": ["jest-docblock@29.7.0", "", { "dependencies": { "detect-newline": "^3.0.0" } }, "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g=="], - - "jest-each": ["jest-each@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "chalk": "^4.0.0", "jest-get-type": "^29.6.3", "jest-util": "^29.7.0", "pretty-format": "^29.7.0" } }, "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ=="], - - "jest-environment-node": ["jest-environment-node@29.7.0", "", { "dependencies": { "@jest/environment": "^29.7.0", "@jest/fake-timers": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "jest-mock": "^29.7.0", "jest-util": "^29.7.0" } }, "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw=="], - - "jest-get-type": ["jest-get-type@29.6.3", "", {}, "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw=="], - - "jest-haste-map": ["jest-haste-map@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "@types/graceful-fs": "^4.1.3", "@types/node": "*", "anymatch": "^3.0.3", "fb-watchman": "^2.0.0", "graceful-fs": "^4.2.9", "jest-regex-util": "^29.6.3", "jest-util": "^29.7.0", "jest-worker": "^29.7.0", "micromatch": "^4.0.4", "walker": "^1.0.8" }, "optionalDependencies": { "fsevents": "^2.3.2" } }, "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA=="], - - "jest-leak-detector": ["jest-leak-detector@29.7.0", "", { "dependencies": { "jest-get-type": "^29.6.3", "pretty-format": "^29.7.0" } }, "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw=="], - - "jest-matcher-utils": ["jest-matcher-utils@29.7.0", "", { "dependencies": { "chalk": "^4.0.0", "jest-diff": "^29.7.0", "jest-get-type": "^29.6.3", "pretty-format": "^29.7.0" } }, "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g=="], - - "jest-message-util": ["jest-message-util@29.7.0", "", { "dependencies": { "@babel/code-frame": "^7.12.13", "@jest/types": "^29.6.3", "@types/stack-utils": "^2.0.0", "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "micromatch": "^4.0.4", "pretty-format": "^29.7.0", "slash": "^3.0.0", "stack-utils": "^2.0.3" } }, "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w=="], - - "jest-mock": ["jest-mock@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "@types/node": "*", "jest-util": "^29.7.0" } }, "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw=="], - - "jest-pnp-resolver": ["jest-pnp-resolver@1.2.3", "", { "peerDependencies": { "jest-resolve": "*" }, "optionalPeers": ["jest-resolve"] }, "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w=="], - - "jest-regex-util": ["jest-regex-util@29.6.3", "", {}, "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg=="], - - "jest-resolve": ["jest-resolve@29.7.0", "", { "dependencies": { "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "jest-pnp-resolver": "^1.2.2", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "resolve": "^1.20.0", "resolve.exports": "^2.0.0", "slash": "^3.0.0" } }, "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA=="], - - "jest-resolve-dependencies": ["jest-resolve-dependencies@29.7.0", "", { "dependencies": { "jest-regex-util": "^29.6.3", "jest-snapshot": "^29.7.0" } }, "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA=="], - - "jest-runner": ["jest-runner@29.7.0", "", { "dependencies": { "@jest/console": "^29.7.0", "@jest/environment": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "emittery": "^0.13.1", "graceful-fs": "^4.2.9", "jest-docblock": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-haste-map": "^29.7.0", "jest-leak-detector": "^29.7.0", "jest-message-util": "^29.7.0", "jest-resolve": "^29.7.0", "jest-runtime": "^29.7.0", "jest-util": "^29.7.0", "jest-watcher": "^29.7.0", "jest-worker": "^29.7.0", "p-limit": "^3.1.0", "source-map-support": "0.5.13" } }, "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ=="], - - "jest-runtime": ["jest-runtime@29.7.0", "", { "dependencies": { "@jest/environment": "^29.7.0", "@jest/fake-timers": "^29.7.0", "@jest/globals": "^29.7.0", "@jest/source-map": "^29.6.3", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "cjs-module-lexer": "^1.0.0", "collect-v8-coverage": "^1.0.0", "glob": "^7.1.3", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "jest-message-util": "^29.7.0", "jest-mock": "^29.7.0", "jest-regex-util": "^29.6.3", "jest-resolve": "^29.7.0", "jest-snapshot": "^29.7.0", "jest-util": "^29.7.0", "slash": "^3.0.0", "strip-bom": "^4.0.0" } }, "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ=="], - - "jest-snapshot": ["jest-snapshot@29.7.0", "", { "dependencies": { "@babel/core": "^7.11.6", "@babel/generator": "^7.7.2", "@babel/plugin-syntax-jsx": "^7.7.2", "@babel/plugin-syntax-typescript": "^7.7.2", "@babel/types": "^7.3.3", "@jest/expect-utils": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "babel-preset-current-node-syntax": "^1.0.0", "chalk": "^4.0.0", "expect": "^29.7.0", "graceful-fs": "^4.2.9", "jest-diff": "^29.7.0", "jest-get-type": "^29.6.3", "jest-matcher-utils": "^29.7.0", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0", "natural-compare": "^1.4.0", "pretty-format": "^29.7.0", "semver": "^7.5.3" } }, "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw=="], - - "jest-util": ["jest-util@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "ci-info": "^3.2.0", "graceful-fs": "^4.2.9", "picomatch": "^2.2.3" } }, "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA=="], - - "jest-validate": ["jest-validate@29.7.0", "", { "dependencies": { "@jest/types": "^29.6.3", "camelcase": "^6.2.0", "chalk": "^4.0.0", "jest-get-type": "^29.6.3", "leven": "^3.1.0", "pretty-format": "^29.7.0" } }, "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw=="], - - "jest-watcher": ["jest-watcher@29.7.0", "", { "dependencies": { "@jest/test-result": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "ansi-escapes": "^4.2.1", "chalk": "^4.0.0", "emittery": "^0.13.1", "jest-util": "^29.7.0", "string-length": "^4.0.1" } }, "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g=="], - - "jest-worker": ["jest-worker@29.7.0", "", { "dependencies": { "@types/node": "*", "jest-util": "^29.7.0", "merge-stream": "^2.0.0", "supports-color": "^8.0.0" } }, "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw=="], - - "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - - "js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], - - "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], - - "json-parse-even-better-errors": ["json-parse-even-better-errors@2.3.1", "", {}, "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="], - - "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], - - "kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="], - - "leven": ["leven@3.1.0", "", {}, "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A=="], - - "lines-and-columns": ["lines-and-columns@1.2.4", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="], - - "locate-path": ["locate-path@5.0.0", "", { "dependencies": { "p-locate": "^4.1.0" } }, "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g=="], - - "lodash.memoize": ["lodash.memoize@4.1.2", "", {}, "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag=="], - - "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], - - "make-dir": ["make-dir@4.0.0", "", { "dependencies": { "semver": "^7.5.3" } }, "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw=="], - - "make-error": ["make-error@1.3.6", "", {}, "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw=="], - - "makeerror": ["makeerror@1.0.12", "", { "dependencies": { "tmpl": "1.0.5" } }, "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg=="], - - "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], - - "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], - - "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], - - "mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], - - "mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - - "mimic-fn": ["mimic-fn@2.1.0", "", {}, "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="], - - "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], - - "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], - - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - - "natural-compare": ["natural-compare@1.4.0", "", {}, "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="], - - "neo-async": ["neo-async@2.6.2", "", {}, "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="], - - "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], - - "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], - - "node-int64": ["node-int64@0.4.0", "", {}, "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw=="], - - "node-releases": ["node-releases@2.0.27", "", {}, "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA=="], - - "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], - - "npm-run-path": ["npm-run-path@4.0.1", "", { "dependencies": { "path-key": "^3.0.0" } }, "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw=="], - - "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], - - "onetime": ["onetime@5.1.2", "", { "dependencies": { "mimic-fn": "^2.1.0" } }, "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg=="], - - "p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], - - "p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="], - - "p-try": ["p-try@2.2.0", "", {}, "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="], - - "parse-json": ["parse-json@5.2.0", "", { "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", "json-parse-even-better-errors": "^2.3.0", "lines-and-columns": "^1.1.6" } }, "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg=="], - - "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], - - "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], - - "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], - - "path-parse": ["path-parse@1.0.7", "", {}, "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="], - - "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - - "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], - - "pirates": ["pirates@4.0.7", "", {}, "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA=="], - - "pkg-dir": ["pkg-dir@4.2.0", "", { "dependencies": { "find-up": "^4.0.0" } }, "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ=="], - - "pretty-format": ["pretty-format@29.7.0", "", { "dependencies": { "@jest/schemas": "^29.6.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" } }, "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ=="], - - "prompts": ["prompts@2.4.2", "", { "dependencies": { "kleur": "^3.0.3", "sisteransi": "^1.0.5" } }, "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q=="], - - "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], - - "react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - - "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], - - "resolve": ["resolve@1.22.11", "", { "dependencies": { "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" } }, "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ=="], - - "resolve-cwd": ["resolve-cwd@3.0.0", "", { "dependencies": { "resolve-from": "^5.0.0" } }, "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg=="], - - "resolve-from": ["resolve-from@5.0.0", "", {}, "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw=="], - - "resolve.exports": ["resolve.exports@2.0.3", "", {}, "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A=="], - - "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - - "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], - - "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - - "signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], - - "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], - - "slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], - - "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], - - "source-map-support": ["source-map-support@0.5.13", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w=="], - - "sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="], - - "stack-utils": ["stack-utils@2.0.6", "", { "dependencies": { "escape-string-regexp": "^2.0.0" } }, "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ=="], - - "string-length": ["string-length@4.0.2", "", { "dependencies": { "char-regex": "^1.0.2", "strip-ansi": "^6.0.0" } }, "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ=="], - - "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], - - "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - - "strip-bom": ["strip-bom@4.0.0", "", {}, "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w=="], - - "strip-final-newline": ["strip-final-newline@2.0.0", "", {}, "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="], - - "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], - - "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - - "supports-preserve-symlinks-flag": ["supports-preserve-symlinks-flag@1.0.0", "", {}, "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w=="], - - "test-exclude": ["test-exclude@6.0.0", "", { "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", "minimatch": "^3.0.4" } }, "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w=="], - - "tmpl": ["tmpl@1.0.5", "", {}, "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw=="], - - "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], - - "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], - - "ts-jest": ["ts-jest@29.4.6", "", { "dependencies": { "bs-logger": "^0.2.6", "fast-json-stable-stringify": "^2.1.0", "handlebars": "^4.7.8", "json5": "^2.2.3", "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", "semver": "^7.7.3", "type-fest": "^4.41.0", "yargs-parser": "^21.1.1" }, "peerDependencies": { "@babel/core": ">=7.0.0-beta.0 <8", "@jest/transform": "^29.0.0 || ^30.0.0", "@jest/types": "^29.0.0 || ^30.0.0", "babel-jest": "^29.0.0 || ^30.0.0", "jest": "^29.0.0 || ^30.0.0", "jest-util": "^29.0.0 || ^30.0.0", "typescript": ">=4.3 <6" }, "optionalPeers": ["@babel/core", "@jest/transform", "@jest/types", "babel-jest", "jest-util"], "bin": { "ts-jest": "cli.js" } }, "sha512-fSpWtOO/1AjSNQguk43hb/JCo16oJDnMJf3CdEGNkqsEX3t0KX96xvyX1D7PfLCpVoKu4MfVrqUkFyblYoY4lA=="], - - "type-detect": ["type-detect@4.0.8", "", {}, "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g=="], - - "type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="], + "bun-types": ["bun-types@1.3.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-OlFwHcnNV99r//9v5IIOgQ9Uk37gZqrNMCcqEaExdkVq3Avwqok1bJFmvGMCkCE0FqzdY8VMOZpfpR3lwI+CsQ=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - "uglify-js": ["uglify-js@3.19.3", "", { "bin": { "uglifyjs": "bin/uglifyjs" } }, "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ=="], - "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], - "update-browserslist-db": ["update-browserslist-db@1.1.4", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A=="], - - "v8-to-istanbul": ["v8-to-istanbul@9.3.0", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", "convert-source-map": "^2.0.0" } }, "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA=="], - - "walker": ["walker@1.0.8", "", { "dependencies": { "makeerror": "1.0.12" } }, "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ=="], - - "web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="], - - "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], - - "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], - - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], - - "wordwrap": ["wordwrap@1.0.0", "", {}, "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q=="], - - "wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], - - "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], - - "write-file-atomic": ["write-file-atomic@4.0.2", "", { "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^3.0.7" } }, "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg=="], - - "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], - - "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], - - "yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], - - "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], - - "yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], - "zod": ["zod@4.0.0", "", {}, "sha512-9diLdTPc/L7w/5jI4C3gHYNiGHDV9IZYxo1e5LSD8cabi65WVTWWb+g2BGPEpUUCOxR4D+6O5B0AzyMdUAXwrw=="], - - "@babel/core/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], - - "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], - - "@honcho-ai/core/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], - - "@istanbuljs/load-nyc-config/camelcase": ["camelcase@5.3.1", "", {}, "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="], - - "ansi-escapes/type-fest": ["type-fest@0.21.3", "", {}, "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="], - - "babel-plugin-istanbul/istanbul-lib-instrument": ["istanbul-lib-instrument@5.2.1", "", { "dependencies": { "@babel/core": "^7.12.3", "@babel/parser": "^7.14.7", "@istanbuljs/schema": "^0.1.2", "istanbul-lib-coverage": "^3.2.0", "semver": "^6.3.0" } }, "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg=="], - - "chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], - - "jest-worker/supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], - - "p-locate/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], - - "wrap-ansi/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], - - "@honcho-ai/core/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], - - "babel-plugin-istanbul/istanbul-lib-instrument/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], } } diff --git a/sdks/typescript/src/client.ts b/sdks/typescript/src/client.ts index bc818146..53801d9d 100644 --- a/sdks/typescript/src/client.ts +++ b/sdks/typescript/src/client.ts @@ -775,12 +775,12 @@ export class Honcho { */ async scheduleDream(options: { observer: string | Peer - session: string | Session + session?: string | Session observed?: string | Peer }): Promise { await this._ensureWorkspace() const observerId = resolveId(options.observer) - const sessionId = resolveId(options.session) + const sessionId = options.session ? resolveId(options.session) : undefined const observedId = options.observed ? resolveId(options.observed) : observerId diff --git a/sdks/typescript/src/conclusions.ts b/sdks/typescript/src/conclusions.ts index dda19787..28cd06e9 100644 --- a/sdks/typescript/src/conclusions.ts +++ b/sdks/typescript/src/conclusions.ts @@ -16,7 +16,7 @@ export interface ConclusionCreateParams { /** The conclusion content/text */ content: string /** The session this conclusion relates to (ID string or Session object) */ - sessionId: string | Session + sessionId?: string | Session } /** @@ -30,7 +30,7 @@ export class Conclusion { readonly content: string readonly observerId: string readonly observedId: string - readonly sessionId: string + readonly sessionId: string | null readonly createdAt: string constructor( @@ -38,7 +38,7 @@ export class Conclusion { content: string, observerId: string, observedId: string, - sessionId: string, + sessionId: string | null, createdAt: string ) { this.id = id @@ -128,7 +128,7 @@ export class ConclusionScope { private async _create(params: { conclusions: Array<{ content: string - session_id: string + session_id: string | null observer_id: string observed_id: string }> @@ -259,7 +259,11 @@ export class ConclusionScope { const requestConclusions = conclusionArray.map((obs) => ({ content: obs.content, session_id: - typeof obs.sessionId === 'string' ? obs.sessionId : obs.sessionId.id, + obs.sessionId === undefined + ? null + : typeof obs.sessionId === 'string' + ? obs.sessionId + : obs.sessionId.id, observer_id: this.observer, observed_id: this.observed, })) diff --git a/sdks/typescript/src/session.ts b/sdks/typescript/src/session.ts index e8f12f05..6d257f71 100644 --- a/sdks/typescript/src/session.ts +++ b/sdks/typescript/src/session.ts @@ -186,7 +186,7 @@ export class Session { private async _getContext(params: { tokens?: number summary?: boolean - last_message?: string + search_query?: string peer_target?: string peer_perspective?: string limit_to_session?: boolean @@ -699,7 +699,7 @@ export class Session { summary?: boolean tokens?: number peerTarget?: string | Peer - lastUserMessage?: string | Message + searchQuery?: string | Message peerPerspective?: string | Peer limitToSession?: boolean representationOptions?: RepresentationOptions @@ -713,30 +713,30 @@ export class Session { typeof opts.peerPerspective === 'object' ? opts.peerPerspective.id : opts.peerPerspective - const lastUserMessageText = - typeof opts.lastUserMessage === 'string' - ? opts.lastUserMessage - : opts.lastUserMessage?.content + const searchQueryText = + typeof opts.searchQuery === 'string' + ? opts.searchQuery + : opts.searchQuery?.content const contextParams = ContextParamsSchema.parse({ summary: opts.summary, tokens: opts.tokens, peerTarget: peerTargetId, - lastUserMessage: lastUserMessageText, + searchQuery: searchQueryText, peerPerspective: peerPerspectiveId, limitToSession: opts.limitToSession, representationOptions: opts.representationOptions, }) - const lastMessageText = - typeof contextParams.lastUserMessage === 'string' - ? contextParams.lastUserMessage - : contextParams.lastUserMessage?.content + const searchQueryParsed = + typeof contextParams.searchQuery === 'string' + ? contextParams.searchQuery + : contextParams.searchQuery?.content const context = await this._getContext({ tokens: contextParams.tokens, summary: contextParams.summary, - last_message: lastMessageText, + search_query: searchQueryParsed, peer_target: contextParams.peerTarget, peer_perspective: contextParams.peerPerspective, limit_to_session: contextParams.limitToSession, diff --git a/sdks/typescript/src/types/api.ts b/sdks/typescript/src/types/api.ts index 6e1a53b3..9c71d558 100644 --- a/sdks/typescript/src/types/api.ts +++ b/sdks/typescript/src/types/api.ts @@ -155,7 +155,7 @@ export interface SessionPeerConfigParams { export interface SessionContextParams { tokens?: number summary?: boolean - last_message?: string + search_query?: string peer_target?: string peer_perspective?: string limit_to_session?: boolean @@ -243,7 +243,7 @@ export interface ConclusionResponse { content: string observer_id: string observed_id: string - session_id: string + session_id: string | null created_at: string } @@ -251,7 +251,7 @@ export interface ConclusionCreateParams { content: string observer_id: string observed_id: string - session_id: string + session_id: string | null } export interface ConclusionBatchCreateParams { diff --git a/sdks/typescript/src/validation.ts b/sdks/typescript/src/validation.ts index 073f8f5c..3a5e8584 100644 --- a/sdks/typescript/src/validation.ts +++ b/sdks/typescript/src/validation.ts @@ -271,9 +271,9 @@ export const ContextParamsSchema = z .object({ summary: z.boolean().optional(), tokens: z.int('Token limit must be an integer').optional(), - lastUserMessage: z + searchQuery: z .union([ - z.string().min(1, 'Last user message must be a non-empty string'), + z.string().min(1, 'Search query must be a non-empty string'), MessageResponseSchema, ]) .optional(), @@ -283,11 +283,11 @@ export const ContextParamsSchema = z representationOptions: RepresentationOptionsSchema.optional(), }) .superRefine((data, ctx) => { - if (data.lastUserMessage && !data.peerTarget) { + if (data.searchQuery && !data.peerTarget) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: 'peerTarget is required when lastUserMessage is provided', - path: ['lastUserMessage'], + message: 'peerTarget is required when searchQuery is provided', + path: ['searchQuery'], }) } diff --git a/src/cache/client.py b/src/cache/client.py index 0381dd4a..0b8480ca 100644 --- a/src/cache/client.py +++ b/src/cache/client.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio import logging from typing import cast +from urllib.parse import urlparse, urlunparse +import redis.asyncio as aioredis import sentry_sdk from cashews import cache from cashews.picklers import PicklerType @@ -126,9 +128,11 @@ async def is_deriver_flush_enabled() -> bool: if not is_cache_enabled(): return False try: - import redis.asyncio as aioredis + # Strip query parameters - redis-py doesn't support custom params like ?suppress=true + parsed = urlparse(settings.CACHE.URL) + clean_url = urlunparse(parsed._replace(query="")) - redis_client = aioredis.from_url(settings.CACHE.URL) # pyright: ignore[reportUnknownMemberType] + redis_client = aioredis.from_url(clean_url) # pyright: ignore[reportUnknownMemberType] try: result = await redis_client.get(DERIVER_FLUSH_KEY) return result == b"1" diff --git a/src/config.py b/src/config.py index c17a35cc..de09422a 100644 --- a/src/config.py +++ b/src/config.py @@ -68,7 +68,7 @@ class TomlConfigSettingsSource(PydanticBaseSettingsSource): "WEBHOOK": "webhook", "DREAM": "dream", "VECTOR_STORE": "vector_store", - "OTEL": "otel", + "METRICS": "metrics", "TELEMETRY": "telemetry", "": "app", # For AppSettings with no prefix } @@ -537,33 +537,10 @@ class WebhookSettings(HonchoSettings): MAX_WORKSPACE_LIMIT: int = 10 -class OpenTelemetrySettings(HonchoSettings): - """OpenTelemetry settings for push-based metrics via OTLP. - - These settings configure the OTel SDK to push metrics via OTLP HTTP - to any compatible backend (Mimir, Grafana Cloud, etc.). - """ - - model_config = SettingsConfigDict(env_prefix="OTEL_", extra="ignore") # pyright: ignore - - # Master toggle for OTel metrics +class MetricsSettings(HonchoSettings): + model_config = SettingsConfigDict(env_prefix="METRICS_", extra="ignore") # pyright: ignore ENABLED: bool = False - - # OTLP HTTP endpoint for metrics (e.g., "https://mimir.example.com/otlp/v1/metrics") - # For Mimir, the endpoint is typically: /otlp/v1/metrics - # For Grafana Cloud: https://otlp-gateway-.grafana.net/otlp/v1/metrics - ENDPOINT: str | None = None - - HEADERS: dict[str, str] | None = None - - # Export interval in milliseconds (default: 60 seconds) - EXPORT_INTERVAL_MILLIS: int = 60000 - - # Service name for resource attributes (identifies what this service is) - SERVICE_NAME: str = "honcho" - - # Service namespace for resource attributes (defaults to top-level NAMESPACE if not set) - SERVICE_NAMESPACE: str | None = None + NAMESPACE: str | None = None class TelemetrySettings(HonchoSettings): @@ -768,7 +745,7 @@ class AppSettings(HonchoSettings): PEER_CARD: PeerCardSettings = Field(default_factory=PeerCardSettings) SUMMARY: SummarySettings = Field(default_factory=SummarySettings) WEBHOOK: WebhookSettings = Field(default_factory=WebhookSettings) - OTEL: OpenTelemetrySettings = Field(default_factory=OpenTelemetrySettings) + METRICS: MetricsSettings = Field(default_factory=MetricsSettings) TELEMETRY: TelemetrySettings = Field(default_factory=TelemetrySettings) CACHE: CacheSettings = Field(default_factory=CacheSettings) DREAM: DreamSettings = Field(default_factory=DreamSettings) @@ -783,20 +760,15 @@ class AppSettings(HonchoSettings): @model_validator(mode="after") def propagate_namespace(self) -> "AppSettings": - """Propagate top-level NAMESPACE to nested settings if not explicitly set. - - After this validator runs, CACHE.NAMESPACE, - VECTOR_STORE.NAMESPACE, TELEMETRY.NAMESPACE, and OTEL.SERVICE_NAMESPACE - are guaranteed to exist. Explicitly provided nested namespaces are preserved. - """ + """Propagate top-level NAMESPACE to nested settings if not explicitly set.""" if "NAMESPACE" not in self.CACHE.model_fields_set: self.CACHE.NAMESPACE = self.NAMESPACE if "NAMESPACE" not in self.VECTOR_STORE.model_fields_set: self.VECTOR_STORE.NAMESPACE = self.NAMESPACE if "NAMESPACE" not in self.TELEMETRY.model_fields_set: self.TELEMETRY.NAMESPACE = self.NAMESPACE - if "SERVICE_NAMESPACE" not in self.OTEL.model_fields_set: - self.OTEL.SERVICE_NAMESPACE = self.NAMESPACE + if "NAMESPACE" not in self.METRICS.model_fields_set: + self.METRICS.NAMESPACE = self.NAMESPACE return self diff --git a/src/crud/document.py b/src/crud/document.py index 0f59d4b2..c06bddb0 100644 --- a/src/crud/document.py +++ b/src/crud/document.py @@ -610,7 +610,8 @@ async def create_observations( collection_pairs: set[tuple[str, str]] = set() for obs in observations: - sessions_to_validate.add(obs.session_id) + if obs.session_id is not None: + sessions_to_validate.add(obs.session_id) peers_to_validate.add(obs.observer_id) peers_to_validate.add(obs.observed_id) collection_pairs.add((obs.observer_id, obs.observed_id)) diff --git a/src/deriver/__main__.py b/src/deriver/__main__.py index c57ac0aa..778f1803 100644 --- a/src/deriver/__main__.py +++ b/src/deriver/__main__.py @@ -3,17 +3,20 @@ import logging import os import uvloop +from prometheus_client import start_http_server from src.config import settings -from src.telemetry import ( - initialize_telemetry, - initialize_telemetry_async, - shutdown_telemetry, -) +from src.telemetry import initialize_telemetry_async, shutdown_telemetry from .queue_manager import main +def start_metrics_server() -> None: + """Start the Prometheus metrics HTTP server on port 9090.""" + start_http_server(9090) + print("[DERIVER] Prometheus metrics server started on port 9090") + + def setup_logging(): """ Configure logging for the deriver process. @@ -55,7 +58,7 @@ async def run_deriver(): try: await main() finally: - # Shutdown telemetry (flush CloudEvents buffer, shutdown OTel metrics) + # Shutdown telemetry (flush CloudEvents buffer) await shutdown_telemetry() @@ -67,8 +70,10 @@ if __name__ == "__main__": asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) try: - # Initialize sync telemetry (OTel metrics) - initialize_telemetry() + # Start Prometheus metrics server if enabled + if settings.METRICS.ENABLED: + start_metrics_server() + print("[DERIVER] Running main loop") asyncio.run(run_deriver()) except KeyboardInterrupt: diff --git a/src/deriver/consumer.py b/src/deriver/consumer.py index 28eca6d8..9077fea4 100644 --- a/src/deriver/consumer.py +++ b/src/deriver/consumer.py @@ -158,7 +158,7 @@ async def process_representation_batch( *, observers: list[str] | None, observed: str | None, - queue_items_count: int, + queue_item_message_ids: list[int], ) -> None: """ Prepares and processes a batch of messages for representation tasks. @@ -168,6 +168,7 @@ async def process_representation_batch( message_level_configuration: Resolved configuration for this batch observers: List of observers for the messages observed: The observed of the messages + queue_item_message_ids: Message IDs from queue items """ if not messages or not messages[0]: logger.debug("process_representation_batch received no messages") @@ -181,7 +182,7 @@ async def process_representation_batch( message_level_configuration, observers=observers, observed=observed, - queue_items_count=queue_items_count, + queue_item_message_ids=queue_item_message_ids, ) diff --git a/src/deriver/deriver.py b/src/deriver/deriver.py index 965b84f8..b8735e3c 100644 --- a/src/deriver/deriver.py +++ b/src/deriver/deriver.py @@ -7,16 +7,20 @@ from src.crud.representation import RepresentationManager from src.dependencies import tracked_db from src.models import Message from src.schemas import ResolvedConfiguration -from src.telemetry import otel_metrics +from src.telemetry import prometheus_metrics from src.telemetry.events import RepresentationCompletedEvent, emit from src.telemetry.logging import accumulate_metric, log_performance_metrics -from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes +from src.telemetry.prometheus.metrics import ( + DeriverComponents, + DeriverTaskTypes, + TokenTypes, +) from src.telemetry.sentry import with_sentry_transaction from src.utils.clients import honcho_llm_call from src.utils.config_helpers import get_configuration from src.utils.formatting import format_new_turn_with_timestamp from src.utils.representation import PromptRepresentation, Representation -from src.utils.tokens import estimate_tokens, track_deriver_input_tokens +from src.utils.tokens import track_deriver_input_tokens from .prompts import estimate_minimal_deriver_prompt_tokens, minimal_deriver_prompt @@ -30,7 +34,7 @@ async def process_representation_tasks_batch( *, observers: list[str], observed: str, - queue_items_count: int, + queue_item_message_ids: list[int], ) -> None: """ Process messages with minimal overhead - single LLM call, save to multiple collections. @@ -40,7 +44,7 @@ async def process_representation_tasks_batch( message_level_configuration: Optional configuration override. observers: List of observer peer IDs (collections to save to). observed: The observed peer ID. - queue_items_count: Number of QueueItem records being processed in this batch. + queue_item_message_ids: Message IDs from queue items being processed """ if not messages: return @@ -89,9 +93,12 @@ async def process_representation_tasks_batch( for msg in messages ) - # Track token usage + # Track token usage - count only tokens from messages being processed prompt_tokens = estimate_minimal_deriver_prompt_tokens() - messages_tokens = estimate_tokens(formatted_messages) + queue_item_message_ids_set = set(queue_item_message_ids) + messages_tokens = sum( + msg.token_count for msg in messages if msg.id in queue_item_message_ids_set + ) track_deriver_input_tokens( task_type=DeriverTaskTypes.INGESTION, components={ @@ -141,9 +148,9 @@ async def process_representation_tasks_batch( "ms", ) - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_deriver_tokens( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_deriver_tokens( count=response.output_tokens, task_type=DeriverTaskTypes.INGESTION.value, token_type=TokenTypes.OUTPUT.value, @@ -231,7 +238,7 @@ async def process_representation_tasks_batch( workspace_name=latest_message.workspace_name, session_name=latest_message.session_name, observed=observed, - queue_items_processed=queue_items_count, + queue_items_processed=len(queue_item_message_ids), earliest_message_id=earliest_message.public_id, latest_message_id=latest_message.public_id, message_count=len(messages), @@ -239,7 +246,7 @@ async def process_representation_tasks_batch( context_preparation_ms=context_prep_duration, llm_call_ms=llm_duration, total_duration_ms=overall_duration, - input_tokens=response.input_tokens, + input_tokens=messages_tokens, output_tokens=response.output_tokens, ) ) diff --git a/src/deriver/enqueue.py b/src/deriver/enqueue.py index 9641bdfe..09870782 100644 --- a/src/deriver/enqueue.py +++ b/src/deriver/enqueue.py @@ -399,7 +399,7 @@ def create_dream_record( observer: str, observed: str, dream_type: schemas.DreamType, - session_name: str, + session_name: str | None = None, ) -> dict[str, Any]: """ Create a queue record for a dream task. @@ -409,7 +409,7 @@ def create_dream_record( observer: Name of the observer peer observed: Name of the observed peer dream_type: Type of dream to execute - session_name: Name of the session to scope the dream to + session_name: Name of the session to scope the dream to if specified Returns: Queue record dictionary with workspace_name and other fields @@ -437,7 +437,7 @@ async def enqueue_dream( observed: str, dream_type: schemas.DreamType, document_count: int, - session_name: str, + session_name: str | None = None, ) -> None: """ Enqueue a dream task for immediate processing by the deriver. @@ -452,7 +452,7 @@ async def enqueue_dream( observed: Name of the observed peer dream_type: Type of dream to execute document_count: Current document count for metadata update - session_name: Name of the session to scope the dream to + session_name: Name of the session to scope the dream to if specified """ async with tracked_db("dream_enqueue") as db_session: try: diff --git a/src/deriver/queue_manager.py b/src/deriver/queue_manager.py index f41abde4..c1f48d86 100644 --- a/src/deriver/queue_manager.py +++ b/src/deriver/queue_manager.py @@ -35,7 +35,7 @@ from src.reconciler import ( set_reconciler_scheduler, ) from src.schemas import ResolvedConfiguration -from src.telemetry import otel_metrics +from src.telemetry import prometheus_metrics from src.telemetry.sentry import initialize_sentry from src.utils.work_unit import parse_work_unit_key from src.webhooks.events import ( @@ -450,12 +450,17 @@ class QueueManager: else: observers = [] + queue_item_message_ids = [ + item.message_id + for item in items_to_process + if item.message_id is not None + ] await process_representation_batch( messages_context, message_level_configuration, observers=observers, observed=work_unit.observed, - queue_items_count=len(items_to_process), + queue_item_message_ids=queue_item_message_ids, ) await self.mark_queue_items_as_processed( items_to_process, work_unit_key @@ -791,9 +796,9 @@ class QueueManager: if ( work_unit.task_type in ["representation", "summary"] and work_unit.workspace_name is not None - and settings.OTEL.ENABLED + and settings.METRICS.ENABLED ): - otel_metrics.record_deriver_queue_item( + prometheus_metrics.record_deriver_queue_item( count=len(items), workspace_name=work_unit.workspace_name, task_type=work_unit.task_type, diff --git a/src/dialectic/core.py b/src/dialectic/core.py index 64d1a6e3..9a0114dd 100644 --- a/src/dialectic/core.py +++ b/src/dialectic/core.py @@ -17,14 +17,14 @@ from sqlalchemy.ext.asyncio import AsyncSession from src import crud from src.config import ReasoningLevel, settings from src.dialectic import prompts -from src.telemetry import otel_metrics +from src.telemetry import prometheus_metrics from src.telemetry.events import DialecticCompletedEvent, DialecticPhaseMetrics, emit from src.telemetry.logging import ( accumulate_metric, log_performance_metrics, log_token_usage_metrics, ) -from src.telemetry.otel.metrics import DialecticComponents, TokenTypes +from src.telemetry.prometheus.metrics import DialecticComponents, TokenTypes from src.utils.agent_tools import ( DIALECTIC_TOOLS, DIALECTIC_TOOLS_MINIMAL, @@ -338,15 +338,15 @@ class DialecticAgent: if not self.metric_key and run_id is not None: log_performance_metrics("dialectic_chat", run_id) - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_dialectic_tokens( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_dialectic_tokens( count=input_tokens, token_type=TokenTypes.INPUT.value, component=DialecticComponents.TOTAL.value, reasoning_level=self.reasoning_level, ) - otel_metrics.record_dialectic_tokens( + prometheus_metrics.record_dialectic_tokens( count=output_tokens, token_type=TokenTypes.OUTPUT.value, component=DialecticComponents.TOTAL.value, @@ -615,21 +615,6 @@ class DialecticAgent: if not self.metric_key and run_id is not None: log_performance_metrics("dialectic_chat", run_id) - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_dialectic_tokens( - count=total_input_tokens, - token_type=TokenTypes.INPUT.value, - component=DialecticComponents.TOTAL.value, - reasoning_level=self.reasoning_level, - ) - otel_metrics.record_dialectic_tokens( - count=total_output_tokens, - token_type=TokenTypes.OUTPUT.value, - component=DialecticComponents.TOTAL.value, - reasoning_level=self.reasoning_level, - ) - # Get model/provider info for phase metrics level_settings = settings.DIALECTIC.LEVELS[self.reasoning_level] synthesis_settings = level_settings.SYNTHESIS diff --git a/src/dreamer/orchestrator.py b/src/dreamer/orchestrator.py index 05c9474e..91517cd8 100644 --- a/src/dreamer/orchestrator.py +++ b/src/dreamer/orchestrator.py @@ -86,7 +86,7 @@ async def run_dream( workspace_name: str, observer: str, observed: str, - session_name: str, + session_name: str | None = None, ) -> DreamResult | None: """ Run a full dream cycle with optional surprisal-based sampling. @@ -101,7 +101,7 @@ async def run_dream( workspace_name: Workspace identifier observer: Observer peer name observed: Observed peer name - session_name: Session identifier + session_name: Session identifier if specified """ if not settings.DREAM.ENABLED: return None @@ -114,9 +114,13 @@ async def run_dream( f"[{run_id}] Starting dream cycle for {workspace_name}/{observer}/{observed}" ) - session = await crud.get_session( - db, workspace_name=workspace_name, session_name=session_name - ) + if session_name is not None: + session = await crud.get_session( + db, workspace_name=workspace_name, session_name=session_name + ) + else: + session = None + workspace = await crud.get_workspace(db, workspace_name=workspace_name) configuration = get_configuration(None, session, workspace) diff --git a/src/dreamer/specialists.py b/src/dreamer/specialists.py index 369ee810..e224b33c 100644 --- a/src/dreamer/specialists.py +++ b/src/dreamer/specialists.py @@ -22,10 +22,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from src.config import settings from src.schemas import ResolvedConfiguration -from src.telemetry import otel_metrics +from src.telemetry import prometheus_metrics from src.telemetry.events import DreamSpecialistEvent, emit from src.telemetry.logging import accumulate_metric, log_performance_metrics -from src.telemetry.otel.metrics import TokenTypes +from src.telemetry.prometheus.metrics import TokenTypes from src.utils.agent_tools import ( DEDUCTION_SPECIALIST_TOOLS, INDUCTION_SPECIALIST_TOOLS, @@ -96,7 +96,7 @@ class BaseSpecialist(ABC): workspace_name: str, observer: str, observed: str, - session_name: str, + session_name: str | None, probing_questions: list[str], configuration: ResolvedConfiguration | None = None, parent_run_id: str | None = None, @@ -186,14 +186,14 @@ class BaseSpecialist(ABC): accumulate_metric(task_name, "input_tokens", response.input_tokens, "count") accumulate_metric(task_name, "output_tokens", response.output_tokens, "count") - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_dreamer_tokens( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_dreamer_tokens( count=response.input_tokens, specialist_name=self.name, token_type=TokenTypes.INPUT.value, ) - otel_metrics.record_dreamer_tokens( + prometheus_metrics.record_dreamer_tokens( count=response.output_tokens, specialist_name=self.name, token_type=TokenTypes.OUTPUT.value, diff --git a/src/dreamer/trees/graph.py b/src/dreamer/trees/graph.py index b4778f13..1899ee27 100644 --- a/src/dreamer/trees/graph.py +++ b/src/dreamer/trees/graph.py @@ -4,19 +4,22 @@ Graph-theoretic surprisal using k-NN graph and random walk. from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np from numpy.typing import NDArray -from sklearn.neighbors import NearestNeighbors from .base import SurprisalTree +if TYPE_CHECKING: + from sklearn.neighbors import NearestNeighbors + def _knn_indices( points: NDArray[np.floating[Any]], n_neighbors: int ) -> NDArray[np.intp]: """Get k-nearest neighbor indices for each point.""" + from sklearn.neighbors import NearestNeighbors knn: NearestNeighbors = NearestNeighbors(n_neighbors=n_neighbors, algorithm="auto") knn.fit(points) # pyright: ignore[reportUnknownMemberType] @@ -26,6 +29,7 @@ def _knn_indices( def _nearest_index(points: NDArray[np.floating[Any]], query: np.ndarray) -> int: """Find index of nearest point to query.""" + from sklearn.neighbors import NearestNeighbors knn: NearestNeighbors = NearestNeighbors(n_neighbors=1) knn.fit(points) # pyright: ignore[reportUnknownMemberType] diff --git a/src/dreamer/trees/sklearn_wrapper.py b/src/dreamer/trees/sklearn_wrapper.py index 42aca2d5..86b9522e 100644 --- a/src/dreamer/trees/sklearn_wrapper.py +++ b/src/dreamer/trees/sklearn_wrapper.py @@ -2,17 +2,19 @@ Wrapper for sklearn's KDTree and BallTree. """ -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np from numpy.typing import NDArray -from sklearn.neighbors import ( - BallTree, # pyright: ignore[reportUnknownVariableType] - KDTree, # pyright: ignore[reportUnknownVariableType] -) from .base import SurprisalTree +if TYPE_CHECKING: + from sklearn.neighbors import ( + BallTree, # pyright: ignore[reportUnknownVariableType] + KDTree, # pyright: ignore[reportUnknownVariableType] + ) + class SklearnTreeWrapper(SurprisalTree): """ @@ -23,7 +25,7 @@ class SklearnTreeWrapper(SurprisalTree): tree_type: str k: int points: list[NDArray[np.floating[Any]]] - tree: KDTree | BallTree | None + tree: "KDTree | BallTree | None" total_points: int def __init__( @@ -50,6 +52,11 @@ class SklearnTreeWrapper(SurprisalTree): if len(self.points) == 0: return + from sklearn.neighbors import ( + BallTree, # pyright: ignore[reportUnknownVariableType] + KDTree, # pyright: ignore[reportUnknownVariableType] + ) + points_array: NDArray[np.floating[Any]] = np.array(self.points) if self.tree_type == "kd": self.tree = KDTree(points_array) diff --git a/src/main.py b/src/main.py index 73eedcc5..0ca1f9d3 100644 --- a/src/main.py +++ b/src/main.py @@ -30,9 +30,9 @@ from src.routers import ( ) from src.security import create_admin_jwt from src.telemetry import ( - initialize_telemetry, initialize_telemetry_async, - otel_metrics, + metrics_endpoint, + prometheus_metrics, shutdown_telemetry, ) from src.telemetry.logging import get_route_template @@ -120,8 +120,7 @@ if SENTRY_ENABLED: @asynccontextmanager async def lifespan(_: FastAPI): - # Initialize telemetry (OTel metrics + CloudEvents emitter) - initialize_telemetry() + # Initialize CloudEvents telemetry await initialize_telemetry_async() try: @@ -140,7 +139,7 @@ async def lifespan(_: FastAPI): await close_external_vector_store() await close_cache() await engine.dispose() - # Shutdown telemetry (flush CloudEvents buffer, shutdown OTel metrics) + # Shutdown telemetry (flush CloudEvents buffer) await shutdown_telemetry() @@ -153,7 +152,7 @@ app = FastAPI( title="Honcho API", summary="The Identity Layer for the Agentic World", description="""Honcho is a platform for giving agents user-centric memory and social cognition.""", - version="3.0.0", + version="3.0.1", contact={ "name": "Plastic Labs", "url": "https://honcho.dev", @@ -191,6 +190,9 @@ app.include_router(conclusions.router, prefix="/v3") app.include_router(keys.router, prefix="/v3") app.include_router(webhooks.router, prefix="/v3") +# Prometheus metrics endpoint +app.add_route("/metrics", metrics_endpoint, methods=["GET"]) + # Global exception handlers @app.exception_handler(HonchoException) @@ -233,9 +235,9 @@ async def track_request( response = await call_next(request) # Track metrics if enabled - if settings.OTEL.ENABLED: + if settings.METRICS.ENABLED: template = get_route_template(request) - otel_metrics.record_api_request( + prometheus_metrics.record_api_request( method=request.method, endpoint=template, status_code=str(response.status_code), diff --git a/src/models.py b/src/models.py index 12d342a3..c2aa84dd 100644 --- a/src/models.py +++ b/src/models.py @@ -399,7 +399,7 @@ class Document(Base): workspace_name: Mapped[str] = mapped_column( ForeignKey("workspaces.name"), nullable=False, index=True ) - session_name: Mapped[str] = mapped_column(TEXT, index=True) + session_name: Mapped[str | None] = mapped_column(TEXT, nullable=True, index=True) deleted_at: Mapped[datetime.datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, index=True, default=None ) diff --git a/src/routers/messages.py b/src/routers/messages.py index d5746d12..3acdc1e5 100644 --- a/src/routers/messages.py +++ b/src/routers/messages.py @@ -22,7 +22,7 @@ from src.dependencies import db from src.deriver import enqueue from src.exceptions import FileTooLargeError, ResourceNotFoundException from src.security import require_auth -from src.telemetry import otel_metrics +from src.telemetry import prometheus_metrics from src.utils.files import process_file_uploads_for_messages logger = logging.getLogger(__name__) @@ -100,9 +100,9 @@ async def create_messages_for_session( session_name=session_id, ) - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_messages_created( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_messages_created( count=len(created_messages), workspace_name=workspace_id, ) @@ -199,9 +199,9 @@ async def create_messages_with_file( len(created_messages), ) - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_messages_created( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_messages_created( count=len(created_messages), workspace_name=workspace_id, ) diff --git a/src/routers/peers.py b/src/routers/peers.py index c9ceba75..3bff98a3 100644 --- a/src/routers/peers.py +++ b/src/routers/peers.py @@ -14,7 +14,7 @@ from src.dependencies import db, tracked_db from src.dialectic.chat import agentic_chat, agentic_chat_stream from src.exceptions import AuthenticationException, ResourceNotFoundException from src.security import JWTParams, require_auth -from src.telemetry import otel_metrics +from src.telemetry import prometheus_metrics from src.utils.search import search logger = logging.getLogger(__name__) @@ -184,9 +184,9 @@ async def chat( yield f"data: {json.dumps({'delta': {'content': chunk}, 'done': False})}\n\n" yield f"data: {json.dumps({'done': True})}\n\n" - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_dialectic_call( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_dialectic_call( workspace_name=workspace_id, reasoning_level=options.reasoning_level, ) @@ -216,9 +216,9 @@ async def chat( reasoning_level=options.reasoning_level, ) - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_dialectic_call( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_dialectic_call( workspace_name=workspace_id, reasoning_level=options.reasoning_level, ) diff --git a/src/routers/sessions.py b/src/routers/sessions.py index 4e141837..7cad530e 100644 --- a/src/routers/sessions.py +++ b/src/routers/sessions.py @@ -508,9 +508,9 @@ async def get_session_context( description=f"Number of tokens to use for the context. Includes summary if set to true. Includes representation and peer card if they are included in the response. If not provided, the context will be exhaustive (within {config.settings.GET_CONTEXT_MAX_TOKENS} tokens)", ), *, - last_message: str | None = Query( + search_query: str | None = Query( None, - description="The most recent message, used to fetch semantically relevant conclusions", + description="A query string used to fetch semantically relevant conclusions", ), include_summary: bool = Query( default=True, @@ -527,29 +527,29 @@ async def get_session_context( ), limit_to_session: bool = Query( default=False, - description="Only used if `last_message` is provided. Whether to limit the representation to the session (as opposed to everything known about the target peer)", + description="Only used if `search_query` is provided. Whether to limit the representation to the session (as opposed to everything known about the target peer)", ), search_top_k: int | None = Query( None, ge=1, le=100, - description="Only used if `last_message` is provided. The number of semantic-search-retrieved conclusions to include in the representation", + description="Only used if `search_query` is provided. The number of semantic-search-retrieved conclusions to include in the representation", ), search_max_distance: float | None = Query( None, ge=0.0, le=1.0, - description="Only used if `last_message` is provided. The maximum distance to search for semantically relevant conclusions", + description="Only used if `search_query` is provided. The maximum distance to search for semantically relevant conclusions", ), include_most_frequent: bool = Query( default=False, - description="Only used if `last_message` is provided. Whether to include the most frequent conclusions in the representation", + description="Only used if `search_query` is provided. Whether to include the most frequent conclusions in the representation", ), max_conclusions: int | None = Query( None, ge=1, le=100, - description="Only used if `last_message` is provided. The maximum number of conclusions to include in the representation", + description="Only used if `search_query` is provided. The maximum number of conclusions to include in the representation", ), ): """ @@ -585,7 +585,7 @@ async def get_session_context( # with tracked_db creating separate database sessions representation = await _get_working_representation_task( workspace_id, - last_message, + search_query, observer=observer, observed=observed, session_name=session_id if limit_to_session else None, diff --git a/src/schemas.py b/src/schemas.py index 2f9fceb6..a476ee92 100644 --- a/src/schemas.py +++ b/src/schemas.py @@ -1,7 +1,7 @@ import datetime import ipaddress from enum import Enum -from typing import Annotated, Any, Self +from typing import Annotated, Any, Self, cast from urllib.parse import urlparse import tiktoken @@ -173,6 +173,20 @@ class ResolvedConfiguration(BaseModel): summary: ResolvedSummaryConfiguration dream: ResolvedDreamConfiguration + @model_validator(mode="before") + @classmethod + def migrate_deriver_to_reasoning(cls, data: Any) -> Any: + """Handle v3.0.0 migration: 'deriver' was renamed to 'reasoning'.""" + if not isinstance(data, dict): + return data + + config = cast(dict[str, Any], data) + + if "deriver" in config and "reasoning" not in config: + config["reasoning"] = config.pop("deriver") + + return config + class PeerConfig(BaseModel): # TODO: Update description - should say "Whether honcho forms a representation of the peer itself" @@ -526,8 +540,9 @@ class DocumentMetadata(BaseModel): class DocumentCreate(DocumentBase): content: Annotated[str, Field(min_length=1, max_length=100000)] - session_name: str = Field( - description="The session from which the document was derived" + session_name: str | None = Field( + default=None, + description="The session from which the document was derived (NULL for global observations)", ) level: DocumentLevel = Field( default="explicit", @@ -566,7 +581,7 @@ class Conclusion(BaseModel): description="The peer the conclusion is about", serialization_alias="observed_id", ) - session_name: str = Field(serialization_alias="session_id") + session_name: str | None = Field(default=None, serialization_alias="session_id") created_at: datetime.datetime model_config = ConfigDict( # pyright: ignore @@ -603,7 +618,10 @@ class ConclusionCreate(BaseModel): content: Annotated[str, Field(min_length=1, max_length=65535)] observer_id: str = Field(..., description="The peer making the conclusion") observed_id: str = Field(..., description="The peer the conclusion is about") - session_id: str = Field(..., description="The session this conclusion relates to") + session_id: str | None = Field( + default=None, + description="A session ID to store the conclusion in, if specified", + ) _token_count: int = PrivateAttr(default=0) @@ -767,7 +785,9 @@ class ScheduleDreamRequest(BaseModel): None, description="Observed peer name (defaults to observer if not specified)" ) dream_type: DreamType = Field(..., description="Type of dream to schedule") - session_id: str = Field(..., description="Session ID to scope the dream to") + session_id: str | None = Field( + None, description="Session ID to scope the dream to if specified" + ) # Webhook endpoint schemas diff --git a/src/telemetry/__init__.py b/src/telemetry/__init__.py index 87c86c8c..401be523 100644 --- a/src/telemetry/__init__.py +++ b/src/telemetry/__init__.py @@ -3,7 +3,7 @@ Telemetry module for Honcho. This module consolidates all telemetry, metrics, and observability functionality: - Sentry: Error tracking and performance tracing -- OTel: Push-based metrics via OTLP to any compatible backend (e.g., Mimir) +- Prometheus: Pull-based metrics scraped by Fly.io - CloudEvents: Structured events for analytics (push-based) - Logging: Langfuse integration, Rich console output, metric accumulation - Tracing: Sentry transaction decorators @@ -12,54 +12,27 @@ This module consolidates all telemetry, metrics, and observability functionality """ from src.telemetry.events import emit -from src.telemetry.otel import get_meter, initialize_otel_metrics, shutdown_otel_metrics -from src.telemetry.otel.metrics import otel_metrics +from src.telemetry.prometheus import metrics_endpoint, prometheus_metrics __all__ = [ "emit", - "get_meter", - "initialize_otel_metrics", - "initialize_telemetry", "initialize_telemetry_async", - "otel_metrics", - "shutdown_otel_metrics", + "metrics_endpoint", + "prometheus_metrics", "shutdown_telemetry", ] -def initialize_telemetry() -> None: - """ - Initialize all telemetry systems based on configuration. - - This should be called once at application startup (in main.py lifespan). - It reads configuration from settings and initializes: - - OTel metrics (if OTEL_ENABLED=true) - - Note: CloudEvents telemetry requires async initialization and should be - initialized separately using initialize_telemetry_async(). - - Sentry is initialized separately in sentry.py as it has its own lifecycle. - """ - from src.config import settings - - if settings.OTEL.ENABLED: - initialize_otel_metrics( - endpoint=settings.OTEL.ENDPOINT, - headers=settings.OTEL.HEADERS, - export_interval_millis=settings.OTEL.EXPORT_INTERVAL_MILLIS, - service_name=settings.OTEL.SERVICE_NAME, - service_namespace=settings.OTEL.SERVICE_NAMESPACE, - enabled=True, - ) - - async def initialize_telemetry_async() -> None: """ Initialize async telemetry systems based on configuration. - This should be called once at application startup (in main.py lifespan), - after initialize_telemetry(). It initializes: + This should be called once at application startup (in main.py lifespan). + It initializes: - CloudEvents emitter (if TELEMETRY_ENABLED=true) + + Note: Prometheus metrics are pull-based and require no initialization. + Sentry is initialized separately in sentry.py as it has its own lifecycle. """ from src.config import settings from src.telemetry.events import initialize_telemetry_events @@ -73,13 +46,9 @@ async def shutdown_telemetry() -> None: Shutdown all telemetry systems gracefully. This should be called during application shutdown to ensure: - - OTel metrics are flushed - CloudEvents buffer is flushed """ from src.telemetry.events import shutdown_telemetry_events # Shutdown CloudEvents emitter (flushes buffer) await shutdown_telemetry_events() - - # Shutdown OTel metrics - shutdown_otel_metrics() diff --git a/src/telemetry/events/dream.py b/src/telemetry/events/dream.py index 2b65a2ea..33ce19d3 100644 --- a/src/telemetry/events/dream.py +++ b/src/telemetry/events/dream.py @@ -32,7 +32,7 @@ class DreamRunEvent(BaseEvent): workspace_name: str = Field(..., description="Workspace name") # Session context - session_name: str = Field(..., description="Most recent session name") + session_name: str | None = Field(None, description="Session name if specified") # Peer context observer: str = Field(..., description="Observer peer name") diff --git a/src/telemetry/otel/__init__.py b/src/telemetry/otel/__init__.py deleted file mode 100644 index 08c6ede8..00000000 --- a/src/telemetry/otel/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -OpenTelemetry metrics module for Honcho. - -This module provides push-based metrics using OpenTelemetry SDK -with OTLP HTTP export to any compatible backend (Mimir, Grafana Cloud, etc.). -""" - -from src.telemetry.otel.metrics import ( - DeriverComponents, - DeriverTaskTypes, - DialecticComponents, - TokenTypes, - get_meter, - initialize_otel_metrics, - shutdown_otel_metrics, -) - -__all__ = [ - "DeriverComponents", - "DeriverTaskTypes", - "DialecticComponents", - "TokenTypes", - "get_meter", - "initialize_otel_metrics", - "shutdown_otel_metrics", -] diff --git a/src/telemetry/otel/metrics.py b/src/telemetry/otel/metrics.py deleted file mode 100644 index 0c3ec7c7..00000000 --- a/src/telemetry/otel/metrics.py +++ /dev/null @@ -1,481 +0,0 @@ -""" -OpenTelemetry metrics implementation with OTLP export. - -This module provides push-based metrics that replace the pull-based Prometheus -/metrics endpoint. Metrics are pushed via OTLP to any compatible backend -(Mimir, Grafana Cloud, etc.) at configurable intervals. - -Usage: - from src.telemetry.otel import get_meter, initialize_otel_metrics - - # Initialize once at startup - initialize_otel_metrics() - - # Get a meter for your component - meter = get_meter("honcho.deriver") - - # Create instruments - counter = meter.create_counter("tokens_processed", unit="tokens") - counter.add(100, {"task_type": "ingestion"}) -""" - -from __future__ import annotations - -import atexit -import logging -from enum import Enum -from typing import TYPE_CHECKING, final - -from opentelemetry import metrics -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader -from opentelemetry.sdk.resources import Resource - -if TYPE_CHECKING: - from opentelemetry.metrics import Counter, Meter - -logger = logging.getLogger(__name__) - -# ============================================================================= -# Metric label enums -# ============================================================================= - - -class TokenTypes(Enum): - INPUT = "input" - OUTPUT = "output" - - -class DeriverTaskTypes(Enum): - INGESTION = "ingestion" - SUMMARY = "summary" - - -class DeriverComponents(Enum): - PROMPT = "prompt" # used in ingestion and summary - MESSAGES = "messages" # used in ingestion and summary - PREVIOUS_SUMMARY = "previous_summary" # only used for summary - OUTPUT_TOTAL = "output_total" - - -class DialecticComponents(Enum): - TOTAL = "total" - - -# ============================================================================= -# OTel metrics infrastructure -# ============================================================================= - -# Global state -_meter_provider: MeterProvider | None = None -_initialized: bool = False - - -def initialize_otel_metrics( - *, - endpoint: str | None = None, - headers: dict[str, str] | None = None, - export_interval_millis: int = 60000, - service_name: str = "honcho", - service_namespace: str | None = None, - enabled: bool = True, -) -> None: - """ - Initialize OpenTelemetry metrics with OTLP export. - - This should be called once at application startup. If already initialized, - subsequent calls are no-ops. - - Args: - endpoint: OTLP HTTP endpoint URL (e.g., "https://mimir.example.com/otlp/v1/metrics"). - If None, metrics are collected but not exported (useful for testing). - headers: Optional headers to include in requests (e.g., {"X-Scope-OrgID": "tenant"}). - export_interval_millis: How often to export metrics (default: 60 seconds). - service_name: Service name for resource attributes (default: "honcho"). - service_namespace: Optional namespace for the service. - enabled: If False, metrics are no-ops (default: True). - """ - global _meter_provider, _initialized - - if _initialized: - logger.debug("OTel metrics already initialized, skipping") - return - - if not enabled: - logger.info("OTel metrics disabled") - _initialized = True - return - - # Build resource attributes - resource_attributes = { - "service.name": service_name, - } - if service_namespace: - resource_attributes["service.namespace"] = service_namespace - - resource = Resource.create(resource_attributes) - - # Create metric reader - readers: list[PeriodicExportingMetricReader] = [] - - if endpoint: - try: - from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( - OTLPMetricExporter, - ) - - exporter = OTLPMetricExporter( - endpoint=endpoint, - headers=headers or {}, - ) - reader = PeriodicExportingMetricReader( - exporter, - export_interval_millis=export_interval_millis, - ) - readers.append(reader) - logger.info(f"OTel metrics configured to push via OTLP to {endpoint}") - except Exception as e: - logger.error(f"Failed to configure OTLP metrics exporter: {e}") - # Continue without exporter - metrics still work locally - else: - logger.info( - "OTel metrics initialized without remote export (no endpoint configured)" - ) - - # Create and set the meter provider - # Note: empty list is valid for metric_readers (metrics still work, just not exported) - _meter_provider = MeterProvider( - resource=resource, - metric_readers=readers, - ) - metrics.set_meter_provider(_meter_provider) - - # Register shutdown handler - atexit.register(shutdown_otel_metrics) - - _initialized = True - logger.info("OTel metrics initialized successfully") - - -def shutdown_otel_metrics() -> None: - """ - Shutdown the OTel metrics provider, flushing any pending metrics. - - This is automatically called at process exit via atexit, but can be - called manually for graceful shutdown. - """ - global _meter_provider, _initialized - - if _meter_provider is not None: - try: - _meter_provider.shutdown() - logger.info("OTel metrics shutdown complete") - except Exception as e: - logger.error(f"Error during OTel metrics shutdown: {e}") - finally: - _meter_provider = None - _initialized = False - - -def get_meter(name: str, version: str = "") -> Meter: - """ - Get an OTel Meter for creating instruments. - - Args: - name: The name of the instrumentation scope (e.g., "honcho.deriver"). - version: Optional version of the instrumentation scope. - - Returns: - An OTel Meter instance for creating counters, histograms, etc. - - Example: - meter = get_meter("honcho.deriver") - counter = meter.create_counter("tokens_processed", unit="tokens") - counter.add(100, {"task_type": "ingestion"}) - """ - return metrics.get_meter(name, version) - - -# ============================================================================= -# Pre-defined metrics that mirror existing Prometheus counters -# ============================================================================= - -# These are created lazily on first use to avoid issues with initialization order - - -@final -class OTelMetrics: - """ - Container for OTel metrics that mirror existing Prometheus counters. - - This class provides a bridge during migration - the same metrics are - available via both Prometheus (pull) and OTel (push). - - Namespace is managed at the instance level (from settings), not per-call. - """ - - _instance: OTelMetrics | None = None - _is_initialized: bool = False - _namespace: str = "honcho" - - # Meters (lazily initialized) - _api_meter: Meter | None = None - _deriver_meter: Meter | None = None - _dialectic_meter: Meter | None = None - _dreamer_meter: Meter | None = None - - # Counters (lazily initialized) - _api_requests: Counter | None = None - _messages_created: Counter | None = None - _dialectic_calls: Counter | None = None - _deriver_queue_items: Counter | None = None - _deriver_tokens: Counter | None = None - _dialectic_tokens: Counter | None = None - _dreamer_tokens: Counter | None = None - - def __new__(cls) -> OTelMetrics: - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance - - def _ensure_initialized(self) -> None: - """Lazily initialize meters and instruments.""" - if self._is_initialized: - return - - # Get namespace from settings (same as Prometheus) - from src.config import settings - - self._namespace = settings.OTEL.SERVICE_NAMESPACE or "honcho" - - # Get meters for different components - self._api_meter = get_meter("honcho.api") - self._deriver_meter = get_meter("honcho.deriver") - self._dialectic_meter = get_meter("honcho.dialectic") - self._dreamer_meter = get_meter("honcho.dreamer") - - # Create counters that mirror Prometheus metrics - # API requests - self._api_requests = self._api_meter.create_counter( - name="api_requests", - unit="requests", - description="Total API requests", - ) - - # Messages created - self._messages_created = self._api_meter.create_counter( - name="messages_created", - unit="messages", - description="Total messages created", - ) - - # Dialectic calls - self._dialectic_calls = self._dialectic_meter.create_counter( - name="dialectic_calls", - unit="calls", - description="Total dialectic calls", - ) - - # Deriver queue items processed - self._deriver_queue_items = self._deriver_meter.create_counter( - name="deriver_queue_items_processed", - unit="items", - description="Total deriver queue items processed", - ) - - # Token counters - self._deriver_tokens = self._deriver_meter.create_counter( - name="deriver_tokens_processed", - unit="tokens", - description="Total tokens processed by the deriver", - ) - - self._dialectic_tokens = self._dialectic_meter.create_counter( - name="dialectic_tokens_processed", - unit="tokens", - description="Total tokens processed by the dialectic", - ) - - self._dreamer_tokens = self._dreamer_meter.create_counter( - name="dreamer_tokens_processed", - unit="tokens", - description="Total tokens processed by the dreamer", - ) - - self._is_initialized = True - - def _handle_metric_error(self, method_name: str, error: Exception) -> None: - """Handle errors from metric recording by logging to Sentry.""" - import sentry_sdk - - sentry_sdk.capture_exception(error) - logger.warning( - "Failed to record OTel metric in %s: %s", method_name, str(error) - ) - - def record_api_request( - self, - *, - method: str, - endpoint: str, - status_code: str, - ) -> None: - """Record an API request metric.""" - try: - self._ensure_initialized() - if self._api_requests is None: - return # Not initialized, skip silently - self._api_requests.add( - 1, - { - "method": method, - "endpoint": endpoint, - "status_code": status_code, - "namespace": self._namespace, - }, - ) - except Exception as e: - self._handle_metric_error("record_api_request", e) - - def record_messages_created( - self, - *, - count: int, - workspace_name: str, - ) -> None: - """Record messages created metric.""" - try: - self._ensure_initialized() - if self._messages_created is None: - return # Not initialized, skip silently - self._messages_created.add( - count, - { - "workspace_name": workspace_name, - "namespace": self._namespace, - }, - ) - except Exception as e: - self._handle_metric_error("record_messages_created", e) - - def record_dialectic_call( - self, - *, - workspace_name: str, - reasoning_level: str, - ) -> None: - """Record a dialectic call metric.""" - try: - self._ensure_initialized() - if self._dialectic_calls is None: - return # Not initialized, skip silently - self._dialectic_calls.add( - 1, - { - "workspace_name": workspace_name, - "reasoning_level": reasoning_level, - "namespace": self._namespace, - }, - ) - except Exception as e: - self._handle_metric_error("record_dialectic_call", e) - - def record_deriver_queue_item( - self, - *, - count: int, - workspace_name: str, - task_type: str, - ) -> None: - """Record deriver queue items processed metric.""" - try: - self._ensure_initialized() - if self._deriver_queue_items is None: - return # Not initialized, skip silently - self._deriver_queue_items.add( - count, - { - "workspace_name": workspace_name, - "task_type": task_type, - "namespace": self._namespace, - }, - ) - except Exception as e: - self._handle_metric_error("record_deriver_queue_item", e) - - def record_deriver_tokens( - self, - *, - count: int, - task_type: str, - token_type: str, - component: str, - ) -> None: - """Record deriver token usage metric.""" - try: - self._ensure_initialized() - if self._deriver_tokens is None: - return # Not initialized, skip silently - self._deriver_tokens.add( - count, - { - "task_type": task_type, - "token_type": token_type, - "component": component, - "namespace": self._namespace, - }, - ) - except Exception as e: - self._handle_metric_error("record_deriver_tokens", e) - - def record_dialectic_tokens( - self, - *, - count: int, - token_type: str, - component: str, - reasoning_level: str, - ) -> None: - """Record dialectic token usage metric.""" - try: - self._ensure_initialized() - if self._dialectic_tokens is None: - return # Not initialized, skip silently - self._dialectic_tokens.add( - count, - { - "token_type": token_type, - "component": component, - "reasoning_level": reasoning_level, - "namespace": self._namespace, - }, - ) - except Exception as e: - self._handle_metric_error("record_dialectic_tokens", e) - - def record_dreamer_tokens( - self, - *, - count: int, - specialist_name: str, - token_type: str, - ) -> None: - """Record dreamer token usage metric.""" - try: - self._ensure_initialized() - if self._dreamer_tokens is None: - return # Not initialized, skip silently - self._dreamer_tokens.add( - count, - { - "specialist_name": specialist_name, - "token_type": token_type, - "namespace": self._namespace, - }, - ) - except Exception as e: - self._handle_metric_error("record_dreamer_tokens", e) - - -# Singleton instance -otel_metrics = OTelMetrics() diff --git a/src/telemetry/prometheus/__init__.py b/src/telemetry/prometheus/__init__.py new file mode 100644 index 00000000..ab0a2616 --- /dev/null +++ b/src/telemetry/prometheus/__init__.py @@ -0,0 +1,26 @@ +""" +Prometheus telemetry module. + +Exports: +- prometheus_metrics: Singleton for recording metrics +- metrics_endpoint: Async endpoint for /metrics route +- Label enums for metric values +""" + +from src.telemetry.prometheus.metrics import ( + DeriverComponents, + DeriverTaskTypes, + DialecticComponents, + TokenTypes, + metrics_endpoint, + prometheus_metrics, +) + +__all__ = [ + "DeriverComponents", + "DeriverTaskTypes", + "DialecticComponents", + "TokenTypes", + "metrics_endpoint", + "prometheus_metrics", +] diff --git a/src/telemetry/prometheus/metrics.py b/src/telemetry/prometheus/metrics.py new file mode 100644 index 00000000..f64399b7 --- /dev/null +++ b/src/telemetry/prometheus/metrics.py @@ -0,0 +1,234 @@ +"""Prometheus metrics for Honcho.""" + +from __future__ import annotations + +import logging +from enum import Enum +from typing import cast, final + +from prometheus_client import ( + CONTENT_TYPE_LATEST, + REGISTRY, + Counter, + disable_created_metrics, + generate_latest, +) +from starlette.requests import Request +from starlette.responses import Response + +from src.config import settings + +disable_created_metrics() + +logger = logging.getLogger(__name__) + + +class NamespacedCounter(Counter): + def labels(self, **kwargs: str) -> NamespacedCounter: + kwargs["namespace"] = cast(str, settings.METRICS.NAMESPACE) + return super().labels(**kwargs) # type: ignore[return-value] + + +class TokenTypes(Enum): + INPUT = "input" + OUTPUT = "output" + + +class DeriverTaskTypes(Enum): + INGESTION = "ingestion" + SUMMARY = "summary" + + +class DeriverComponents(Enum): + PROMPT = "prompt" + MESSAGES = "messages" + PREVIOUS_SUMMARY = "previous_summary" + OUTPUT_TOTAL = "output_total" + + +class DialecticComponents(Enum): + TOTAL = "total" + + +api_requests_counter = NamespacedCounter( + "api_requests", + "Total API requests", + ["namespace", "method", "endpoint", "status_code"], +) + +messages_created_counter = NamespacedCounter( + "messages_created", + "Total messages created", + ["namespace", "workspace_name"], +) + +dialectic_calls_counter = NamespacedCounter( + "dialectic_calls", + "Total dialectic calls", + ["namespace", "workspace_name", "reasoning_level"], +) + +deriver_queue_items_processed_counter = NamespacedCounter( + "deriver_queue_items_processed", + "Total deriver queue items processed", + ["namespace", "workspace_name", "task_type"], +) + +deriver_tokens_processed_counter = NamespacedCounter( + "deriver_tokens_processed", + "Total tokens processed by the deriver", + ["namespace", "task_type", "token_type", "component"], +) + +dialectic_tokens_processed_counter = NamespacedCounter( + "dialectic_tokens_processed", + "Total tokens processed by the dialectic", + ["namespace", "token_type", "component", "reasoning_level"], +) + +dreamer_tokens_processed_counter = NamespacedCounter( + "dreamer_tokens_processed", + "Total tokens processed by the dreamer", + ["namespace", "specialist_name", "token_type"], +) + + +@final +class PrometheusMetrics: + _instance: PrometheusMetrics | None = None + + def __new__(cls) -> PrometheusMetrics: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def _handle_metric_error(self, method_name: str, error: Exception) -> None: + import sentry_sdk + + sentry_sdk.capture_exception(error) + logger.warning( + "Failed to record Prometheus metric in %s: %s", method_name, str(error) + ) + + def record_api_request( + self, + *, + method: str, + endpoint: str, + status_code: str, + ) -> None: + try: + api_requests_counter.labels( + method=method, + endpoint=endpoint, + status_code=status_code, + ).inc() + except Exception as e: + self._handle_metric_error("record_api_request", e) + + def record_messages_created( + self, + *, + count: int, + workspace_name: str, + ) -> None: + try: + messages_created_counter.labels( + workspace_name=workspace_name, + ).inc(count) + except Exception as e: + self._handle_metric_error("record_messages_created", e) + + def record_dialectic_call( + self, + *, + workspace_name: str, + reasoning_level: str, + ) -> None: + try: + dialectic_calls_counter.labels( + workspace_name=workspace_name, + reasoning_level=reasoning_level, + ).inc() + except Exception as e: + self._handle_metric_error("record_dialectic_call", e) + + def record_deriver_queue_item( + self, + *, + count: int, + workspace_name: str, + task_type: str, + ) -> None: + try: + deriver_queue_items_processed_counter.labels( + workspace_name=workspace_name, + task_type=task_type, + ).inc(count) + except Exception as e: + self._handle_metric_error("record_deriver_queue_item", e) + + def record_deriver_tokens( + self, + *, + count: int, + task_type: str, + token_type: str, + component: str, + ) -> None: + try: + deriver_tokens_processed_counter.labels( + task_type=task_type, + token_type=token_type, + component=component, + ).inc(count) + except Exception as e: + self._handle_metric_error("record_deriver_tokens", e) + + def record_dialectic_tokens( + self, + *, + count: int, + token_type: str, + component: str, + reasoning_level: str, + ) -> None: + try: + dialectic_tokens_processed_counter.labels( + token_type=token_type, + component=component, + reasoning_level=reasoning_level, + ).inc(count) + except Exception as e: + self._handle_metric_error("record_dialectic_tokens", e) + + def record_dreamer_tokens( + self, + *, + count: int, + specialist_name: str, + token_type: str, + ) -> None: + try: + dreamer_tokens_processed_counter.labels( + specialist_name=specialist_name, + token_type=token_type, + ).inc(count) + except Exception as e: + self._handle_metric_error("record_dreamer_tokens", e) + + +prometheus_metrics = PrometheusMetrics() + + +async def metrics_endpoint(_request: Request) -> Response: + if not settings.METRICS.ENABLED: + return Response("Metrics are disabled", status_code=404) + try: + return Response( + content=generate_latest(REGISTRY), + media_type=CONTENT_TYPE_LATEST, + ) + except Exception as e: + logger.error(f"Failed to generate metrics: {e}", exc_info=True) + return Response("Failed to generate metrics", status_code=500) diff --git a/src/utils/agent_tools.py b/src/utils/agent_tools.py index 5c8df29b..c3577c67 100644 --- a/src/utils/agent_tools.py +++ b/src/utils/agent_tools.py @@ -558,7 +558,7 @@ async def create_observations( observations: list[dict[str, Any]], observer: str, observed: str, - session_name: str, + session_name: str | None, workspace_name: str, message_ids: list[int], message_created_at: str, @@ -941,12 +941,9 @@ async def _handle_create_observations( message_ids = [msg.id for msg in ctx.current_messages] message_created_at = str(ctx.current_messages[-1].created_at) - obs_session_name = ctx.session_name or ctx.current_messages[0].session_name + obs_session_name = ctx.session_name else: # Dreamer/Dialectic agent: allow deductive and inductive, no source messages - if not ctx.session_name: - return "ERROR: Cannot create observations without a session context" - for i, obs in enumerate(observations): if "content" not in obs: return f"ERROR: observation {i} missing 'content' field" diff --git a/src/utils/files.py b/src/utils/files.py index 1cc40ba1..4dff50f4 100644 --- a/src/utils/files.py +++ b/src/utils/files.py @@ -3,7 +3,6 @@ import logging from io import BytesIO from typing import Any, Protocol -import pdfplumber from fastapi import UploadFile from nanoid import generate as generate_nanoid from sqlalchemy import Integer, select @@ -27,6 +26,8 @@ class PDFProcessor: return content_type == "application/pdf" async def extract_text(self, content: bytes) -> str: + import pdfplumber + with pdfplumber.open(BytesIO(content)) as pdf_reader: text_parts: list[str] = [] for page_num, page in enumerate(pdf_reader.pages): diff --git a/src/utils/queue_payload.py b/src/utils/queue_payload.py index 9adeae92..c593cf94 100644 --- a/src/utils/queue_payload.py +++ b/src/utils/queue_payload.py @@ -56,7 +56,7 @@ class DreamPayload(BasePayload): dream_type: DreamType observer: str observed: str - session_name: str + session_name: str | None = None class DeletionPayload(BasePayload): @@ -89,7 +89,7 @@ def create_dream_payload( *, observer: str, observed: str, - session_name: str, + session_name: str | None = None, ) -> dict[str, Any]: """Create a dream payload.""" return DreamPayload( diff --git a/src/utils/representation.py b/src/utils/representation.py index 7f63e000..674b25bb 100644 --- a/src/utils/representation.py +++ b/src/utils/representation.py @@ -53,7 +53,7 @@ class ObservationMetadata(BaseModel): id: str = Field(default="", description="Document ID for this observation") created_at: datetime message_ids: list[int] - session_name: str + session_name: str | None = None class ExplicitObservationBase(BaseModel): diff --git a/src/utils/summarizer.py b/src/utils/summarizer.py index e7fec89d..46415a3a 100644 --- a/src/utils/summarizer.py +++ b/src/utils/summarizer.py @@ -16,10 +16,14 @@ from src.crud.session import session_cache_key from src.dependencies import tracked_db from src.exceptions import ResourceNotFoundException from src.models import Message -from src.telemetry import otel_metrics +from src.telemetry import prometheus_metrics from src.telemetry.events import AgentToolSummaryCreatedEvent, emit from src.telemetry.logging import accumulate_metric, conditional_observe -from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes +from src.telemetry.prometheus.metrics import ( + DeriverComponents, + DeriverTaskTypes, + TokenTypes, +) from src.utils.clients import HonchoLLMCallResponse, honcho_llm_call from src.utils.formatting import utc_now_iso from src.utils.tokens import estimate_tokens, track_deriver_input_tokens @@ -449,8 +453,8 @@ async def _create_and_save_summary( ) # Track output tokens - if settings.OTEL.ENABLED: - otel_metrics.record_deriver_tokens( + if settings.METRICS.ENABLED: + prometheus_metrics.record_deriver_tokens( count=new_summary["token_count"], task_type=DeriverTaskTypes.SUMMARY.value, token_type=TokenTypes.OUTPUT.value, diff --git a/src/utils/tokens.py b/src/utils/tokens.py index 4e71e3fa..0a08702d 100644 --- a/src/utils/tokens.py +++ b/src/utils/tokens.py @@ -1,8 +1,12 @@ import tiktoken from src.config import settings -from src.telemetry import otel_metrics -from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes +from src.telemetry import prometheus_metrics +from src.telemetry.prometheus.metrics import ( + DeriverComponents, + DeriverTaskTypes, + TokenTypes, +) tokenizer = tiktoken.get_encoding("o200k_base") @@ -31,9 +35,9 @@ def track_deriver_input_tokens( components: Dict mapping component names to token counts """ for component, token_count in components.items(): - # OTel metrics (push-based) - if settings.OTEL.ENABLED: - otel_metrics.record_deriver_tokens( + # Prometheus metrics + if settings.METRICS.ENABLED: + prometheus_metrics.record_deriver_tokens( count=token_count, task_type=task_type.value, token_type=TokenTypes.INPUT.value, diff --git a/src/vector_store/__init__.py b/src/vector_store/__init__.py index 7a04eaa2..c77064a7 100644 --- a/src/vector_store/__init__.py +++ b/src/vector_store/__init__.py @@ -192,17 +192,18 @@ class VectorStore(ABC): ... -# Import implementations after base classes are defined to avoid circular imports -from src.vector_store.lancedb import LanceDBVectorStore # noqa: E402 -from src.vector_store.turbopuffer import TurbopufferVectorStore # noqa: E402 from src.vector_store.utils import upsert_with_retry # noqa: E402 def _create_store_by_type(store_type: str) -> VectorStore: """Create a vector store instance by type name.""" if store_type == "turbopuffer": + from src.vector_store.turbopuffer import TurbopufferVectorStore + return TurbopufferVectorStore() elif store_type == "lancedb": + from src.vector_store.lancedb import LanceDBVectorStore + return LanceDBVectorStore() else: raise ValueError(f"Unknown vector store type: {store_type}") diff --git a/tests/alembic/revisions/__init__.py b/tests/alembic/revisions/__init__.py index c5da5e8a..77c15483 100644 --- a/tests/alembic/revisions/__init__.py +++ b/tests/alembic/revisions/__init__.py @@ -22,6 +22,7 @@ from . import ( test_bb6fb3a7a643_add_message_seq_in_session_column, test_c3828084f472_add_indexes_for_messages_and_, test_d429de0e5338_adopt_peer_paradigm, + test_e4eba9cfaa6f_make_document_session_name_nullable, test_e9b705f9adf9_add_server_defaults_to_timestamp_, test_ec8f94139b02_codify_workspace_name_and_message_id_in_, test_f1a2b3c4d5e6_add_reasoning_tree_columns, @@ -49,6 +50,7 @@ __all__ = [ "test_bb6fb3a7a643_add_message_seq_in_session_column", "test_c3828084f472_add_indexes_for_messages_and_", "test_d429de0e5338_adopt_peer_paradigm", + "test_e4eba9cfaa6f_make_document_session_name_nullable", "test_e9b705f9adf9_add_server_defaults_to_timestamp_", "test_ec8f94139b02_codify_workspace_name_and_message_id_in_", "test_f1a2b3c4d5e6_add_reasoning_tree_columns", diff --git a/tests/alembic/revisions/test_e4eba9cfaa6f_make_document_session_name_nullable.py b/tests/alembic/revisions/test_e4eba9cfaa6f_make_document_session_name_nullable.py new file mode 100644 index 00000000..32b46647 --- /dev/null +++ b/tests/alembic/revisions/test_e4eba9cfaa6f_make_document_session_name_nullable.py @@ -0,0 +1,141 @@ +"""Hooks for revision e4eba9cfaa6f (make_document_session_name_nullable).""" + +from __future__ import annotations + +from nanoid import generate as generate_nanoid +from sqlalchemy import text + +from tests.alembic.registry import register_after_upgrade, register_before_upgrade +from tests.alembic.verifier import MigrationVerifier + +# Test IDs for seeding data +WORKSPACE_NAME = generate_nanoid() +PEER_NAME = generate_nanoid() +SESSION_NAME = generate_nanoid() +DOCUMENT_ID = generate_nanoid() + + +@register_before_upgrade("e4eba9cfaa6f") +def prepare_make_document_session_name_nullable(verifier: MigrationVerifier) -> None: + """Seed state and assertions before upgrading to e4eba9cfaa6f.""" + # Verify session_name column is NOT nullable before migration + verifier.assert_column_exists("documents", "session_name", nullable=False) + + schema = verifier.schema + connection = verifier.conn + + # Seed workspace + connection.execute( + text( + f""" + INSERT INTO "{schema}"."workspaces" ("id", "name") + VALUES (:id, :name) + """ + ), + {"id": generate_nanoid(), "name": WORKSPACE_NAME}, + ) + + # Seed peer + connection.execute( + text( + f""" + INSERT INTO "{schema}"."peers" ("id", "name", "workspace_name") + VALUES (:id, :name, :workspace_name) + """ + ), + {"id": generate_nanoid(), "name": PEER_NAME, "workspace_name": WORKSPACE_NAME}, + ) + + # Seed session + connection.execute( + text( + f""" + INSERT INTO "{schema}"."sessions" ("id", "name", "workspace_name", "is_active") + VALUES (:id, :name, :workspace_name, true) + """ + ), + { + "id": generate_nanoid(), + "name": SESSION_NAME, + "workspace_name": WORKSPACE_NAME, + }, + ) + + # Seed collection + connection.execute( + text( + f""" + INSERT INTO "{schema}"."collections" + ("id", "workspace_name", "observer", "observed") + VALUES (:id, :workspace_name, :observer, :observed) + """ + ), + { + "id": generate_nanoid(), + "workspace_name": WORKSPACE_NAME, + "observer": PEER_NAME, + "observed": PEER_NAME, + }, + ) + + # Seed document with session_name (required before migration) + connection.execute( + text( + f""" + INSERT INTO "{schema}"."documents" + ("id", "workspace_name", "observer", "observed", "content", "session_name") + VALUES (:id, :workspace_name, :observer, :observed, :content, :session_name) + """ + ), + { + "id": DOCUMENT_ID, + "workspace_name": WORKSPACE_NAME, + "observer": PEER_NAME, + "observed": PEER_NAME, + "content": "Test document with session", + "session_name": SESSION_NAME, + }, + ) + + +@register_after_upgrade("e4eba9cfaa6f") +def verify_make_document_session_name_nullable(verifier: MigrationVerifier) -> None: + """Add assertions validating the effects of e4eba9cfaa6f.""" + schema = verifier.schema + connection = verifier.conn + + # Verify session_name column IS nullable after migration + verifier.assert_column_exists("documents", "session_name", nullable=True) + + # Verify existing document still has its session_name intact + row = connection.execute( + text(f'SELECT "session_name" FROM "{schema}"."documents" WHERE "id" = :id'), + {"id": DOCUMENT_ID}, + ).one() + assert row.session_name == SESSION_NAME + + # Verify we can now insert a document without session_name (NULL) + null_session_doc_id = generate_nanoid() + connection.execute( + text( + f""" + INSERT INTO "{schema}"."documents" + ("id", "workspace_name", "observer", "observed", "content", "session_name") + VALUES (:id, :workspace_name, :observer, :observed, :content, NULL) + """ + ), + { + "id": null_session_doc_id, + "workspace_name": WORKSPACE_NAME, + "observer": PEER_NAME, + "observed": PEER_NAME, + "content": "Test document without session (global)", + }, + ) + + # Verify the NULL was persisted + null_row = connection.execute( + text(f'SELECT "session_name" FROM "{schema}"."documents" WHERE "id" = :id'), + {"id": null_session_doc_id}, + ).one() + assert null_row.session_name is None diff --git a/tests/bench/beam.py b/tests/bench/beam.py index 352106f5..b61c0e81 100644 --- a/tests/bench/beam.py +++ b/tests/bench/beam.py @@ -66,7 +66,6 @@ Optional arguments: import argparse import asyncio import os -import time from datetime import datetime from pathlib import Path from typing import Any, cast @@ -82,7 +81,6 @@ from .beam_common import ( ConversationResult, QuestionResult, calculate_ability_scores, - format_duration, generate_json_summary, judge_event_ordering, judge_nugget_based, @@ -91,11 +89,11 @@ from .beam_common import ( print_summary, ) from .runner_common import ( - ReasoningLevel, - RunnerMixin, + BaseRunner, + ItemContext, + RunnerConfig, add_common_arguments, create_openai_client, - export_metrics, validate_common_arguments, ) @@ -104,48 +102,33 @@ bench_dir = Path(__file__).parent load_dotenv(bench_dir / ".env") -class BEAMRunner(RunnerMixin): +class BEAMRunner(BaseRunner[ConversationResult]): """ Executes BEAM benchmark tests against a Honcho instance. """ def __init__( self, + config: RunnerConfig, data_dir: Path, - base_api_port: int = 8000, - pool_size: int = 1, - timeout_seconds: int | None = None, - cleanup_workspace: bool = True, - use_get_context: bool = False, - redis_url: str = "redis://localhost:6379/0", - reasoning_level: ReasoningLevel | None = None, + context_length: str, + conversation_ids: list[str] | None = None, ): """ Initialize the BEAM test runner. Args: + config: Common runner configuration data_dir: Path to the BEAM data directory - base_api_port: Base port for Honcho API instances (default: 8000) - pool_size: Number of Honcho instances in the pool (default: 1) - timeout_seconds: Timeout for deriver queue in seconds - cleanup_workspace: If True, delete workspace after executing conversation - use_get_context: If True, use get_context + judge LLM instead of dialectic .chat endpoint - redis_url: Redis URL for flush mode signaling (default: redis://localhost:6379/0) - reasoning_level: Reasoning level for dialectic chat (default: None) + context_length: Context length (100K, 500K, 1M, 10M) + conversation_ids: Optional list of specific conversation IDs to run """ self.data_dir: Path = data_dir - self.base_api_port: int = base_api_port - self.pool_size: int = pool_size - self.timeout_seconds: int = ( - timeout_seconds if timeout_seconds is not None else 600 - ) - self.cleanup_workspace: bool = cleanup_workspace - self.use_get_context: bool = use_get_context - self.redis_url: str = redis_url - self.reasoning_level: ReasoningLevel | None = reasoning_level + self.context_length: str = context_length + self.conversation_ids: list[str] | None = conversation_ids - # Initialize common components (metrics, logging) - self._init_common("beam") + # Initialize base class + super().__init__(config) # Initialize OpenRouter client for judging openrouter_base_url = os.getenv( @@ -161,10 +144,122 @@ class BEAMRunner(RunnerMixin): "BEAM_JUDGE_MODEL", "anthropic/claude-sonnet-4.5" ) + def get_metrics_prefix(self) -> str: + return "beam" + + def load_items(self) -> list[Any]: + """Load conversation IDs to process.""" + if self.conversation_ids: + return self.conversation_ids + return list_conversations(self.data_dir, self.context_length) + + def get_workspace_id(self, item: Any) -> str: + """Return workspace ID for a conversation.""" + return f"beam_{self.context_length}_{item}" + + def get_session_id(self, item: Any, workspace_id: str) -> str: + """Return session ID for a conversation.""" + return f"{workspace_id}_session" + + async def setup_peers(self, ctx: ItemContext, item: Any) -> None: + """Create user and assistant peers.""" + ctx.peers["user"] = await ctx.honcho_client.aio.peer(id="user") + ctx.peers["assistant"] = await ctx.honcho_client.aio.peer(id="assistant") + + async def setup_session(self, ctx: ItemContext, item: Any) -> None: + """Create and configure session - observe the user peer.""" + user_peer = ctx.peers["user"] + assistant_peer = ctx.peers["assistant"] + + ctx.session = await ctx.honcho_client.aio.session( + id=ctx.session_id, configuration=self._get_session_configuration() + ) + + await ctx.session.aio.add_peers( + [ + (user_peer, SessionPeerConfig(observe_me=True, observe_others=False)), + ( + assistant_peer, + SessionPeerConfig(observe_me=False, observe_others=False), + ), + ] + ) + + async def ingest_messages(self, ctx: ItemContext, item: Any) -> int: + """Ingest conversation turns into the session.""" + conversation_id = item + user_peer = ctx.peers["user"] + assistant_peer = ctx.peers["assistant"] + + # Load conversation data + conv_data = load_conversation( + self.data_dir, self.context_length, conversation_id + ) + chat_data = conv_data["chat"] + + # Store questions data for later use + ctx.peers["_questions_data"] = conv_data["questions"] + + messages: list[MessageCreateParams] = [] + + # Handle different data structures for 10M vs other sizes + for batch in chat_data: + if any(key.startswith("plan-") for key in batch): + # 10M structure: { "plan-1": [...], "plan-2": [...], ... } + for plan_name, plan_batches in batch.items(): + if not plan_name.startswith("plan-"): + continue + for plan_batch in plan_batches: + for turn_group in plan_batch.get("turns", []): + for turn in turn_group: + messages.extend( + self._process_turn(turn, user_peer, assistant_peer) + ) + else: + # Standard structure for 100K, 500K, 1M + for turn_group in batch.get("turns", []): + for turn in turn_group: + messages.extend( + self._process_turn(turn, user_peer, assistant_peer) + ) + + # Add messages in batches of 100 + for i in range(0, len(messages), 100): + batch = messages[i : i + 100] + await ctx.session.aio.add_messages(batch) + + return len(messages) + + def _process_turn( + self, turn: dict[str, Any], user_peer: Any, assistant_peer: Any + ) -> list[MessageCreateParams]: + """Process a single turn, handling long message splitting.""" + role = turn["role"] + content = turn["content"] + messages: list[MessageCreateParams] = [] + + if len(content) > 25000: + chunks = [content[i : i + 25000] for i in range(0, len(content), 25000)] + for chunk in chunks: + if role == "user": + messages.append(user_peer.message(chunk)) + elif role == "assistant": + messages.append(assistant_peer.message(chunk)) + else: + if role == "user": + messages.append(user_peer.message(content)) + elif role == "assistant": + messages.append(assistant_peer.message(content)) + + return messages + + def get_dream_observers(self, item: Any) -> list[str]: + """Return the observer - always user for BEAM.""" + return ["user"] + async def _process_single_question( self, - session: Any, - user_peer: Any, + ctx: ItemContext, ability: str, q_idx: int, q_data: dict[str, Any], @@ -184,18 +279,16 @@ class BEAMRunner(RunnerMixin): # Execute question using dialectic # For instruction_following, always use get_context + OpenRouter API - # so the LLM can follow user-specified instructions from Honcho context - if self.use_get_context or ability == "instruction_following": - context = await session.aio.context( + if self.config.use_get_context or ability == "instruction_following": + context = await ctx.session.aio.context( summary=True, peer_target="user", - last_user_message=question, + search_query=question, ) context_messages = context.to_openai(assistant="assistant") context_messages.append({"role": "user", "content": question}) - # For instruction_following, add a system prompt that tells the LLM - # to follow any stored user preferences/instructions in the context + # For instruction_following, add a system prompt system_prompt = None if ability == "instruction_following": system_prompt = """You are a helpful assistant with memory of the user's preferences and instructions from previous conversations. @@ -209,7 +302,6 @@ IMPORTANT: You MUST follow any instructions or preferences the user has previous Review the context carefully for any such instructions before responding.""" - # Prepare messages: OpenAI format uses system role in messages array messages: list[dict[str, Any]] = [] if system_prompt: messages.append({"role": "system", "content": system_prompt}) @@ -218,7 +310,7 @@ Review the context carefully for any such instructions before responding.""" response = await self.openrouter_client.chat.completions.create( model=self.judge_model, max_tokens=settings.DIALECTIC.MAX_OUTPUT_TOKENS, - messages=cast(Any, messages), # type: ignore[arg-type] + messages=cast(Any, messages), ) if not response.choices or not response.choices[0].message: @@ -226,9 +318,9 @@ Review the context carefully for any such instructions before responding.""" else: actual_response = response.choices[0].message.content or "" else: - actual_response = await user_peer.aio.chat( + actual_response = await ctx.peers["user"].aio.chat( question, - reasoning_level=self.reasoning_level, + reasoning_level=self.config.reasoning_level, ) actual_response = ( actual_response if isinstance(actual_response, str) else "" @@ -283,35 +375,16 @@ Review the context carefully for any such instructions before responding.""" return question_result - async def execute_conversation( - self, context_length: str, conversation_id: str, honcho_url: str + async def execute_questions( + self, ctx: ItemContext, item: Any ) -> ConversationResult: - """ - Execute BEAM benchmark for a single conversation. - - Args: - context_length: Context length (100K, 500K, 1M, 10M) - conversation_id: Conversation ID - honcho_url: URL of the Honcho instance to use - - Returns: - Conversation execution results - """ - start_time = time.time() - - print(f"\n{'=' * 80}") - print( - f"Executing BEAM conversation {conversation_id} ({context_length} context)" - ) - print(f"{'=' * 80}") - - # Create workspace for this conversation - workspace_id = f"beam_{context_length}_{conversation_id}" - honcho_client = self.create_honcho_client(workspace_id, honcho_url) + """Execute all questions for the conversation.""" + conversation_id = item + workspace_id = ctx.workspace_id result: ConversationResult = { "conversation_id": conversation_id, - "context_length": context_length, + "context_length": self.context_length, "workspace_id": workspace_id, "total_turns": 0, "total_messages": 0, @@ -319,267 +392,78 @@ Review the context carefully for any such instructions before responding.""" "ability_scores": {}, "overall_score": 0.0, "error": None, - "start_time": start_time, + "start_time": 0.0, "end_time": 0.0, "duration_seconds": 0.0, } - try: - # Load conversation data - conv_data = load_conversation( - self.data_dir, context_length, conversation_id - ) - chat_data = conv_data["chat"] - questions_data = conv_data["questions"] + questions_data = ctx.peers.get("_questions_data", {}) - # Create peers - user_peer = await honcho_client.aio.peer(id="user") - assistant_peer = await honcho_client.aio.peer(id="assistant") + # Execute questions for each memory ability + question_tasks: list[Any] = [] + semaphore = asyncio.Semaphore(5) - # Create session for this conversation - session_id = f"{workspace_id}_session" - session = await honcho_client.aio.session(id=session_id) + for ability, questions in questions_data.items(): + print(f"\n[{workspace_id}] Queuing {ability} ({len(questions)} questions)") - # Configure peer observation - observe the user peer - await session.aio.add_peers( - [ - ( - user_peer, - SessionPeerConfig(observe_me=True, observe_others=False), - ), - ( - assistant_peer, - SessionPeerConfig(observe_me=False, observe_others=False), - ), - ] - ) - - # Ingest conversation turns - print(f"[{workspace_id}] Ingesting conversation turns...") - messages: list[MessageCreateParams] = [] - - # Handle different data structures for 10M vs other sizes - for batch in chat_data: - # Check if this is a 10M conversation with plan-based structure - if any(key.startswith("plan-") for key in batch): - # 10M structure: { "plan-1": [...], "plan-2": [...], ... } - for plan_name, plan_batches in batch.items(): - if not plan_name.startswith("plan-"): - continue - for plan_batch in plan_batches: - for turn_group in plan_batch.get("turns", []): - for turn in turn_group: - role = turn["role"] - content = turn["content"] - result["total_turns"] += 1 - - # Split message if it exceeds 25000 characters - if len(content) > 25000: - chunks = [ - content[i : i + 25000] - for i in range(0, len(content), 25000) - ] - for chunk in chunks: - if role == "user": - messages.append( - user_peer.message(chunk) - ) - elif role == "assistant": - messages.append( - assistant_peer.message(chunk) - ) - else: - if role == "user": - messages.append(user_peer.message(content)) - elif role == "assistant": - messages.append( - assistant_peer.message(content) - ) - else: - # Standard structure for 100K, 500K, 1M - for turn_group in batch.get("turns", []): - for turn in turn_group: - role = turn["role"] - content = turn["content"] - result["total_turns"] += 1 - - # Split message if it exceeds 25000 characters - if len(content) > 25000: - chunks = [ - content[i : i + 25000] - for i in range(0, len(content), 25000) - ] - for chunk in chunks: - if role == "user": - messages.append(user_peer.message(chunk)) - elif role == "assistant": - messages.append(assistant_peer.message(chunk)) - else: - if role == "user": - messages.append(user_peer.message(content)) - elif role == "assistant": - messages.append(assistant_peer.message(content)) - - result["total_messages"] = len(messages) - - # Add messages in batches of 100 - for i in range(0, len(messages), 100): - batch = messages[i : i + 100] - await session.aio.add_messages(batch) - - print( - f"[{workspace_id}] Ingested {result['total_messages']} messages. Waiting for deriver queue..." - ) - - # Wait for deriver queue to empty - await asyncio.sleep(1) - await self.flush_deriver_queue() - queue_empty = await self.wait_for_deriver_queue_empty(honcho_client) - if not queue_empty: - result["error"] = "Deriver queue timeout" - result["end_time"] = time.time() - result["duration_seconds"] = result["end_time"] - result["start_time"] - print( - f"\n[{workspace_id}] ERROR: Deriver queue timeout after {self.timeout_seconds}s" - ) - print( - f"[{workspace_id}] Failed to complete in {format_duration(result['duration_seconds'])}" - ) - return result - - print(f"[{workspace_id}] Deriver queue empty. Triggering dream...") - - # Single orchestrated dream handles all reasoning types - dream_success = await self.trigger_dream_and_wait( - honcho_client, - workspace_id, - observer="user", - session_id=session_id, - ) - if not dream_success: - print(f"[{workspace_id}] Warning: Dream did not complete") - print(f"[{workspace_id}] Dream completed. Executing questions...") - - # Execute questions for each memory ability - question_tasks: list[Any] = [] - semaphore = asyncio.Semaphore(5) - - for ability, questions in questions_data.items(): - print( - f"\n[{workspace_id}] Queuing {ability} ({len(questions)} questions)" - ) - - for q_idx, q_data in enumerate(questions): - question_tasks.append( - self._process_single_question( - session, - user_peer, - ability, - q_idx, - q_data, - semaphore, - ) + for q_idx, q_data in enumerate(questions): + question_tasks.append( + self._process_single_question( + ctx, ability, q_idx, q_data, semaphore ) - - results = await asyncio.gather(*question_tasks) - result["question_results"] = list(results) - - # Calculate ability scores - result["ability_scores"] = calculate_ability_scores( - result["question_results"] - ) - - # Calculate overall score - if result["ability_scores"]: - result["overall_score"] = sum(result["ability_scores"].values()) / len( - result["ability_scores"] ) - # Cleanup workspace if requested - if self.cleanup_workspace: - try: - await honcho_client.aio.delete_workspace(workspace_id) - print(f"[{workspace_id}] Cleaned up workspace") - except Exception as e: - print(f"Failed to delete workspace: {e}") + results = await asyncio.gather(*question_tasks) + result["question_results"] = list(results) - result["end_time"] = time.time() - result["duration_seconds"] = result["end_time"] - result["start_time"] + # Calculate ability scores + result["ability_scores"] = calculate_ability_scores(result["question_results"]) - print( - f"\n[{workspace_id}] Completed in {format_duration(result['duration_seconds'])}" + # Calculate overall score + if result["ability_scores"]: + result["overall_score"] = sum(result["ability_scores"].values()) / len( + result["ability_scores"] ) - print(f"Overall Score: {result['overall_score']:.3f}") - except Exception as e: - self.logger.error(f"Error executing conversation {conversation_id}: {e}") - result["error"] = str(e) - result["end_time"] = time.time() - result["duration_seconds"] = result["end_time"] - result["start_time"] + print(f"\nOverall Score: {result['overall_score']:.3f}") return result - async def run_conversations( - self, - context_length: str, - conversation_ids: list[str], - batch_size: int = 1, - ) -> tuple[list[ConversationResult], float]: - """ - Run multiple conversations from the BEAM benchmark. + def print_summary( + self, results: list[ConversationResult], total_duration: float + ) -> None: + """Print summary using the common function.""" + print_summary(results, total_duration) - Args: - context_length: Context length (100K, 500K, 1M, 10M) - conversation_ids: List of conversation IDs to run - batch_size: Number of conversations to run concurrently in each batch + def generate_output( + self, results: list[ConversationResult], total_duration: float + ) -> None: + """Generate JSON output file.""" + if self.config.json_output: + output_file = self.config.json_output + else: + output_file = Path( + f"tests/bench/eval_results/beam_{self.context_length}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) - Returns: - Tuple of (list of conversation results, total duration) - """ - print( - f"Running {len(conversation_ids)} conversations from {context_length} context length" + generate_json_summary( + results, + self.context_length, + total_duration, + output_file, + metadata_extra={ + "base_api_port": self.config.base_api_port, + "pool_size": self.config.pool_size, + "timeout_seconds": self.config.timeout_seconds, + "reasoning_level": self.config.reasoning_level, + "deriver_settings": settings.DERIVER.model_dump(), + "dialectic_settings": settings.DIALECTIC.model_dump(), + "dream_settings": settings.DREAM.model_dump(), + }, ) - if self.pool_size > 1: - print( - f"Distributing conversations across {self.pool_size} Honcho instances" - ) - - overall_start = time.time() - all_results: list[ConversationResult] = [] - - for i in range(0, len(conversation_ids), batch_size): - batch = conversation_ids[i : i + batch_size] - batch_num = (i // batch_size) + 1 - total_batches = (len(conversation_ids) + batch_size - 1) // batch_size - - print(f"\n{'=' * 80}") - print( - f"Processing batch {batch_num}/{total_batches} ({len(batch)} conversations)" - ) - print(f"{'=' * 80}") - - # Run conversations in current batch concurrently - batch_results: list[ConversationResult] = await asyncio.gather( - *[ - self.execute_conversation( - context_length, conv_id, self.get_honcho_url_for_index(i + idx) - ) - for idx, conv_id in enumerate(batch) - ] - ) - - all_results.extend(batch_results) - - overall_end = time.time() - overall_duration = overall_end - overall_start - - # Finalize metrics collection - self.metrics_collector.finalize_collection() - - return all_results, overall_duration -async def main() -> int: +def main() -> int: """Main entry point for the BEAM test runner.""" parser = argparse.ArgumentParser( description="Run BEAM benchmark tests against a Honcho instance", @@ -624,72 +508,23 @@ Examples: print(f"Error: BEAM data directory not found at {data_dir}") return 1 - # Create runner + # Parse conversation IDs + conversation_ids = None + if args.conversation_ids: + conversation_ids = args.conversation_ids.split(",") + + # Create config and runner + config = RunnerConfig.from_args(args, default_timeout=600) + runner = BEAMRunner( + config=config, data_dir=data_dir, - base_api_port=args.base_api_port, - pool_size=args.pool_size, - timeout_seconds=args.timeout, - cleanup_workspace=args.cleanup_workspace, - use_get_context=args.use_get_context, - redis_url=args.redis_url, - reasoning_level=args.reasoning_level, + context_length=args.context_length, + conversation_ids=conversation_ids, ) - try: - # Determine which conversations to run - if args.conversation_ids: - conversation_ids = args.conversation_ids.split(",") - else: - conversation_ids = list_conversations(data_dir, args.context_length) - - # Run conversations - results, total_elapsed = await runner.run_conversations( - args.context_length, conversation_ids, args.batch_size - ) - - print_summary(results, total_elapsed) - - # Generate JSON output - if args.json_output: - output_file = args.json_output - else: - output_file = Path( - f"tests/bench/eval_results/beam_{args.context_length}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - ) - - generate_json_summary( - results, - args.context_length, - total_elapsed, - output_file, - metadata_extra={ - "base_api_port": runner.base_api_port, - "pool_size": runner.pool_size, - "timeout_seconds": runner.timeout_seconds, - "reasoning_level": runner.reasoning_level, - "deriver_settings": settings.DERIVER.model_dump(), - "dialectic_settings": settings.DIALECTIC.model_dump(), - "dream_settings": settings.DREAM.model_dump(), - }, - ) - - # Export metrics - export_metrics(runner.metrics_collector, "beam") - - return 0 - - except KeyboardInterrupt: - print("\nTest execution interrupted by user") - return 1 - except Exception as e: - print(f"Error running tests: {e}") - import traceback - - traceback.print_exc() - return 1 + return runner.run_and_summarize() if __name__ == "__main__": - exit_code = asyncio.run(main()) - exit(exit_code) + exit(main()) diff --git a/tests/bench/calculate_expected_events.py b/tests/bench/calculate_expected_events.py new file mode 100644 index 00000000..348cd95c --- /dev/null +++ b/tests/bench/calculate_expected_events.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +""" +Calculate expected CloudEvents and input tokens from longmemeval test case files. + +This script parses longmemeval test files (like longmemeval_sanity.json, longmemeval_oracle.json) +and calculates the expected number of CloudEvents and input tokens that should be processed +when running the test. + +Supported test files: +- longmemeval_sanity.json (5 questions, quick verification) +- longmemeval_oracle.json (full evaluation set) +- longmemeval_oracle_100.json (100-question subset) +- longmemeval_s_cleaned.json (cleaned dataset) + +Event types calculated: +- representation.completed: One per (session, observed) batch processed by deriver +- dialectic.completed: One per dialectic chat query +- dream.run: One per dream consolidation trigger + +Token counting: +- Uses tiktoken (o200k_base encoding) for accurate token estimation +- Falls back to character/4 estimation if tiktoken unavailable + +Usage: + python calculate_expected_events.py [options] + +Examples: + # Calculate events for sanity test (default: separate sessions, dialectic chat, with dream) + python calculate_expected_events.py longmemeval_data/longmemeval_sanity.json + + # Calculate events for full oracle test + python calculate_expected_events.py longmemeval_data/longmemeval_oracle.json + + # Calculate with merged sessions + python calculate_expected_events.py longmemeval_data/longmemeval_sanity.json --merge-sessions + + # Calculate without dream events + python calculate_expected_events.py longmemeval_data/longmemeval_sanity.json --no-dream + + # Calculate with get_context instead of dialectic (no dialectic events) + python calculate_expected_events.py longmemeval_data/longmemeval_sanity.json --use-get-context + + # Calculate for first 10 questions only + python calculate_expected_events.py longmemeval_data/longmemeval_oracle.json --test-count 10 + + # Output as JSON for programmatic use + python calculate_expected_events.py longmemeval_data/longmemeval_sanity.json --json +""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# Token counting setup +try: + import tiktoken + + _tokenizer = tiktoken.get_encoding("o200k_base") + _USE_TIKTOKEN = True +except ImportError: + _tokenizer = None + _USE_TIKTOKEN = False # pyright: ignore[reportConstantRedefinition] + + +def estimate_tokens(text: str) -> int: + """Estimate token count using tiktoken, with fallback to char/4.""" + if not text: + return 0 + if _USE_TIKTOKEN and _tokenizer is not None: + try: + return len(_tokenizer.encode(text)) + except Exception: + pass + # Fallback: rough estimate of 4 chars per token + return len(text) // 4 + + +@dataclass +class QuestionEventCounts: + """Event counts and token metrics for a single question.""" + + question_id: str + question_type: str + sessions_count: int + total_messages: int + observed_peer: str + representation_events: int + dialectic_events: int + dream_events: int + # Token counts + total_input_tokens: int = 0 + user_tokens: int = 0 + assistant_tokens: int = 0 + question_tokens: int = 0 + + @property + def total_events(self) -> int: + return self.representation_events + self.dialectic_events + self.dream_events + + +@dataclass +class TestFileEventSummary: + """Summary of expected events for a test file.""" + + file_path: str + total_questions: int + merge_sessions: bool + use_get_context: bool + dream_enabled: bool + questions: list[QuestionEventCounts] = field(default_factory=list) + + @property + def total_representation_events(self) -> int: + return sum(q.representation_events for q in self.questions) + + @property + def total_dialectic_events(self) -> int: + return sum(q.dialectic_events for q in self.questions) + + @property + def total_dream_events(self) -> int: + return sum(q.dream_events for q in self.questions) + + @property + def total_events(self) -> int: + return ( + self.total_representation_events + + self.total_dialectic_events + + self.total_dream_events + ) + + @property + def total_input_tokens(self) -> int: + return sum(q.total_input_tokens for q in self.questions) + + @property + def total_user_tokens(self) -> int: + return sum(q.user_tokens for q in self.questions) + + @property + def total_assistant_tokens(self) -> int: + return sum(q.assistant_tokens for q in self.questions) + + @property + def total_question_tokens(self) -> int: + return sum(q.question_tokens for q in self.questions) + + def by_question_type(self) -> dict[str, dict[str, int]]: + """Group event counts by question type.""" + result: dict[str, dict[str, int]] = {} + for q in self.questions: + if q.question_type not in result: + result[q.question_type] = { + "questions": 0, + "representation_events": 0, + "dialectic_events": 0, + "dream_events": 0, + "total_events": 0, + "total_input_tokens": 0, + "user_tokens": 0, + "assistant_tokens": 0, + } + result[q.question_type]["questions"] += 1 + result[q.question_type]["representation_events"] += q.representation_events + result[q.question_type]["dialectic_events"] += q.dialectic_events + result[q.question_type]["dream_events"] += q.dream_events + result[q.question_type]["total_events"] += q.total_events + result[q.question_type]["total_input_tokens"] += q.total_input_tokens + result[q.question_type]["user_tokens"] += q.user_tokens + result[q.question_type]["assistant_tokens"] += q.assistant_tokens + return result + + +def count_messages_by_role( + session_messages: list[dict[str, Any]], +) -> dict[str, int]: + """Count messages by role in a session.""" + counts = {"user": 0, "assistant": 0} + for msg in session_messages: + role = msg.get("role", "") + if role in counts: + counts[role] += 1 + return counts + + +def calculate_question_events( + question_data: dict[str, Any], + merge_sessions: bool, + use_get_context: bool, + dream_enabled: bool, +) -> QuestionEventCounts: + """ + Calculate expected events and tokens for a single question. + + The calculation is based on how the longmem.py runner processes test cases: + - Creates workspace per question + - Creates sessions (merged or separate based on config) + - Adds messages to sessions (triggers representation events) + - Triggers dream consolidation (if enabled) + - Queries via dialectic chat (if not using get_context) + + Args: + question_data: Question data from the test file + merge_sessions: Whether sessions are merged into one + use_get_context: Whether using get_context instead of dialectic chat + dream_enabled: Whether dream consolidation is enabled + + Returns: + QuestionEventCounts with calculated event counts and token metrics + """ + question_id = question_data.get("question_id", "unknown") + question_type = question_data.get("question_type", "unknown") + haystack_sessions = question_data.get("haystack_sessions", []) + question_text = question_data.get("question", "") + + # Determine which peer is observed based on question type + # single-session-assistant observes assistant, all others observe user + is_assistant_type = question_type == "single-session-assistant" + observed_peer = "assistant" if is_assistant_type else "user" + + # Count total messages and tokens + total_messages = 0 + user_tokens = 0 + assistant_tokens = 0 + + for session in haystack_sessions: + for msg in session: + total_messages += 1 + content = msg.get("content", "") + role = msg.get("role", "") + tokens = estimate_tokens(content) + if role == "user": + user_tokens += tokens + elif role == "assistant": + assistant_tokens += tokens + + # Token count for the question itself (used in dialectic) + question_tokens = estimate_tokens(question_text) + + # Total input tokens = all message content + total_input_tokens = user_tokens + assistant_tokens + + # Calculate representation events + # Each unique (session, observed) pair generates one representation event + # (assuming messages fit within REPRESENTATION_BATCH_MAX_TOKENS) + # When merge_sessions=True, all messages go into one session + if merge_sessions: + # One merged session = one representation event + representation_events = 1 + sessions_count = 1 + else: + # One representation event per session + sessions_count = len(haystack_sessions) + representation_events = sessions_count + + # Calculate dialectic events + # One per chat query (if using dialectic chat) + dialectic_events = 0 if use_get_context else 1 + + # Calculate dream events + # One per dream trigger + dream_events = 1 if dream_enabled else 0 + + return QuestionEventCounts( + question_id=question_id, + question_type=question_type, + sessions_count=sessions_count, + total_messages=total_messages, + observed_peer=observed_peer, + representation_events=representation_events, + dialectic_events=dialectic_events, + dream_events=dream_events, + total_input_tokens=total_input_tokens, + user_tokens=user_tokens, + assistant_tokens=assistant_tokens, + question_tokens=question_tokens, + ) + + +def calculate_test_file_events( + test_file: Path, + merge_sessions: bool = False, + use_get_context: bool = False, + dream_enabled: bool = True, + question_ids: list[str] | None = None, + test_count: int | None = None, +) -> TestFileEventSummary: + """ + Calculate expected events for all questions in a test file. + + Args: + test_file: Path to the test file (JSON) + merge_sessions: Whether to merge all sessions into one per question + use_get_context: Whether using get_context instead of dialectic chat + dream_enabled: Whether dream consolidation is enabled + question_ids: Optional list of specific question IDs to include + test_count: Optional limit on number of questions to process + + Returns: + TestFileEventSummary with calculated event counts + """ + with open(test_file) as f: + questions = json.load(f) + + # Filter by question_ids if specified + if question_ids: + questions = [q for q in questions if q.get("question_id") in question_ids] + + # Limit by test_count if specified + if test_count is not None: + questions = questions[:test_count] + + summary = TestFileEventSummary( + file_path=str(test_file), + total_questions=len(questions), + merge_sessions=merge_sessions, + use_get_context=use_get_context, + dream_enabled=dream_enabled, + ) + + for question_data in questions: + counts = calculate_question_events( + question_data, merge_sessions, use_get_context, dream_enabled + ) + summary.questions.append(counts) + + return summary + + +def format_tokens(tokens: int) -> str: + """Format token count with K/M suffix for readability.""" + if tokens >= 1_000_000: + return f"{tokens / 1_000_000:.1f}M" + elif tokens >= 1_000: + return f"{tokens / 1_000:.1f}K" + return str(tokens) + + +def print_summary(summary: TestFileEventSummary, verbose: bool = False) -> None: + """Print a formatted summary of expected events and tokens.""" + print(f"\n{'=' * 60}") + print(f"Expected CloudEvents for: {summary.file_path}") + print(f"{'=' * 60}") + print("\nConfiguration:") + print(f" - Merge sessions: {summary.merge_sessions}") + print(f" - Use get_context: {summary.use_get_context}") + print(f" - Dream enabled: {summary.dream_enabled}") + print( + f" - Token counting: {'tiktoken (o200k_base)' if _USE_TIKTOKEN else 'estimate (chars/4)'}" + ) + + print(f"\nTotal Questions: {summary.total_questions}") + + print(f"\n{'─' * 40}") + print("Expected Event Counts:") + print(f"{'─' * 40}") + print(f" representation.completed: {summary.total_representation_events:>6}") + print(f" dialectic.completed: {summary.total_dialectic_events:>6}") + print(f" dream.run: {summary.total_dream_events:>6}") + print(f"{'─' * 40}") + print(f" TOTAL: {summary.total_events:>6}") + print(f"{'─' * 40}") + + print(f"\n{'─' * 40}") + print("Expected Input Tokens:") + print(f"{'─' * 40}") + print(f" User messages: {summary.total_user_tokens:>12,}") + print(f" Assistant messages: {summary.total_assistant_tokens:>12,}") + print(f" Questions: {summary.total_question_tokens:>12,}") + print(f"{'─' * 40}") + print( + f" TOTAL (messages): {summary.total_input_tokens:>12,} ({format_tokens(summary.total_input_tokens)})" + ) + print(f"{'─' * 40}") + + # Breakdown by question type + by_type = summary.by_question_type() + if len(by_type) > 1: + print("\nBreakdown by Question Type:") + print(f"{'─' * 70}") + for qtype, counts in sorted(by_type.items()): + print(f"\n {qtype}:") + print(f" Questions: {counts['questions']:>4}") + print(f" representation: {counts['representation_events']:>4}") + print(f" dialectic: {counts['dialectic_events']:>4}") + print(f" dream: {counts['dream_events']:>4}") + print(f" total events: {counts['total_events']:>4}") + print( + f" input tokens: {counts['total_input_tokens']:>10,} ({format_tokens(counts['total_input_tokens'])})" + ) + + if verbose: + print(f"\n{'─' * 70}") + print("Per-Question Details:") + print(f"{'─' * 70}") + for q in summary.questions: + print(f"\n {q.question_id} ({q.question_type}):") + print(f" Sessions: {q.sessions_count}, Messages: {q.total_messages}") + print(f" Observed: {q.observed_peer}") + print( + f" Events: repr={q.representation_events}, dial={q.dialectic_events}, dream={q.dream_events}" + ) + print( + f" Tokens: user={q.user_tokens:,}, asst={q.assistant_tokens:,}, total={q.total_input_tokens:,}" + ) + + print() + + +def output_json(summary: TestFileEventSummary) -> str: + """Output summary as JSON for programmatic use.""" + return json.dumps( + { + "file_path": summary.file_path, + "configuration": { + "merge_sessions": summary.merge_sessions, + "use_get_context": summary.use_get_context, + "dream_enabled": summary.dream_enabled, + "token_counting": "tiktoken" if _USE_TIKTOKEN else "estimate", + }, + "total_questions": summary.total_questions, + "expected_events": { + "representation_completed": summary.total_representation_events, + "dialectic_completed": summary.total_dialectic_events, + "dream_run": summary.total_dream_events, + "total": summary.total_events, + }, + "expected_tokens": { + "user_messages": summary.total_user_tokens, + "assistant_messages": summary.total_assistant_tokens, + "questions": summary.total_question_tokens, + "total_messages": summary.total_input_tokens, + }, + "by_question_type": summary.by_question_type(), + "questions": [ + { + "question_id": q.question_id, + "question_type": q.question_type, + "sessions_count": q.sessions_count, + "total_messages": q.total_messages, + "observed_peer": q.observed_peer, + "representation_events": q.representation_events, + "dialectic_events": q.dialectic_events, + "dream_events": q.dream_events, + "total_events": q.total_events, + "total_input_tokens": q.total_input_tokens, + "user_tokens": q.user_tokens, + "assistant_tokens": q.assistant_tokens, + "question_tokens": q.question_tokens, + } + for q in summary.questions + ], + }, + indent=2, + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Calculate expected CloudEvents from longmemeval test case files", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "test_file", + type=Path, + help="Path to the longmemeval test file (JSON)", + ) + parser.add_argument( + "--merge-sessions", + action="store_true", + help="Calculate with merged sessions (default: separate sessions)", + ) + parser.add_argument( + "--use-get-context", + action="store_true", + help="Calculate for get_context mode (no dialectic events)", + ) + parser.add_argument( + "--no-dream", + action="store_true", + help="Calculate without dream events", + ) + parser.add_argument( + "--question-id", + type=str, + action="append", + dest="question_ids", + help="Only include specific question IDs (can be repeated)", + ) + parser.add_argument( + "--test-count", + type=int, + help="Limit to first N questions", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Show per-question details", + ) + parser.add_argument( + "--json", + action="store_true", + help="Output as JSON instead of formatted text", + ) + + args = parser.parse_args() + + if not args.test_file.exists(): + print(f"Error: Test file not found: {args.test_file}") + return + + summary = calculate_test_file_events( + test_file=args.test_file, + merge_sessions=args.merge_sessions, + use_get_context=args.use_get_context, + dream_enabled=not args.no_dream, + question_ids=args.question_ids, + test_count=args.test_count, + ) + + if args.json: + print(output_json(summary)) + else: + print_summary(summary, verbose=args.verbose) + + +if __name__ == "__main__": + main() diff --git a/tests/bench/locomo.py b/tests/bench/locomo.py index cfa2c969..f9400088 100644 --- a/tests/bench/locomo.py +++ b/tests/bench/locomo.py @@ -58,8 +58,6 @@ Optional arguments: """ import argparse -import asyncio -import time from datetime import datetime from pathlib import Path from typing import Any, cast @@ -81,7 +79,6 @@ from .locomo_common import ( calculate_tokens, extract_sessions, filter_questions, - format_duration, generate_json_summary, get_evidence_context, judge_response, @@ -90,12 +87,12 @@ from .locomo_common import ( print_summary, ) from .runner_common import ( - ReasoningLevel, - RunnerMixin, + BaseRunner, + ItemContext, + RunnerConfig, add_common_arguments, create_anthropic_client, create_openai_client, - export_metrics, validate_common_arguments, ) @@ -169,47 +166,38 @@ def determine_question_target(question: str, speaker_a: str, speaker_b: str) -> return speaker_a -class LoCoMoRunner(RunnerMixin): +class LoCoMoRunner(BaseRunner[ConversationResult]): """ Executes LoCoMo benchmark tests against a Honcho instance. """ def __init__( self, - base_api_port: int = 8000, - pool_size: int = 1, + config: RunnerConfig, + data_file: Path, anthropic_api_key: str | None = None, - timeout_seconds: int | None = None, - cleanup_workspace: bool = False, - use_get_context: bool = False, - redis_url: str = "redis://localhost:6379/0", - reasoning_level: ReasoningLevel | None = None, + sample_id: str | None = None, + test_count: int | None = None, + question_count: int | None = None, ): """ Initialize the LoCoMo test runner. Args: - base_api_port: Base port for Honcho API instances (default: 8000) - pool_size: Number of Honcho instances in the pool (default: 1) + config: Common runner configuration + data_file: Path to the LoCoMo JSON file anthropic_api_key: Anthropic API key for judging responses - timeout_seconds: Timeout for deriver queue in seconds - cleanup_workspace: If True, delete workspace after executing conversation - use_get_context: If True, use get_context + judge LLM instead of dialectic .chat endpoint - redis_url: Redis URL for flush mode signaling (default: redis://localhost:6379/0) - reasoning_level: Reasoning level for dialectic chat (default: None) + sample_id: Optional sample_id to run only that conversation + test_count: Optional number of conversations to run + question_count: Optional limit on questions per conversation """ - self.base_api_port: int = base_api_port - self.pool_size: int = pool_size - self.timeout_seconds: int = ( - timeout_seconds if timeout_seconds is not None else 600 - ) - self.cleanup_workspace: bool = cleanup_workspace - self.use_get_context: bool = use_get_context - self.redis_url: str = redis_url - self.reasoning_level: ReasoningLevel | None = reasoning_level + self.data_file: Path = data_file + self.sample_id_filter: str | None = sample_id + self.test_count: int | None = test_count + self.question_count: int | None = question_count - # Initialize common components (metrics, logging) - self._init_common("locomo") + # Initialize base class + super().__init__(config) # Initialize LLM clients self.anthropic_client: AsyncAnthropic = create_anthropic_client( @@ -217,398 +205,309 @@ class LoCoMoRunner(RunnerMixin): ) self.openai_client: AsyncOpenAI = create_openai_client() - async def execute_conversation( - self, - conversation_data: dict[str, Any], - honcho_url: str, - question_count: int | None = None, - ) -> ConversationResult: - """ - Execute LoCoMo benchmark for a single conversation. + def get_metrics_prefix(self) -> str: + return "locomo" - Args: - conversation_data: Dictionary containing conversation and QA data - honcho_url: URL of the Honcho instance to use - question_count: Optional limit on number of questions to run + def load_items(self) -> list[Any]: + """Load conversations from the data file.""" + conversations = load_locomo_data(self.data_file) - Returns: - Conversation execution results - """ - start_time = time.time() + # Filter by sample_id if specified + if self.sample_id_filter is not None: + conversations = [ + c for c in conversations if c.get("sample_id") == self.sample_id_filter + ] + if not conversations: + print( + f"Error: No conversation found with sample_id '{self.sample_id_filter}'" + ) + return [] + print(f"Filtering to sample_id '{self.sample_id_filter}'") - sample_id = conversation_data.get("sample_id", "unknown") - conversation = conversation_data.get("conversation", {}) - qa_list = conversation_data.get("qa", []) + # Limit by test_count + if self.test_count is not None and self.test_count > 0: + conversations = conversations[: self.test_count] + print(f"Limiting to {len(conversations)} conversations") + return conversations + + def get_workspace_id(self, item: Any) -> str: + """Return workspace ID for a conversation.""" + sample_id = item.get("sample_id", "unknown") + return f"locomo_{sample_id}" + + def get_session_id(self, item: Any, workspace_id: str) -> str: + """Return session ID for a conversation.""" + return f"{workspace_id}_session" + + async def setup_peers(self, ctx: ItemContext, item: Any) -> None: + """Create peers using speaker names as IDs.""" + conversation = item.get("conversation", {}) speaker_a = conversation.get("speaker_a", "User") speaker_b = conversation.get("speaker_b", "Assistant") - print(f"\n{'=' * 80}") - print(f"Executing LoCoMo conversation {sample_id}") - print(f"Speakers: {speaker_a} and {speaker_b}") - print(f"{'=' * 80}") + ctx.peers["speaker_a"] = await ctx.honcho_client.aio.peer(id=speaker_a) + ctx.peers["speaker_b"] = await ctx.honcho_client.aio.peer(id=speaker_b) + ctx.peers["_speaker_a_name"] = speaker_a + ctx.peers["_speaker_b_name"] = speaker_b - # Create workspace for this conversation - workspace_id = f"locomo_{sample_id}" - honcho_client = self.create_honcho_client(workspace_id, honcho_url) + async def setup_session(self, ctx: ItemContext, item: Any) -> None: + """Create and configure session - observe BOTH peers.""" + peer_a = ctx.peers["speaker_a"] + peer_b = ctx.peers["speaker_b"] + + ctx.session = await ctx.honcho_client.aio.session( + id=ctx.session_id, configuration=self._get_session_configuration() + ) + + # Observe both peers since questions ask about both speakers + await ctx.session.aio.add_peers( + [ + (peer_a, SessionPeerConfig(observe_me=True, observe_others=False)), + (peer_b, SessionPeerConfig(observe_me=True, observe_others=False)), + ] + ) + + async def ingest_messages(self, ctx: ItemContext, item: Any) -> int: + """Ingest conversation messages into the session.""" + conversation = item.get("conversation", {}) + speaker_a = ctx.peers["_speaker_a_name"] + speaker_b = ctx.peers["_speaker_b_name"] + peer_a = ctx.peers["speaker_a"] + peer_b = ctx.peers["speaker_b"] + + # Extract and ingest all sessions + sessions = extract_sessions(conversation) + + messages: list[MessageCreateParams] = [] + total_tokens = 0 + + for date_str, session_messages in sessions: + session_date = parse_locomo_date(date_str) if date_str else None + + for msg in session_messages: + speaker = msg.get("speaker", "") + content, metadata = format_message_with_image(msg) + total_tokens += calculate_tokens(content) + + # Map speaker to peer by name + if speaker == speaker_a: + messages.append( + peer_a.message( + content, metadata=metadata, created_at=session_date + ) + ) + elif speaker == speaker_b: + messages.append( + peer_b.message( + content, metadata=metadata, created_at=session_date + ) + ) + + # Store token count for results + ctx.peers["_total_tokens"] = total_tokens + ctx.peers["_total_sessions"] = len(sessions) + + # Add messages in batches of 100 + for i in range(0, len(messages), 100): + batch = messages[i : i + 100] + await ctx.session.aio.add_messages(batch) + + return len(messages) + + def get_dream_observers(self, item: Any) -> list[str]: + """Return both speaker names - LoCoMo triggers dreams for both.""" + conversation = item.get("conversation", {}) + speaker_a = conversation.get("speaker_a", "User") + speaker_b = conversation.get("speaker_b", "Assistant") + return [speaker_a, speaker_b] + + async def execute_questions( + self, ctx: ItemContext, item: Any + ) -> ConversationResult: + """Execute all questions for the conversation.""" + sample_id = item.get("sample_id", "unknown") + conversation = item.get("conversation", {}) + qa_list = item.get("qa", []) + workspace_id = ctx.workspace_id + + speaker_a = ctx.peers["_speaker_a_name"] + speaker_b = ctx.peers["_speaker_b_name"] + peer_a = ctx.peers["speaker_a"] + peer_b = ctx.peers["speaker_b"] result: ConversationResult = { "sample_id": sample_id, "speaker_a": speaker_a, "speaker_b": speaker_b, - "total_sessions": 0, + "total_sessions": ctx.peers.get("_total_sessions", 0), "total_turns": 0, - "total_tokens": 0, + "total_tokens": ctx.peers.get("_total_tokens", 0), "question_results": [], "category_scores": {}, "overall_score": 0.0, "error": None, - "start_time": start_time, + "start_time": 0.0, "end_time": 0.0, "duration_seconds": 0.0, } - try: - # Create peers using their actual names as IDs - peer_a = await honcho_client.aio.peer(id=speaker_a) - peer_b = await honcho_client.aio.peer(id=speaker_b) + # Filter questions + filtered_qa = filter_questions( + qa_list, + exclude_adversarial=True, + test_count=self.question_count, + ) - # Create session for this conversation - session_id = f"{workspace_id}_session" - session = await honcho_client.aio.session(id=session_id) + print(f"[{workspace_id}] Executing {len(filtered_qa)} questions...") - # Configure peer observation - observe BOTH peers since questions ask about both speakers - await session.aio.add_peers( - [ - ( - peer_a, - SessionPeerConfig(observe_me=True, observe_others=False), - ), - ( - peer_b, - SessionPeerConfig(observe_me=True, observe_others=False), - ), - ] - ) + # Execute questions + for q_idx, qa in enumerate(filtered_qa): + question = qa.get("question", "") + expected_answer = qa.get("answer", "") + category = qa.get("category", 0) + evidence = qa.get("evidence", []) + category_name = CATEGORY_NAMES.get(category, f"category_{category}") - # Extract and ingest all sessions - sessions = extract_sessions(conversation) - result["total_sessions"] = len(sessions) - - print(f"[{workspace_id}] Ingesting {len(sessions)} sessions...") - - messages: list[MessageCreateParams] = [] - total_tokens = 0 - - for date_str, session_messages in sessions: - session_date = parse_locomo_date(date_str) if date_str else None - - for msg in session_messages: - speaker = msg.get("speaker", "") - content, metadata = format_message_with_image(msg) - result["total_turns"] += 1 - total_tokens += calculate_tokens(content) - - # Map speaker to peer by name - if speaker == speaker_a: - messages.append( - peer_a.message( - content, metadata=metadata, created_at=session_date - ) - ) - elif speaker == speaker_b: - messages.append( - peer_b.message( - content, metadata=metadata, created_at=session_date - ) - ) - - result["total_tokens"] = total_tokens - - # Add messages in batches of 100 - for i in range(0, len(messages), 100): - batch = messages[i : i + 100] - await session.aio.add_messages(batch) + # Determine which peer the question is about + target_speaker = determine_question_target(question, speaker_a, speaker_b) + target_peer = peer_a if target_speaker == speaker_a else peer_b print( - f"[{workspace_id}] Ingested {len(messages)} messages (~{total_tokens:,} tokens). Waiting for deriver queue..." + f" Q{q_idx + 1} [{category_name}] (asking {target_speaker}): {question}" ) - # Wait for deriver queue to empty - await asyncio.sleep(1) - await self.flush_deriver_queue() - queue_empty = await self.wait_for_deriver_queue_empty(honcho_client) - if not queue_empty: - result["error"] = "Deriver queue timeout" - result["end_time"] = time.time() - result["duration_seconds"] = result["end_time"] - result["start_time"] - print( - f"\n[{workspace_id}] ERROR: Deriver queue timeout after {self.timeout_seconds}s" - ) - return result + try: + if self.config.use_get_context: + # Use get_context + LLM + context = await ctx.session.aio.context( + summary=True, + peer_target=target_speaker, + search_query=question, + ) + context_messages = context.to_anthropic(assistant="assistant") + context_messages.append({"role": "user", "content": question}) - print( - f"[{workspace_id}] Deriver queue empty. Triggering dream consolidation for both peers..." - ) + response = await self.anthropic_client.messages.create( + model="claude-sonnet-4-5", + max_tokens=1024, + messages=cast(list[MessageParam], context_messages), + ) - # Trigger dream for memory consolidation for BOTH peers - # Dream for speaker_a - dream_success_a = await self.trigger_dream_and_wait( - honcho_client, - workspace_id, - observer=speaker_a, - session_id=session_id, - ) + if not response.content: + raise ValueError("Anthropic returned empty response") - if not dream_success_a: - print( - f"[{workspace_id}] Warning: Dream for {speaker_a} did not complete, proceeding anyway" - ) - else: - print(f"[{workspace_id}] Dream for {speaker_a} completed.") - - # Dream for speaker_b - dream_success_b = await self.trigger_dream_and_wait( - honcho_client, - workspace_id, - observer=speaker_b, - session_id=session_id, - ) - - if not dream_success_b: - print( - f"[{workspace_id}] Warning: Dream for {speaker_b} did not complete, proceeding anyway" - ) - else: - print(f"[{workspace_id}] Dream for {speaker_b} completed.") - - # Filter questions - filtered_qa = filter_questions( - qa_list, - exclude_adversarial=True, - test_count=question_count, - ) - - print(f"[{workspace_id}] Executing {len(filtered_qa)} questions...") - - # Execute questions - for q_idx, qa in enumerate(filtered_qa): - question = qa.get("question", "") - expected_answer = qa.get("answer", "") - category = qa.get("category", 0) - evidence = qa.get("evidence", []) - category_name = CATEGORY_NAMES.get(category, f"category_{category}") - - # Determine which peer the question is about (returns speaker name) - target_speaker = determine_question_target( - question, speaker_a, speaker_b - ) - target_peer = peer_a if target_speaker == speaker_a else peer_b - - print( - f" Q{q_idx + 1} [{category_name}] (asking {target_speaker}): {question}" - ) - - try: - if self.use_get_context: - # Use get_context + LLM - target the appropriate peer - context = await session.aio.context( - summary=True, - peer_target=target_speaker, - last_user_message=question, - ) - context_messages = context.to_anthropic(assistant="assistant") - context_messages.append({"role": "user", "content": question}) - - response = await self.anthropic_client.messages.create( - model="claude-sonnet-4-5", - max_tokens=1024, - messages=cast(list[MessageParam], context_messages), - ) - - if not response.content: - raise ValueError("Anthropic returned empty response") - - content_block = response.content[0] - actual_response = getattr(content_block, "text", "") - else: - # Use dialectic .chat endpoint on the appropriate peer - actual_response = await target_peer.aio.chat( - question, - session=session_id, - reasoning_level=self.reasoning_level, - ) - actual_response = ( - actual_response if isinstance(actual_response, str) else "" - ) - - # Get evidence context for the judge - evidence_context = get_evidence_context(conversation, evidence) - - # Judge the response - judgment = await judge_response( - self.openai_client, + actual_response = getattr(response.content[0], "text", "") + else: + # Use dialectic .chat endpoint + actual_response = await target_peer.aio.chat( question, - str(expected_answer), - actual_response, - evidence_context=evidence_context, + session=ctx.session_id, + reasoning_level=self.config.reasoning_level, + ) + actual_response = ( + actual_response if isinstance(actual_response, str) else "" ) - passed = judgment.get("passed", False) + # Get evidence context for the judge + evidence_context = get_evidence_context(conversation, evidence) - question_result: QuestionResult = { - "question_id": q_idx, - "question": question, - "expected_answer": str(expected_answer), - "actual_response": actual_response, - "category": category, - "category_name": category_name, - "evidence": evidence, - "judgment": judgment, - "passed": passed, - } - - result["question_results"].append(question_result) - - status = "PASS" if passed else "FAIL" - print(f" [{status}]") - if not passed: - print(f" Expected: {expected_answer}") - print(f" Got: {actual_response[:200]}...") - - except Exception as e: - self.logger.error(f"Error executing question {q_idx}: {e}") - question_result = QuestionResult( - question_id=q_idx, - question=question, - expected_answer=str(expected_answer), - actual_response=f"ERROR: {e}", - category=category, - category_name=category_name, - evidence=evidence, - judgment={"passed": False, "reasoning": str(e)}, - passed=False, - ) - result["question_results"].append(question_result) - - # Calculate category scores - result["category_scores"] = calculate_category_scores( - result["question_results"] - ) - - # Calculate overall score (pass rate) - if result["question_results"]: - passed_count = sum( - 1 for qr in result["question_results"] if qr["passed"] + # Judge the response + judgment = await judge_response( + self.openai_client, + question, + str(expected_answer), + actual_response, + evidence_context=evidence_context, ) - result["overall_score"] = passed_count / len(result["question_results"]) - # Cleanup workspace if requested - if self.cleanup_workspace: - try: - await honcho_client.aio.delete_workspace(workspace_id) - print(f"[{workspace_id}] Cleaned up workspace") - except Exception as e: - print(f"Failed to delete workspace: {e}") + passed = judgment.get("passed", False) - result["end_time"] = time.time() - result["duration_seconds"] = result["end_time"] - result["start_time"] + question_result: QuestionResult = { + "question_id": q_idx, + "question": question, + "expected_answer": str(expected_answer), + "actual_response": actual_response, + "category": category, + "category_name": category_name, + "evidence": evidence, + "judgment": judgment, + "passed": passed, + } - print( - f"\n[{workspace_id}] Completed in {format_duration(result['duration_seconds'])}" - ) - print(f"Overall Score: {result['overall_score']:.3f}") + result["question_results"].append(question_result) - except Exception as e: - self.logger.error(f"Error executing conversation {sample_id}: {e}") - result["error"] = str(e) - result["end_time"] = time.time() - result["duration_seconds"] = result["end_time"] - result["start_time"] + status = "PASS" if passed else "FAIL" + print(f" [{status}]") + if not passed: + print(f" Expected: {expected_answer}") + print(f" Got: {actual_response[:200]}...") + + except Exception as e: + self.logger.error(f"Error executing question {q_idx}: {e}") + question_result = QuestionResult( + question_id=q_idx, + question=question, + expected_answer=str(expected_answer), + actual_response=f"ERROR: {e}", + category=category, + category_name=category_name, + evidence=evidence, + judgment={"passed": False, "reasoning": str(e)}, + passed=False, + ) + result["question_results"].append(question_result) + + # Calculate category scores + result["category_scores"] = calculate_category_scores( + result["question_results"] + ) + + # Calculate overall score (pass rate) + if result["question_results"]: + passed_count = sum(1 for qr in result["question_results"] if qr["passed"]) + result["overall_score"] = passed_count / len(result["question_results"]) + + print(f"\nOverall Score: {result['overall_score']:.3f}") return result - async def run_conversations( - self, - data_file: Path, - batch_size: int = 1, - test_count: int | None = None, - sample_id: str | None = None, - question_count: int | None = None, - ) -> tuple[list[ConversationResult], float]: - """ - Run multiple conversations from the LoCoMo benchmark. + def print_summary( + self, results: list[ConversationResult], total_duration: float + ) -> None: + """Print summary using the common function.""" + print_summary(results, total_duration) - Args: - data_file: Path to the LoCoMo JSON file - batch_size: Number of conversations to run concurrently in each batch - test_count: Optional number of conversations to run - sample_id: Optional sample_id to run only that conversation - question_count: Optional limit on questions per conversation - - Returns: - Tuple of (list of conversation results, total duration) - """ - conversations = load_locomo_data(data_file) - - # Filter by sample_id if specified - if sample_id is not None: - conversations = [ - c for c in conversations if c.get("sample_id") == sample_id - ] - if not conversations: - print(f"Error: No conversation found with sample_id '{sample_id}'") - return [], 0.0 - print(f"Filtering to sample_id '{sample_id}'") - - # Limit by test_count - if test_count is not None and test_count > 0: - conversations = conversations[:test_count] - print(f"Limiting to {len(conversations)} conversations") - - print(f"Running {len(conversations)} conversations from {data_file}") - if self.pool_size > 1: - print( - f"Distributing conversations across {self.pool_size} Honcho instances" + def generate_output( + self, results: list[ConversationResult], total_duration: float + ) -> None: + """Generate JSON output file.""" + if self.config.json_output: + output_file = self.config.json_output + else: + output_file = Path( + f"tests/bench/eval_results/locomo_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" ) - overall_start = time.time() - all_results: list[ConversationResult] = [] - - for i in range(0, len(conversations), batch_size): - batch = conversations[i : i + batch_size] - batch_num = (i // batch_size) + 1 - total_batches = (len(conversations) + batch_size - 1) // batch_size - - print(f"\n{'=' * 80}") - print( - f"Processing batch {batch_num}/{total_batches} ({len(batch)} conversations)" - ) - print(f"{'=' * 80}") - - # Run conversations in current batch concurrently - batch_results: list[ConversationResult] = await asyncio.gather( - *[ - self.execute_conversation( - conv, - self.get_honcho_url_for_index(i + idx), - question_count=question_count, - ) - for idx, conv in enumerate(batch) - ] - ) - - all_results.extend(batch_results) - - overall_end = time.time() - overall_duration = overall_end - overall_start - - # Finalize metrics collection - self.metrics_collector.finalize_collection() - - return all_results, overall_duration + generate_json_summary( + results, + total_duration, + output_file, + metadata_extra={ + "data_file": str(self.data_file), + "base_api_port": self.config.base_api_port, + "pool_size": self.config.pool_size, + "timeout_seconds": self.config.timeout_seconds, + "reasoning_level": self.config.reasoning_level, + "deriver_settings": settings.DERIVER.model_dump(), + "dialectic_settings": settings.DIALECTIC.model_dump(), + "dream_settings": settings.DREAM.model_dump(), + "summary_settings": settings.SUMMARY.model_dump(), + }, + ) -async def main() -> int: +def main() -> int: """Main entry point for the LoCoMo test runner.""" parser = argparse.ArgumentParser( description="Run LoCoMo benchmark tests against a Honcho instance", @@ -671,78 +570,20 @@ Examples: print(f"Error: Data file {args.data_file} does not exist") return 1 - # Create test runner + # Create config and runner + config = RunnerConfig.from_args(args, default_timeout=600) + runner = LoCoMoRunner( - base_api_port=args.base_api_port, - pool_size=args.pool_size, + config=config, + data_file=args.data_file, anthropic_api_key=args.anthropic_api_key, - timeout_seconds=args.timeout, - cleanup_workspace=args.cleanup_workspace, - use_get_context=args.use_get_context, - redis_url=args.redis_url, - reasoning_level=args.reasoning_level, + sample_id=args.sample_id, + test_count=args.test_count, + question_count=args.question_count, ) - try: - # Run conversations - results, total_elapsed = await runner.run_conversations( - args.data_file, - args.batch_size, - args.test_count, - args.sample_id, - args.question_count, - ) - - print_summary(results, total_elapsed) - - # Print metrics summary - runner.metrics_collector.print_summary() - - # Generate JSON output - if args.json_output: - output_file = args.json_output - else: - output_file = Path( - f"tests/bench/eval_results/locomo_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - ) - - generate_json_summary( - results, - total_elapsed, - output_file, - metadata_extra={ - "data_file": str(args.data_file), - "base_api_port": runner.base_api_port, - "pool_size": runner.pool_size, - "timeout_seconds": runner.timeout_seconds, - "reasoning_level": runner.reasoning_level, - "deriver_settings": settings.DERIVER.model_dump(), - "dialectic_settings": settings.DIALECTIC.model_dump(), - "dream_settings": settings.DREAM.model_dump(), - "summary_settings": settings.SUMMARY.model_dump(), - }, - ) - - # Export metrics to JSON file - export_metrics(runner.metrics_collector, "locomo") - - # Return exit code based on results - avg_score = ( - sum(r["overall_score"] for r in results) / len(results) if results else 0 - ) - return 0 if avg_score >= 0.5 else 1 - - except KeyboardInterrupt: - print("\nTest execution interrupted by user") - return 1 - except Exception as e: - print(f"Error running tests: {e}") - import traceback - - traceback.print_exc() - return 1 + return runner.run_and_summarize() if __name__ == "__main__": - exit_code = asyncio.run(main()) - exit(exit_code) + exit(main()) diff --git a/tests/bench/longmem.py b/tests/bench/longmem.py index 8072d90f..09a417c8 100644 --- a/tests/bench/longmem.py +++ b/tests/bench/longmem.py @@ -55,7 +55,6 @@ Optional arguments: """ import argparse -import asyncio import json import time from datetime import datetime @@ -77,19 +76,19 @@ from .longmem_common import ( calculate_total_tokens, calculate_type_statistics, filter_questions, - format_duration, judge_response, load_test_file, parse_longmemeval_date, write_json_summary, ) from .runner_common import ( - ReasoningLevel, - RunnerMixin, + BaseRunner, + ItemContext, + RunnerConfig, add_common_arguments, create_anthropic_client, create_openai_client, - export_metrics, + format_duration, validate_common_arguments, ) @@ -126,53 +125,40 @@ class TestResult(TypedDict): start_time: float end_time: float duration_seconds: float - output_lines: list[str] -class LongMemEvalRunner(RunnerMixin): +class LongMemEvalRunner(BaseRunner[TestResult]): """ Executes longmemeval JSON tests against a Honcho instance. """ def __init__( self, - base_api_port: int = 8000, - pool_size: int = 1, + config: RunnerConfig, + test_file: Path, anthropic_api_key: str | None = None, - timeout_seconds: int | None = None, merge_sessions: bool = False, - cleanup_workspace: bool = False, - use_get_context: bool = False, - redis_url: str = "redis://localhost:6379/0", - reasoning_level: ReasoningLevel | None = None, + test_count: int | None = None, + question_id: str | None = None, ): """ Initialize the test runner. Args: - base_api_port: Base port for Honcho API instances (default: 8000) - pool_size: Number of Honcho instances in the pool (default: 1) + config: Common runner configuration + test_file: Path to the longmemeval JSON file anthropic_api_key: Anthropic API key for judging responses - timeout_seconds: Timeout for deriver queue in seconds merge_sessions: If True, merge all sessions within a question into one session - cleanup_workspace: If True, delete workspace after executing question (default: False) - use_get_context: If True, use get_context + judge LLM instead of dialectic .chat endpoint - redis_url: Redis URL for flush mode signaling (default: redis://localhost:6379/0) - reasoning_level: Reasoning level for dialectic chat (default: None) + test_count: Optional number of tests to run (runs first N tests) + question_id: Optional question_id to run (skips all others) """ - self.base_api_port: int = base_api_port - self.pool_size: int = pool_size - self.timeout_seconds: int = ( - timeout_seconds if timeout_seconds is not None else 10000 - ) + self.test_file: Path = test_file self.merge_sessions: bool = merge_sessions - self.cleanup_workspace: bool = cleanup_workspace - self.use_get_context: bool = use_get_context - self.redis_url: str = redis_url - self.reasoning_level: ReasoningLevel | None = reasoning_level + self.test_count: int | None = test_count + self.question_id_filter: str | None = question_id - # Initialize common components (metrics, logging) - self._init_common("longmem") + # Initialize base class (sets up metrics collector and logger) + super().__init__(config) # Initialize LLM clients self.anthropic_client: AsyncAnthropic = create_anthropic_client( @@ -180,138 +166,147 @@ class LongMemEvalRunner(RunnerMixin): ) self.openai_client: AsyncOpenAI = create_openai_client() - def _get_latest_input_tokens_used(self) -> int | None: - """Get the uncached input tokens from the most recent dialectic_chat metric. + def get_metrics_prefix(self) -> str: + return "longmem" - Returns: - Number of tokens used, or None if not found - """ - metrics_file = Path(settings.LOCAL_METRICS_FILE) - if not metrics_file.exists(): - return None + def load_items(self) -> list[Any]: + """Load questions from the test file.""" + questions = load_test_file(self.test_file) + questions = filter_questions( + questions, self.test_file, self.question_id_filter, self.test_count + ) + return questions - # Read the file and find the most recent dialectic_chat metric - try: - with open(metrics_file) as f: - lines = f.readlines() + def get_workspace_id(self, item: Any) -> str: + """Return workspace ID for a question.""" + return f"{item['question_id']}_{item['question_type']}" - # Search backwards through the file for the most recent dialectic_chat - for line in reversed(lines): - if not line.strip(): - continue - try: - data = json.loads(line) - task_name = data.get("task_name", "") - if task_name.startswith("dialectic_chat_"): - for metric in data.get("metrics", []): - metric_name = metric.get("name", "") - if metric_name.endswith("uncached_input_tokens"): - return int(metric.get("value", 0)) - except (json.JSONDecodeError, KeyError, ValueError): - continue - - except Exception as e: - self.logger.warning(f"Error reading metrics file: {e}") - - return None - - async def execute_question( - self, question_data: dict[str, Any], honcho_url: str - ) -> TestResult: - """ - Execute a single longmemeval question. - - Args: - question_data: Dictionary containing question data - honcho_url: URL of the Honcho instance to use - - Returns: - Test execution results - """ - question_id = question_data["question_id"] - question_type = question_data["question_type"] - question = question_data["question"] - expected_answer = question_data["answer"] - question_date = question_data.get("question_date", "") - - question_with_date = ( - f"[{question_date}] {question}" if question_date else question + def get_session_id(self, item: Any, workspace_id: str) -> str: + """Return session ID for a question.""" + if self.merge_sessions: + return f"{workspace_id}_merged" + # For non-merged, we use the first haystack session ID + haystack_session_ids = item.get("haystack_session_ids", []) + return ( + haystack_session_ids[0] + if haystack_session_ids + else f"{workspace_id}_session" ) - output_lines: list[str] = [] - output_lines.append( - f"\033[1mExecuting question {question_id} ({question_type})\033[0m" - ) - output_lines.append(f"Question: {question_with_date}") - output_lines.append(f"Expected: {expected_answer}") - output_lines.append(f"Using Honcho instance: {honcho_url}") + async def setup_peers(self, ctx: ItemContext, item: Any) -> None: + """Create user and assistant peers.""" + ctx.peers["user"] = await ctx.honcho_client.aio.peer(id="user") + ctx.peers["assistant"] = await ctx.honcho_client.aio.peer(id="assistant") - # Create workspace for this question - workspace_id = f"{question_id}_{question_type}" - honcho_client = self.create_honcho_client(workspace_id, honcho_url) + async def setup_session(self, ctx: ItemContext, item: Any) -> None: + """Create and configure session with appropriate observation settings.""" + is_assistant_type = item["question_type"] == "single-session-assistant" + user_peer = ctx.peers["user"] + assistant_peer = ctx.peers["assistant"] - results: TestResult = { - "question_id": question_id, - "question_type": question_type, - "workspace_id": workspace_id, - "sessions_created": [], - "query_executed": None, - "passed": False, - "error": None, - "start_time": time.time(), - "end_time": 0.0, - "duration_seconds": 0.0, - "output_lines": output_lines, - } - - try: - user_peer = await honcho_client.aio.peer(id="user") - assistant_peer = await honcho_client.aio.peer(id="assistant") - - # Process haystack sessions - haystack_dates = question_data.get("haystack_dates", []) - haystack_sessions = question_data.get("haystack_sessions", []) - haystack_session_ids = question_data.get("haystack_session_ids", []) - - # Validate alignment of dates, session IDs, and sessions - if len(haystack_dates) != len(haystack_sessions): - raise ValueError( - f"Misaligned data: {len(haystack_dates)} dates but {len(haystack_sessions)} sessions" - ) - if len(haystack_session_ids) != len(haystack_sessions): - raise ValueError( - f"Misaligned data: {len(haystack_session_ids)} session IDs but {len(haystack_sessions)} sessions" - ) - - # Parse all dates upfront to catch parsing errors early - parsed_dates: list[datetime] = [] - for date_str in haystack_dates: - try: - parsed_dates.append(parse_longmemeval_date(date_str)) - except ValueError as e: - raise ValueError(f"Error parsing date '{date_str}': {e}") from e - - haystack_total_messages = sum(len(session) for session in haystack_sessions) - - # Calculate total tokens available in the sessions for this question - total_available_tokens = calculate_total_tokens(haystack_sessions) - - print( - f"[{workspace_id}] processing {len(haystack_sessions)} sessions with {haystack_total_messages} total messages ({total_available_tokens} total tokens)" + if self.merge_sessions: + # Create a single merged session + ctx.session = await ctx.honcho_client.aio.session( + id=ctx.session_id, configuration=self._get_session_configuration() ) - # Determine which peer should be observed based on question type - is_assistant_type = question_type == "single-session-assistant" + if is_assistant_type: + await ctx.session.aio.add_peers( + [ + ( + user_peer, + SessionPeerConfig(observe_me=False, observe_others=False), + ), + ( + assistant_peer, + SessionPeerConfig(observe_me=True, observe_others=False), + ), + ] + ) + else: + await ctx.session.aio.add_peers( + [ + ( + user_peer, + SessionPeerConfig(observe_me=True, observe_others=False), + ), + ( + assistant_peer, + SessionPeerConfig(observe_me=False, observe_others=False), + ), + ] + ) + else: + # Sessions are created during ingestion for non-merged mode + ctx.session = None - # Initialize merged_session_id for potential use in dream trigger - merged_session_id: str | None = None + async def ingest_messages(self, ctx: ItemContext, item: Any) -> int: + """Ingest haystack messages into session(s).""" + is_assistant_type = item["question_type"] == "single-session-assistant" + user_peer = ctx.peers["user"] + assistant_peer = ctx.peers["assistant"] - if self.merge_sessions: - # Create a single merged session for all messages - merged_session_id = f"{workspace_id}_merged" - session = await honcho_client.aio.session(id=merged_session_id) + haystack_dates = item.get("haystack_dates", []) + haystack_sessions = item.get("haystack_sessions", []) + haystack_session_ids = item.get("haystack_session_ids", []) + + # Parse dates + parsed_dates = [parse_longmemeval_date(d) for d in haystack_dates] + + total_messages = 0 + + if self.merge_sessions: + # Collect all messages from all sessions + all_messages: list[MessageCreateParams] = [] + for session_date, session_messages in zip( + parsed_dates, haystack_sessions, strict=True + ): + for msg in session_messages: + role = msg["role"] + content = msg["content"] + + # Split long messages + if len(content) > 25000: + chunks = [ + content[i : i + 25000] + for i in range(0, len(content), 25000) + ] + for chunk in chunks: + if role == "user": + all_messages.append( + user_peer.message(chunk, created_at=session_date) + ) + elif role == "assistant": + all_messages.append( + assistant_peer.message( + chunk, created_at=session_date + ) + ) + else: + if role == "user": + all_messages.append( + user_peer.message(content, created_at=session_date) + ) + elif role == "assistant": + all_messages.append( + assistant_peer.message(content, created_at=session_date) + ) + + # Add messages in batches + for i in range(0, len(all_messages), 100): + batch = all_messages[i : i + 100] + await ctx.session.aio.add_messages(batch) + + total_messages = len(all_messages) + else: + # Create separate sessions for each haystack session + for session_date, session_id, session_messages in zip( + parsed_dates, haystack_session_ids, haystack_sessions, strict=True + ): + session = await ctx.honcho_client.aio.session( + id=session_id, configuration=self._get_session_configuration() + ) - # Configure peer observation based on question type if is_assistant_type: await session.aio.add_peers( [ @@ -347,426 +342,218 @@ class LongMemEvalRunner(RunnerMixin): ] ) - # Collect all messages from all sessions in chronological order - all_messages: list[MessageCreateParams] = [] - for session_date, session_messages in zip( - parsed_dates, haystack_sessions, strict=True - ): - for msg in session_messages: - role = msg["role"] - content = msg["content"] + honcho_messages: list[MessageCreateParams] = [] + for msg in session_messages: + role = msg["role"] + content = msg["content"] - # Split message if it exceeds 25000 characters - if len(content) > 25000: - chunks = [ - content[i : i + 25000] - for i in range(0, len(content), 25000) - ] - for chunk in chunks: - if role == "user": - all_messages.append( - user_peer.message( - chunk, created_at=session_date - ) - ) - elif role == "assistant": - all_messages.append( - assistant_peer.message( - chunk, created_at=session_date - ) - ) - else: + if len(content) > 25000: + chunks = [ + content[i : i + 25000] + for i in range(0, len(content), 25000) + ] + for chunk in chunks: if role == "user": - all_messages.append( - user_peer.message(content, created_at=session_date) + honcho_messages.append( + user_peer.message(chunk, created_at=session_date) ) elif role == "assistant": - all_messages.append( + honcho_messages.append( assistant_peer.message( - content, created_at=session_date + chunk, created_at=session_date ) ) + else: + if role == "user": + honcho_messages.append( + user_peer.message(content, created_at=session_date) + ) + elif role == "assistant": + honcho_messages.append( + assistant_peer.message(content, created_at=session_date) + ) - # Add messages in batches of 100 (max supported by add_messages) - if all_messages: - for i in range(0, len(all_messages), 100): - batch = all_messages[i : i + 100] - await session.aio.add_messages(batch) + for i in range(0, len(honcho_messages), 100): + batch = honcho_messages[i : i + 100] + await session.aio.add_messages(batch) - results["sessions_created"].append( - SessionResult( - name=merged_session_id, message_count=len(all_messages) - ) + total_messages += len(honcho_messages) + + return total_messages + + def get_dream_observers(self, item: Any) -> list[str]: + """Return the observer based on question type.""" + is_assistant_type = item["question_type"] == "single-session-assistant" + return ["assistant"] if is_assistant_type else ["user"] + + def _get_latest_input_tokens_used(self) -> int | None: + """Get the uncached input tokens from the most recent dialectic_chat metric.""" + metrics_file = Path(settings.LOCAL_METRICS_FILE) + if not metrics_file.exists(): + return None + + try: + with open(metrics_file) as f: + lines = f.readlines() + + for line in reversed(lines): + if not line.strip(): + continue + try: + data = json.loads(line) + task_name = data.get("task_name", "") + if task_name.startswith("dialectic_chat_"): + for metric in data.get("metrics", []): + metric_name = metric.get("name", "") + if metric_name.endswith("uncached_input_tokens"): + return int(metric.get("value", 0)) + except (json.JSONDecodeError, KeyError, ValueError): + continue + except Exception as e: + self.logger.warning(f"Error reading metrics file: {e}") + + return None + + async def execute_questions(self, ctx: ItemContext, item: Any) -> TestResult: + """Execute the question and judge the response.""" + start_time = time.time() + workspace_id = ctx.workspace_id + + question_id = item["question_id"] + question_type = item["question_type"] + question = item["question"] + expected_answer = item["answer"] + question_date = item.get("question_date", "") + + question_with_date = ( + f"[{question_date}] {question}" if question_date else question + ) + is_assistant_type = question_type == "single-session-assistant" + + # Calculate total tokens for efficiency metrics + haystack_sessions = item.get("haystack_sessions", []) + total_available_tokens = calculate_total_tokens(haystack_sessions) + + result: TestResult = { + "question_id": question_id, + "question_type": question_type, + "workspace_id": workspace_id, + "sessions_created": [], # Populated during ingestion tracking + "query_executed": None, + "passed": False, + "error": None, + "start_time": start_time, + "end_time": 0.0, + "duration_seconds": 0.0, + } + + try: + print(f" Asking: {question_with_date}") + + if self.config.use_get_context: + # Use get_context instead of dialectic .chat endpoint + if not self.merge_sessions or ctx.session is None: + raise ValueError("Merged session required for get_context mode") + + peer_id = "assistant" if is_assistant_type else "user" + context = await ctx.session.aio.context( + summary=True, + peer_target=peer_id, + search_query=question, ) + + context_messages = context.to_anthropic(assistant="assistant") + context_messages.append({"role": "user", "content": question_with_date}) + + response = await self.anthropic_client.messages.create( + model="claude-sonnet-4-5", + max_tokens=1024, + messages=cast(list[MessageParam], context_messages), + ) + + if not response.content: + raise ValueError("Anthropic returned empty response") + + actual_response = getattr(response.content[0], "text", "") else: - # create separate sessions - # Zip together dates, session IDs, and session content - for session_date, session_id, session_messages in zip( - parsed_dates, haystack_session_ids, haystack_sessions, strict=True - ): - session = await honcho_client.aio.session(id=session_id) - - # Configure peer observation based on question type - if is_assistant_type: - # For assistant questions, observe the assistant peer - await session.aio.add_peers( - [ - ( - user_peer, - SessionPeerConfig( - observe_me=False, observe_others=False - ), - ), - ( - assistant_peer, - SessionPeerConfig( - observe_me=True, observe_others=False - ), - ), - ] - ) - else: - # For user questions, observe the user peer (default behavior) - await session.aio.add_peers( - [ - ( - user_peer, - SessionPeerConfig( - observe_me=True, observe_others=False - ), - ), - ( - assistant_peer, - SessionPeerConfig( - observe_me=False, observe_others=False - ), - ), - ] - ) - - honcho_messages: list[MessageCreateParams] = [] - for msg in session_messages: - role = msg["role"] - content = msg["content"] - - # Split message if it exceeds 25000 characters - if len(content) > 25000: - chunks = [ - content[i : i + 25000] - for i in range(0, len(content), 25000) - ] - for chunk in chunks: - # Use the session date as the timestamp for all messages in this session - if role == "user": - honcho_messages.append( - user_peer.message( - chunk, created_at=session_date - ) - ) - elif role == "assistant": - honcho_messages.append( - assistant_peer.message( - chunk, created_at=session_date - ) - ) - else: - # Use the session date as the timestamp for all messages in this session - if role == "user": - honcho_messages.append( - user_peer.message(content, created_at=session_date) - ) - elif role == "assistant": - honcho_messages.append( - assistant_peer.message( - content, created_at=session_date - ) - ) - - if honcho_messages: - for i in range(0, len(honcho_messages), 100): - batch = honcho_messages[i : i + 100] - await session.aio.add_messages(batch) - - results["sessions_created"].append( - SessionResult( - name=session_id, message_count=len(honcho_messages) - ) - ) - - print( - f"[{workspace_id}] fired all messages.\nwaiting for deriver queue to be empty... will time out in {self.timeout_seconds} seconds" - ) - await asyncio.sleep( - 1 - ) # Give time for at least some tasks to be queued, so deriver queue size check doesn't immediately return 0 - - # Enable flush mode to bypass batch token threshold - await self.flush_deriver_queue() - - queue_empty = await self.wait_for_deriver_queue_empty(honcho_client) - if not queue_empty: - output_lines.append("Deriver queue never emptied!!!") - results["error"] = "Deriver queue timeout" - return results - - # Trigger dream for memory consolidation before questions - print( - f"[{workspace_id}] Deriver queue empty. Triggering dream consolidation..." - ) - - # Determine session_id for dream - dream_session_id = ( - merged_session_id - if self.merge_sessions and merged_session_id - else ( - haystack_session_ids[0] - if haystack_session_ids - else f"{workspace_id}_session" + # Use dialectic .chat endpoint + peer = ( + ctx.peers["assistant"] if is_assistant_type else ctx.peers["user"] + ) + actual_response = await peer.aio.chat( + question_with_date, + reasoning_level=self.config.reasoning_level, ) - ) - - # Determine observer based on question type - observer_peer = "assistant" if is_assistant_type else "user" - - # Single orchestrated dream handles all reasoning types - dream_success = await self.trigger_dream_and_wait( - honcho_client, - workspace_id, - observer=observer_peer, - session_id=dream_session_id, - ) - if not dream_success: - print(f"[{workspace_id}] Warning: Dream did not complete") - print(f"[{workspace_id}] Dream completed. Executing question...") - - # Execute the question - output_lines.append(f"\nAsking question: {question_with_date}") - - try: - if self.use_get_context: - # Use get_context instead of dialectic .chat endpoint - # Get the session to retrieve context from - if not self.merge_sessions or merged_session_id is None: - raise ValueError( - "Merged session ID is required when using get_context. Set --merge-sessions to True." - ) - session = await honcho_client.aio.session(id=merged_session_id) - - # Get context for the appropriate peer - peer_id = "assistant" if is_assistant_type else "user" - context = await session.aio.context( - summary=True, - peer_target=peer_id, - last_user_message=question, - ) - - # Format context using to_anthropic method - context_messages = context.to_anthropic(assistant="assistant") - - # Add the question as the final user message - context_messages.append( - {"role": "user", "content": question_with_date} - ) - - # Call Anthropic API to generate response - response = await self.anthropic_client.messages.create( - model="claude-sonnet-4-5", - max_tokens=1024, - messages=cast(list[MessageParam], context_messages), - ) - - if not response.content: - raise ValueError("Anthropic returned empty response") - - content_block = response.content[0] - actual_response = getattr(content_block, "text", "") - else: - # Use the appropriate peer based on question type - if is_assistant_type: - # For assistant questions, use the assistant peer - actual_response = await assistant_peer.aio.chat( - question_with_date, - reasoning_level=self.reasoning_level, - ) - else: - # For user questions, use the user peer (default behavior) - actual_response = await user_peer.aio.chat( - question_with_date, - reasoning_level=self.reasoning_level, - ) - - # Clean up workspace if requested - if self.cleanup_workspace: - try: - await honcho_client.aio.delete_workspace(workspace_id) - print(f"[{workspace_id}] cleaned up workspace") - except Exception as e: - print(f"Failed to delete workspace: {e}") - actual_response = ( actual_response if isinstance(actual_response, str) else "" ) - input_tokens_used = self._get_latest_input_tokens_used() - - token_efficiency = None - if input_tokens_used is not None and total_available_tokens > 0: - efficiency_ratio = input_tokens_used / total_available_tokens - token_efficiency = { - "total_available_tokens": total_available_tokens, - "tokens_used": input_tokens_used, - "efficiency_ratio": efficiency_ratio, - } - output_lines.append( - f" token efficiency: {efficiency_ratio:.4f} ({input_tokens_used}/{total_available_tokens} tokens, {efficiency_ratio * 100:.2f}%)" - ) - - judgment = await judge_response( - self.openai_client, - question_with_date, - expected_answer, - actual_response, - question_type, - question_id, - ) - - query_result: QueryResult = { - "question": question_with_date, - "expected_answer": expected_answer, - "actual_response": actual_response, - "judgment": judgment, - "token_efficiency": token_efficiency, + # Get token efficiency + input_tokens_used = self._get_latest_input_tokens_used() + token_efficiency = None + if input_tokens_used is not None and total_available_tokens > 0: + efficiency_ratio = input_tokens_used / total_available_tokens + token_efficiency = { + "total_available_tokens": total_available_tokens, + "tokens_used": input_tokens_used, + "efficiency_ratio": efficiency_ratio, } - - results["query_executed"] = query_result - results["passed"] = judgment["passed"] - - output_lines.append( - " judgment: \033[1m\033[32mPASS\033[0m" - if judgment["passed"] - else " judgment: \033[1m\033[31mFAIL\033[0m" + print( + f" Token efficiency: {efficiency_ratio:.4f} ({input_tokens_used}/{total_available_tokens})" ) - if not judgment["passed"]: - output_lines.append( - f" got response: \033[3m{actual_response}\033[0m" - ) - output_lines.append(f" expected: {expected_answer}") - output_lines.append(f" reasoning: {judgment['reasoning']}") - except Exception as e: - self.logger.error(f"Error executing question: {e}") - query_result = QueryResult( - question=question_with_date, - expected_answer=expected_answer, - actual_response=f"ERROR: {e}", - judgment={ - "passed": False, - "reasoning": f"Question execution failed: {e}", - }, - token_efficiency=None, - ) - results["query_executed"] = query_result - results["passed"] = False - - results["end_time"] = time.time() - results["duration_seconds"] = results["end_time"] - results["start_time"] - - output_lines.append( - f"\nQuestion {question_id} completed. Status: {'PASS' if results['passed'] else 'FAIL'} (Duration: {format_duration(results['duration_seconds'])})" + # Judge the response + judgment = await judge_response( + self.openai_client, + question_with_date, + expected_answer, + actual_response, + question_type, + question_id, ) + result["query_executed"] = QueryResult( + question=question_with_date, + expected_answer=expected_answer, + actual_response=actual_response, + judgment=judgment, + token_efficiency=token_efficiency, + ) + result["passed"] = judgment["passed"] + + status = ( + "\033[1m\033[32mPASS\033[0m" + if judgment["passed"] + else "\033[1m\033[31mFAIL\033[0m" + ) + print(f" Judgment: {status}") + if not judgment["passed"]: + print(f" Got: \033[3m{actual_response}\033[0m") + print(f" Expected: {expected_answer}") + print(f" Reasoning: {judgment['reasoning']}") + except Exception as e: - self.logger.error(f"Error executing question {question_id}: {e}") - results["error"] = str(e) - results["passed"] = False - results["end_time"] = time.time() - results["duration_seconds"] = results["end_time"] - results["start_time"] - output_lines.append(f"Error executing question {question_id}: {e}") - - return results - - async def run_all_questions( - self, - test_file: Path, - batch_size: int = 10, - test_count: int | None = None, - question_id: str | None = None, - ) -> tuple[list[TestResult], float]: - """ - Run all questions in a longmemeval test file. - - Args: - test_file: Path to the longmemeval JSON file - batch_size: Number of questions to run concurrently in each batch - test_count: Optional number of tests to run (runs first N tests) - question_id: Optional question_id to run (skips all others) - - Returns: - Tuple of (list of test results, total duration) - """ - questions = load_test_file(test_file) - questions = filter_questions(questions, test_file, question_id, test_count) - if not questions: - return [], 0.0 - - print( - f"found {len(questions)} {'question' if len(questions) == 1 else 'questions'} in {test_file}" - ) - if self.pool_size > 1: - print( - f"distributing questions across {self.pool_size} Honcho instances (ports {self.base_api_port}-{self.base_api_port + self.pool_size - 1})" + self.logger.error(f"Error executing question: {e}") + result["query_executed"] = QueryResult( + question=question_with_date, + expected_answer=expected_answer, + actual_response=f"ERROR: {e}", + judgment={ + "passed": False, + "reasoning": f"Question execution failed: {e}", + }, + token_efficiency=None, ) + result["passed"] = False + result["error"] = str(e) - overall_start = time.time() + result["end_time"] = time.time() + result["duration_seconds"] = result["end_time"] - result["start_time"] - # Process questions in batches - all_results: list[TestResult] = [] + return result - for i in range(0, len(questions), batch_size): - batch = questions[i : i + batch_size] - batch_num = (i // batch_size) + 1 - total_batches = (len(questions) + batch_size - 1) // batch_size - - print(f"\n{'=' * 60}") - print( - f"Processing batch {batch_num}/{total_batches} ({len(batch)} questions)" - ) - print(f"{'=' * 60}") - - # Run questions in current batch concurrently, distributing via round-robin - batch_results: list[TestResult] = await asyncio.gather( - *[ - self.execute_question(q, self.get_honcho_url_for_index(i + idx)) - for idx, q in enumerate(batch) - ] - ) - - # Print detailed per-question outputs for this batch - for result in batch_results: - print(f"\n{'=' * 60}") - print("\n".join(result.get("output_lines", []))) - print(f"{'=' * 60}\n") - - all_results.extend(batch_results) - - overall_end = time.time() - overall_duration = overall_end - overall_start - - # Finalize metrics collection - self.metrics_collector.finalize_collection() - - return all_results, overall_duration - - def print_summary( - self, results: list[TestResult], total_elapsed_seconds: float | None = None - ) -> None: - """ - Print a summary of all test results. - - Args: - results: List of test results - total_elapsed_seconds: Total elapsed time - """ + def print_summary(self, results: list[TestResult], total_duration: float) -> None: + """Print a summary of all test results.""" print(f"\n{'=' * 80}") print("LONGMEMEVAL TEST EXECUTION SUMMARY") print(f"{'=' * 80}") @@ -774,19 +561,19 @@ class LongMemEvalRunner(RunnerMixin): total_questions = len(results) passed_questions = sum(1 for r in results if r.get("passed", False)) failed_questions = total_questions - passed_questions - total_test_time = ( - total_elapsed_seconds - if total_elapsed_seconds is not None - else sum(r["duration_seconds"] for r in results) - ) print(f"Total Questions: {total_questions}") print(f"Passed: {passed_questions}") print(f"Failed: {failed_questions}") - print(f"Success Rate: {(passed_questions / total_questions) * 100:.1f}%") - print(f"Total Test Time: {format_duration(total_test_time)}") + print( + f"Success Rate: {(passed_questions / total_questions) * 100:.1f}%" + if total_questions > 0 + else "N/A" + ) + print(f"Total Test Time: {format_duration(total_duration)}") - efficiency_ratios: list[float] = [] + # Token efficiency stats + efficiency_ratios: list[Any] = [] for result in results: query = result.get("query_executed") if query: @@ -796,64 +583,39 @@ class LongMemEvalRunner(RunnerMixin): if efficiency_ratios: avg_efficiency = sum(efficiency_ratios) / len(efficiency_ratios) - min_efficiency = min(efficiency_ratios) - max_efficiency = max(efficiency_ratios) print("\nToken Efficiency:") print( f" Average: {avg_efficiency:.4f} ({avg_efficiency * 100:.2f}% of available tokens used)" ) - print(f" Min: {min_efficiency:.4f} ({min_efficiency * 100:.2f}%)") - print(f" Max: {max_efficiency:.4f} ({max_efficiency * 100:.2f}%)") + print(f" Min: {min(efficiency_ratios):.4f}") + print(f" Max: {max(efficiency_ratios):.4f}") print("\nDetailed Results:") - print( - f"{'Question ID':<15} {'Type':<20} {'Status':<8} {'Duration':<10} {'Workspace ID':<30}" - ) - print(f"{'-' * 15} {'-' * 20} {'-' * 8} {'-' * 10} {'-' * 30}") + print(f"{'Question ID':<15} {'Type':<20} {'Status':<8} {'Duration':<10}") + print(f"{'-' * 15} {'-' * 20} {'-' * 8} {'-' * 10}") for result in results: question_id = result["question_id"] question_type = result["question_type"] status = "PASS" if result.get("passed", False) else "FAIL" duration = format_duration(result["duration_seconds"]) - workspace = result["workspace_id"] - - print( - f"{question_id:<15} {question_type:<20} {status:<8} {duration:<10} {workspace:<30}" - ) + print(f"{question_id:<15} {question_type:<20} {status:<8} {duration:<10}") print(f"{'=' * 80}") - def generate_json_summary( - self, - results: list[TestResult], - test_file: Path, - total_elapsed_seconds: float, - output_file: Path | None = None, - ) -> None: - """ - Generate a comprehensive JSON summary of test results for analytics. - - Args: - results: List of test results - test_file: Path to the test file that was executed - total_elapsed_seconds: Total elapsed time for all tests - output_file: Optional path to write JSON output to - """ + def generate_output(self, results: list[TestResult], total_duration: float) -> None: + """Generate JSON output file.""" total_questions = len(results) passed_questions = sum(1 for r in results if r.get("passed", False)) - failed_questions = total_questions - passed_questions - # Calculate statistics by question type + # Calculate statistics type_stats = calculate_type_statistics(results) + timing_stats = calculate_timing_statistics(results, total_duration) - # Calculate timing statistics - timing_stats = calculate_timing_statistics(results, total_elapsed_seconds) - - # Calculate token efficiency statistics - efficiency_ratios: list[float] = [] - total_available_tokens_list: list[int] = [] - tokens_used_list: list[int] = [] + # Token efficiency stats + efficiency_ratios: list[Any] = [] + total_available_tokens_list: list[Any] = [] + tokens_used_list: list[Any] = [] for result in results: query = result.get("query_executed") if query: @@ -879,16 +641,15 @@ class LongMemEvalRunner(RunnerMixin): "total_questions_with_metrics": len(efficiency_ratios), } - # Create the full summary summary = { "metadata": { - "test_file": str(test_file), + "test_file": str(self.test_file), "execution_timestamp": datetime.now().isoformat(), - "runner_version": "1.0.0", - "base_api_port": self.base_api_port, - "pool_size": self.pool_size, - "timeout_seconds": self.timeout_seconds, - "reasoning_level": self.reasoning_level, + "runner_version": "2.0.0", + "base_api_port": self.config.base_api_port, + "pool_size": self.config.pool_size, + "timeout_seconds": self.config.timeout_seconds, + "reasoning_level": self.config.reasoning_level, "deriver_settings": settings.DERIVER.model_dump(), "dialectic_settings": settings.DIALECTIC.model_dump(), "dream_settings": settings.DREAM.model_dump(), @@ -897,7 +658,7 @@ class LongMemEvalRunner(RunnerMixin): "summary_statistics": { "total_questions": total_questions, "passed": passed_questions, - "failed": failed_questions, + "failed": total_questions - passed_questions, "success_rate_percent": (passed_questions / total_questions) * 100 if total_questions > 0 else 0, @@ -921,14 +682,19 @@ class LongMemEvalRunner(RunnerMixin): ], } - if output_file: - write_json_summary(summary, output_file) + # Determine output file + if self.config.json_output: + output_file = self.config.json_output + else: + output_file = Path( + f"tests/bench/eval_results/longmemeval_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) + + write_json_summary(summary, output_file) -async def main() -> int: - """ - Main entry point for the longmemeval test runner. - """ +def main() -> int: + """Main entry point for the longmemeval test runner.""" parser = argparse.ArgumentParser( description="Run longmemeval tests against a Honcho instance", formatter_class=argparse.RawDescriptionHelpFormatter, @@ -995,58 +761,20 @@ Examples: print(f"Error: Test count must be positive, got {args.test_count}") return 1 - # Create test runner + # Create config and runner + config = RunnerConfig.from_args(args, default_timeout=10000) + runner = LongMemEvalRunner( - base_api_port=args.base_api_port, - pool_size=args.pool_size, + config=config, + test_file=args.test_file, anthropic_api_key=args.anthropic_api_key, - timeout_seconds=args.timeout, merge_sessions=args.merge_sessions, - cleanup_workspace=args.cleanup_workspace, - use_get_context=args.use_get_context, - redis_url=args.redis_url, - reasoning_level=args.reasoning_level, + test_count=args.test_count, + question_id=args.question_id, ) - try: - # Run all questions - results, total_elapsed = await runner.run_all_questions( - args.test_file, args.batch_size, args.test_count, args.question_id - ) - runner.print_summary(results, total_elapsed_seconds=total_elapsed) - - # Print metrics summary - runner.metrics_collector.print_summary() - - # Generate JSON output if requested - if args.json_output: - runner.generate_json_summary( - results, args.test_file, total_elapsed, args.json_output - ) - else: - # Always generate a default JSON output file with timestamp - default_output = Path( - f"tests/bench/eval_results/longmemeval_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - ) - runner.generate_json_summary( - results, args.test_file, total_elapsed, default_output - ) - - # Export metrics to JSON file - export_metrics(runner.metrics_collector, "longmem") - - # Return exit code based on results - all_passed = all(r.get("passed", False) for r in results) - return 0 if all_passed else 1 - - except KeyboardInterrupt: - print("\nTest execution interrupted by user") - return 1 - except Exception as e: - print(f"Error running tests: {e}") - return 1 + return runner.run_and_summarize() if __name__ == "__main__": - exit_code = asyncio.run(main()) - exit(exit_code) + exit(main()) diff --git a/tests/bench/run_tests.py b/tests/bench/run_tests.py index 13ed113b..fcf141de 100644 --- a/tests/bench/run_tests.py +++ b/tests/bench/run_tests.py @@ -23,11 +23,15 @@ import tiktoken from anthropic import AsyncAnthropic from dotenv import load_dotenv from honcho import Honcho +from honcho.api_types import SessionConfiguration, SummaryConfiguration from honcho.session import SessionPeerConfig from typing_extensions import TypedDict load_dotenv() +# Default session configuration with summaries disabled for tests +_TEST_SESSION_CONFIG = SessionConfiguration(summary=SummaryConfiguration(enabled=False)) + class SessionResult(TypedDict): """Type definition for session creation results.""" @@ -319,8 +323,10 @@ Evaluate whether the actual response contains the core correct information from peers[peer_name] = await honcho_client.aio.peer(id=peer_name) for session_name, session_data in sessions.items(): - # Create session - session = await honcho_client.aio.session(id=str(session_name)) + # Create session with summaries disabled + session = await honcho_client.aio.session( + id=str(session_name), configuration=_TEST_SESSION_CONFIG + ) output_lines.append(f"\n session: {session_name}") @@ -507,7 +513,9 @@ Evaluate whether the actual response contains the core correct information from session_name = str(get_context_call["session"]) summary = get_context_call["summary"] max_tokens: int | None = get_context_call.get("max_tokens") - session = await honcho_client.aio.session(id=session_name) + session = await honcho_client.aio.session( + id=session_name, configuration=_TEST_SESSION_CONFIG + ) # Wait for deriver queue to be empty for this session # TODO implement this differently! diff --git a/tests/bench/runner_common.py b/tests/bench/runner_common.py index cbd64cd6..0cfcc79b 100644 --- a/tests/bench/runner_common.py +++ b/tests/bench/runner_common.py @@ -1,9 +1,8 @@ """ Shared utilities for Honcho benchmark test runners. -Contains common functionality for queue management, dream triggering, -Honcho client creation, and CLI argument parsing used across longmem, -beam, and locomo runners. +Contains the BaseRunner abstract class and RunnerConfig dataclass that provide +a common framework for all benchmark runners (longmem, beam, locomo). """ import argparse @@ -11,14 +10,17 @@ import asyncio import logging import os import time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field from datetime import datetime +from logging import Logger from pathlib import Path -from typing import Any, Literal +from typing import Any, Generic, Literal, TypeVar -import httpx import redis.asyncio as aioredis from anthropic import AsyncAnthropic from honcho import Honcho +from honcho.api_types import SessionConfiguration, SummaryConfiguration from openai import AsyncOpenAI from redis.asyncio.client import Redis @@ -28,6 +30,63 @@ from src.telemetry.metrics_collector import MetricsCollector ReasoningLevel = Literal["minimal", "low", "medium", "high", "max"] REASONING_LEVELS: list[str] = ["minimal", "low", "medium", "high", "max"] +# Type variable for result types +ResultT = TypeVar("ResultT") + + +@dataclass +class RunnerConfig: + """Configuration shared across all benchmark runners.""" + + base_api_port: int = 8000 + pool_size: int = 1 + timeout_seconds: int = 600 + batch_size: int = 10 + cleanup_workspace: bool = False + use_get_context: bool = False + redis_url: str = "redis://localhost:6379/0" + reasoning_level: ReasoningLevel | None = None + base_url: str | None = None + api_key: str | None = None + skip_dream: bool = False + json_output: Path | None = None + max_concurrent: int | None = None # None means no limit (use batch_size) + + @classmethod + def from_args( + cls, args: argparse.Namespace, default_timeout: int = 600 + ) -> "RunnerConfig": + """Create config from parsed CLI arguments.""" + return cls( + base_api_port=args.base_api_port, + pool_size=args.pool_size, + timeout_seconds=args.timeout + if args.timeout is not None + else default_timeout, + batch_size=args.batch_size, + cleanup_workspace=args.cleanup_workspace, + use_get_context=args.use_get_context, + redis_url=args.redis_url, + reasoning_level=args.reasoning_level, + base_url=args.base_url, + api_key=args.api_key, + skip_dream=args.skip_dream, + json_output=args.json_output, + max_concurrent=args.max_concurrent, + ) + + +@dataclass +class ItemContext: + """Context for executing a single benchmark item.""" + + workspace_id: str + honcho_client: Honcho + honcho_url: str + session_id: str + peers: dict[str, Any] = field(default_factory=dict) + session: Any = None + def add_common_arguments(parser: argparse.ArgumentParser) -> None: """ @@ -36,6 +95,20 @@ def add_common_arguments(parser: argparse.ArgumentParser) -> None: Args: parser: ArgumentParser to add arguments to """ + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Base URL for remote Honcho instance (e.g., https://groudon.fly.dev). Overrides --base-api-port.", + ) + + parser.add_argument( + "--api-key", + type=str, + default=None, + help="API key for remote Honcho instance authentication", + ) + parser.add_argument( "--base-api-port", type=int, @@ -97,6 +170,19 @@ def add_common_arguments(parser: argparse.ArgumentParser) -> None: help="Reasoning level for dialectic chat: minimal, low, medium, high, max (default: None)", ) + parser.add_argument( + "--skip-dream", + action="store_true", + help="Skip the dream consolidation step (default: False)", + ) + + parser.add_argument( + "--max-concurrent", + type=int, + default=None, + help="Maximum concurrent items executing at once (default: unlimited, use for rate-limited remote instances)", + ) + def validate_common_arguments(args: argparse.Namespace) -> str | None: """ @@ -114,6 +200,9 @@ def validate_common_arguments(args: argparse.Namespace) -> str | None: if args.pool_size <= 0: return f"Error: Pool size must be positive, got {args.pool_size}" + if args.max_concurrent is not None and args.max_concurrent <= 0: + return f"Error: Max concurrent must be positive, got {args.max_concurrent}" + return None @@ -185,105 +274,367 @@ def create_openai_client( return AsyncOpenAI(api_key=key) -def create_metrics_collector(prefix: str) -> MetricsCollector: +def format_duration(seconds: float) -> str: + """Format a duration in seconds to a human-readable string.""" + if seconds < 60: + return f"{seconds:.2f}s" + elif seconds < 3600: + minutes = int(seconds // 60) + secs = seconds % 60 + return f"{minutes}m {secs:.1f}s" + else: + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + return f"{hours}h {minutes}m" + + +class BaseRunner(ABC, Generic[ResultT]): """ - Create and start a MetricsCollector. + Abstract base class for benchmark runners. - Args: - prefix: Prefix for the collection name (e.g., "longmem", "beam", "locomo") + Provides a template method pattern for executing benchmarks with common + infrastructure for Honcho client management, queue waiting, and dream triggering. - Returns: - Started MetricsCollector instance - """ - collector = MetricsCollector() - collector.start_collection(f"{prefix}_{datetime.now().strftime('%Y%m%d_%H%M%S')}") - return collector - - -def export_metrics( - collector: MetricsCollector, - prefix: str, - output_dir: str = "tests/bench/perf_metrics", -) -> Path: - """ - Export metrics to a JSON file and cleanup the collector. - - Args: - collector: MetricsCollector instance - prefix: Prefix for the output filename - output_dir: Directory for output files - - Returns: - Path to the exported metrics file - """ - metrics_output = Path( - f"{output_dir}/{prefix}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - ) - collector.export_to_json(metrics_output) - collector.cleanup_collection() - return metrics_output - - -class RunnerMixin: - """ - Mixin class providing common functionality for benchmark runners. - - Requires the following attributes on the class: - - redis_url: str - - timeout_seconds: int - - base_api_port: int - - pool_size: int - - reasoning_level: ReasoningLevel | None (optional) + Subclasses must implement: + - get_metrics_prefix(): Return the metrics prefix (e.g., "longmem") + - load_items(): Load and return the items to process + - get_workspace_id(item): Return workspace ID for an item + - get_session_id(item): Return session ID for an item + - setup_peers(ctx, item): Create and configure peers + - setup_session(ctx, item): Create and configure session with peers + - ingest_messages(ctx, item): Ingest messages into the session + - get_dream_observers(item): Return list of peer IDs to trigger dreams for + - execute_questions(ctx, item): Execute questions and return result + - print_summary(results, duration): Print summary of results + - generate_output(results, duration): Generate JSON output """ - # These are expected to be set by the inheriting class's __init__ - redis_url: str = "" - timeout_seconds: int = 0 - base_api_port: int = 0 - pool_size: int = 0 - reasoning_level: ReasoningLevel | None = None - # These are initialized by _init_common() - use Any to satisfy type checker - # since the actual type is set at runtime - metrics_collector: Any = None - logger: Any = None - - def _init_common(self, metrics_prefix: str) -> None: + def __init__(self, config: RunnerConfig): """ - Initialize common runner components. - - Call this at the end of your __init__ after setting instance attributes. + Initialize the runner with configuration. Args: - metrics_prefix: Prefix for metrics collection (e.g., "longmem", "beam") + config: Runner configuration """ - self.metrics_collector = create_metrics_collector(metrics_prefix) - self.logger = configure_logging() - - def get_honcho_url_for_index(self, index: int) -> str: - """Get the Honcho URL for a given index using round-robin distribution.""" - instance_id = index % self.pool_size - port = self.base_api_port + instance_id - return f"http://localhost:{port}" - - def create_honcho_client(self, workspace_id: str, honcho_url: str) -> Honcho: - """Create a Honcho client for a specific workspace.""" - return Honcho( - environment="local", - workspace_id=workspace_id, - base_url=honcho_url, - timeout=300.0, + self.config: RunnerConfig = config + self.metrics_collector: MetricsCollector = MetricsCollector() + self.metrics_collector.start_collection( + f"{self.get_metrics_prefix()}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + ) + self.logger: Logger = configure_logging() + # Semaphore for rate limiting concurrent item execution + self._concurrency_semaphore: asyncio.Semaphore | None = ( + asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None ) - async def flush_deriver_queue(self) -> None: - """Enable deriver flush mode to bypass batch token threshold.""" - redis_client: Redis = aioredis.from_url(self.redis_url) # pyright: ignore[reportUnknownMemberType] + # ------------------------------------------------------------------------- + # Abstract methods - must be implemented by subclasses + # ------------------------------------------------------------------------- + + @abstractmethod + def get_metrics_prefix(self) -> str: + """Return the metrics prefix for this runner (e.g., 'longmem', 'beam').""" + ... + + @abstractmethod + def load_items(self) -> list[Any]: + """Load and return the list of items to process.""" + ... + + @abstractmethod + def get_workspace_id(self, item: Any) -> str: + """Return the workspace ID for a given item.""" + ... + + @abstractmethod + def get_session_id(self, item: Any, workspace_id: str) -> str: + """Return the session ID for a given item.""" + ... + + @abstractmethod + async def setup_peers(self, ctx: ItemContext, item: Any) -> None: + """ + Create and configure peers for the item. + + Should populate ctx.peers with peer objects. + """ + ... + + @abstractmethod + async def setup_session(self, ctx: ItemContext, item: Any) -> None: + """ + Create and configure the session with peers. + + Should set ctx.session and add peers to the session. + """ + ... + + @abstractmethod + async def ingest_messages(self, ctx: ItemContext, item: Any) -> int: + """ + Ingest messages into the session. + + Returns: + Number of messages ingested + """ + ... + + @abstractmethod + def get_dream_observers(self, item: Any) -> list[str]: + """Return list of peer IDs to trigger dreams for.""" + ... + + @abstractmethod + async def execute_questions(self, ctx: ItemContext, item: Any) -> ResultT: + """ + Execute questions/queries for the item. + + Returns: + Result object for this item + """ + ... + + @abstractmethod + def print_summary(self, results: list[ResultT], total_duration: float) -> None: + """Print a summary of all results.""" + ... + + @abstractmethod + def generate_output(self, results: list[ResultT], total_duration: float) -> None: + """Generate JSON output file.""" + ... + + # ------------------------------------------------------------------------- + # Template method - the main execution flow + # ------------------------------------------------------------------------- + + async def run(self) -> tuple[list[ResultT], float]: + """ + Run the benchmark. + + This is the main template method that orchestrates the execution flow. + + Returns: + Tuple of (list of results, total duration in seconds) + """ + items = self.load_items() + if not items: + return [], 0.0 + + print(f"Found {len(items)} items to process") + if self.config.pool_size > 1: + print( + f"Distributing across {self.config.pool_size} Honcho instances " + + f"(ports {self.config.base_api_port}-{self.config.base_api_port + self.config.pool_size - 1})" + ) + if self.config.max_concurrent: + print(f"Limiting to {self.config.max_concurrent} concurrent item(s)") + + overall_start = time.time() + all_results: list[ResultT] = [] + + # Process in batches + batch_size = self.config.batch_size + for i in range(0, len(items), batch_size): + batch = items[i : i + batch_size] + batch_num = (i // batch_size) + 1 + total_batches = (len(items) + batch_size - 1) // batch_size + + print(f"\n{'=' * 60}") + print(f"Processing batch {batch_num}/{total_batches} ({len(batch)} items)") + print(f"{'=' * 60}") + + # Run items in batch concurrently (with optional rate limiting) + batch_results = await asyncio.gather( + *[ + self._execute_item_with_limit(item, self._get_honcho_url(i + idx)) + for idx, item in enumerate(batch) + ] + ) + + all_results.extend(batch_results) + + overall_duration = time.time() - overall_start + + # Finalize metrics + self.metrics_collector.finalize_collection() + + return all_results, overall_duration + + async def _execute_item_with_limit(self, item: Any, honcho_url: str) -> ResultT: + """Wrapper that applies concurrency limiting if configured.""" + if self._concurrency_semaphore: + async with self._concurrency_semaphore: + return await self.execute_item(item, honcho_url) + return await self.execute_item(item, honcho_url) + + async def execute_item(self, item: Any, honcho_url: str) -> ResultT: + """ + Execute a single benchmark item. + + This method orchestrates the standard flow: + 1. Create workspace and client + 2. Setup peers and session + 3. Ingest messages + 4. Wait for queue to empty + 5. Trigger dreams + 6. Execute questions + 7. Cleanup (if configured) + + Args: + item: The item to process + honcho_url: URL of the Honcho instance to use + + Returns: + Result for this item + """ + workspace_id = self.get_workspace_id(item) + session_id = self.get_session_id(item, workspace_id) + + print(f"\n{'=' * 80}") + print(f"Executing {workspace_id}") + print(f"Using Honcho instance: {honcho_url}") + print(f"{'=' * 80}") + + # Create context + ctx = ItemContext( + workspace_id=workspace_id, + honcho_client=self._create_honcho_client(workspace_id, honcho_url), + honcho_url=honcho_url, + session_id=session_id, + ) + + start_time = time.time() + try: - await redis_client.set("honcho:deriver:flush_mode", "1", ex=60) - print("Enabled deriver flush mode") + # Setup peers + await self.setup_peers(ctx, item) + + # Setup session + await self.setup_session(ctx, item) + + # Ingest messages + print(f"[{workspace_id}] Ingesting messages...") + message_count = await self.ingest_messages(ctx, item) + print(f"[{workspace_id}] Ingested {message_count} messages") + + # Wait for deriver queue + print(f"[{workspace_id}] Waiting for deriver queue to empty...") + await asyncio.sleep(1) # Give time for tasks to be queued + await self._flush_deriver_queue() + + queue_empty = await self._wait_for_queue_empty(ctx.honcho_client) + if not queue_empty: + raise TimeoutError( + f"Deriver queue timeout after {self.config.timeout_seconds}s" + ) + + # Trigger dreams + print(f"[{workspace_id}] Deriver queue empty. Triggering dreams...") + for observer in self.get_dream_observers(item): + success = await self._trigger_dream( + ctx.honcho_client, workspace_id, observer, session_id + ) + if not success: + print( + f"[{workspace_id}] Warning: Dream for {observer} did not complete" + ) + + # Execute questions + print(f"[{workspace_id}] Executing questions...") + result = await self.execute_questions(ctx, item) + + # Cleanup + if self.config.cleanup_workspace: + try: + await ctx.honcho_client.aio.delete_workspace(workspace_id) + print(f"[{workspace_id}] Cleaned up workspace") + except Exception as e: + print(f"[{workspace_id}] Failed to delete workspace: {e}") + + duration = time.time() - start_time + print(f"[{workspace_id}] Completed in {format_duration(duration)}") + + return result + + except Exception as e: + self.logger.error(f"Error executing {workspace_id}: {e}") + # Let subclass handle error result creation + raise + + def run_and_summarize(self) -> int: + """ + Run the benchmark, print summary, and generate output. + + This is a convenience method that runs the full benchmark flow. + + Returns: + Exit code (0 for success, 1 for failure) + """ + try: + results, total_duration = asyncio.run(self.run()) + + self.print_summary(results, total_duration) + self.metrics_collector.print_summary() + self.generate_output(results, total_duration) + + # Export metrics + metrics_output = Path( + f"tests/bench/perf_metrics/{self.get_metrics_prefix()}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) + self.metrics_collector.export_to_json(metrics_output) + self.metrics_collector.cleanup_collection() + + return 0 + + except KeyboardInterrupt: + print("\nTest execution interrupted by user") + return 1 + except Exception as e: + print(f"Error running tests: {e}") + import traceback + + traceback.print_exc() + return 1 + + # ------------------------------------------------------------------------- + # Infrastructure methods + # ------------------------------------------------------------------------- + + def _get_honcho_url(self, index: int) -> str: + """Get the Honcho URL for a given index using round-robin distribution.""" + if self.config.base_url: + return self.config.base_url + instance_id = index % self.config.pool_size + port = self.config.base_api_port + instance_id + return f"http://localhost:{port}" + + def _create_honcho_client(self, workspace_id: str, honcho_url: str) -> Honcho: + """Create a Honcho client for a specific workspace.""" + return Honcho( + workspace_id=workspace_id, + base_url=honcho_url, + api_key=self.config.api_key, + ) + + def _get_session_configuration(self) -> SessionConfiguration: + """Get default session configuration with summaries disabled.""" + return SessionConfiguration(summary=SummaryConfiguration(enabled=False)) + + async def _flush_deriver_queue(self) -> None: + """Enable deriver flush mode to bypass batch token threshold.""" + default_redis_url = "redis://localhost:6379/0" + if self.config.base_url and self.config.redis_url == default_redis_url: + print("Skipping flush mode (remote instance, no --redis-url provided)") + return + redis_client: Redis = aioredis.from_url(self.config.redis_url) # pyright: ignore[reportUnknownMemberType] + try: + await redis_client.set("honcho:deriver:flush_mode", "1", ex=3600) + print(f"Enabled deriver flush mode via {self.config.redis_url}") finally: await redis_client.aclose() - async def wait_for_deriver_queue_empty( + async def _wait_for_queue_empty( self, honcho_client: Honcho, session_id: str | None = None ) -> bool: """Wait for the deriver queue to be empty.""" @@ -293,79 +644,55 @@ class RunnerMixin: status = await honcho_client.aio.queue_status(session=session_id) except Exception: await asyncio.sleep(1) - elapsed_time = time.time() - start_time - if elapsed_time >= self.timeout_seconds: + if time.time() - start_time >= self.config.timeout_seconds: return False continue if status.pending_work_units == 0 and status.in_progress_work_units == 0: return True - elapsed_time = time.time() - start_time - if elapsed_time >= self.timeout_seconds: + if time.time() - start_time >= self.config.timeout_seconds: return False await asyncio.sleep(1) - async def trigger_dream_and_wait( + async def _trigger_dream( self, honcho_client: Honcho, workspace_id: str, observer: str, + session_id: str, observed: str | None = None, - session_id: str | None = None, ) -> bool: """ Trigger a dream task and wait for it to complete. - Args: - honcho_client: Honcho client instance - workspace_id: Workspace identifier - observer: Observer peer name - observed: Observed peer name (defaults to observer) - session_id: Session ID to scope the dream to - Returns: - True if dream completed successfully, False on timeout + True if dream completed (or was skipped), False on timeout """ - observed = observed or observer - honcho_url = self.get_honcho_url_for_index(0) + if self.config.skip_dream: + print(f"[{workspace_id}] Skipping dream for {observer} (--skip-dream)") + return True - url = f"{honcho_url}/v3/workspaces/{workspace_id}/schedule_dream" - payload: dict[str, Any] = { - "observer": observer, - "observed": observed, - "dream_type": "omni", - "session_id": session_id or f"{workspace_id}_session", - } + observed = observed or observer try: - async with httpx.AsyncClient() as client: - response = await client.post( - url, - json=payload, - timeout=30.0, - ) - if response.status_code != 204: - print( - f"[{workspace_id}] ERROR: Dream trigger failed with status {response.status_code}" - ) - print(f"[{workspace_id}] Response body: {response.text}") - return False + await honcho_client.aio.schedule_dream( + observer=observer, + session=session_id, + observed=observed, + ) except Exception as e: print(f"[{workspace_id}] ERROR: Dream trigger exception: {e}") return False - print( - f"[{workspace_id}] Dream triggered successfully for {observer}/{observed}" - ) + print(f"[{workspace_id}] Dream triggered for {observer}/{observed}") - # Wait for dream queue to empty - print(f"[{workspace_id}] Waiting for dream to complete...") - await asyncio.sleep(2) # Give time for dream to be enqueued - await self.flush_deriver_queue() - success = await self.wait_for_deriver_queue_empty(honcho_client) + # Wait for dream to complete + await asyncio.sleep(2) + await self._flush_deriver_queue() + success = await self._wait_for_queue_empty(honcho_client) if success: - print(f"[{workspace_id}] Dream queue empty") + print(f"[{workspace_id}] Dream for {observer} completed") else: - print(f"[{workspace_id}] Dream queue timeout") + print(f"[{workspace_id}] Dream for {observer} timed out") return success diff --git a/tests/deriver/test_queue_processing.py b/tests/deriver/test_queue_processing.py index 9ca78f97..51c8af07 100644 --- a/tests/deriver/test_queue_processing.py +++ b/tests/deriver/test_queue_processing.py @@ -347,7 +347,7 @@ class TestQueueProcessing: *, observed: str | None = None, # pyright: ignore[reportUnusedParameter] observers: list[str] | None = None, # pyright: ignore[reportUnusedParameter] - queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter] + queue_item_message_ids: list[int] | None = None, # pyright: ignore[reportUnusedParameter] ) -> None: processed_batches.append( { @@ -910,7 +910,7 @@ class TestQueueProcessing: *, observed: str | None = None, # pyright: ignore[reportUnusedParameter] observers: list[str] | None = None, # pyright: ignore[reportUnusedParameter] - queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter] + queue_item_message_ids: list[int] | None = None, # pyright: ignore[reportUnusedParameter] ) -> None: processed_batches.append( { @@ -1029,7 +1029,7 @@ class TestQueueProcessing: *, observed: str | None = None, # pyright: ignore[reportUnusedParameter] observers: list[str] | None = None, # pyright: ignore[reportUnusedParameter] - queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter] + queue_item_message_ids: list[int] | None = None, # pyright: ignore[reportUnusedParameter] ) -> None: processed_batches.append( { diff --git a/tests/integration/test_token_metrics.py b/tests/integration/test_token_metrics.py index aef7c094..31163474 100644 --- a/tests/integration/test_token_metrics.py +++ b/tests/integration/test_token_metrics.py @@ -1,23 +1,21 @@ -"""Integration tests for OpenTelemetry token metrics tracking. +# pyright: reportPrivateUsage=false, reportUnknownVariableType=false +"""Integration tests for Prometheus token metrics tracking. These tests verify that deriver and dialectic token metrics are correctly emitted with accurate token counts when processing messages and dialectic queries. -The approach uses delta-based verification with OTel's InMemoryMetricReader: +The approach uses delta-based verification by directly accessing Prometheus counters: 1. Capture counter values before test execution 2. Run the code under test (with mocked LLM) 3. Verify deltas match expected values """ from collections.abc import Iterator -from typing import Any, cast from unittest.mock import AsyncMock, patch import pytest from nanoid import generate as generate_nanoid -from opentelemetry.metrics import Meter -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from prometheus_client import Counter from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models, schemas @@ -29,12 +27,15 @@ from src.schemas import ( ResolvedReasoningConfiguration, ResolvedSummaryConfiguration, ) -from src.telemetry.otel.metrics import otel_metrics +from src.telemetry.prometheus.metrics import ( + deriver_tokens_processed_counter, + dialectic_tokens_processed_counter, +) from src.utils.clients import HonchoLLMCallResponse from src.utils.representation import ExplicitObservationBase, PromptRepresentation from src.utils.summarizer import ( SummaryType, - _create_and_save_summary, # pyright: ignore[reportPrivateUsage] + _create_and_save_summary, estimate_short_summary_prompt_tokens, ) @@ -43,137 +44,70 @@ from src.utils.summarizer import ( # ============================================================================= -class OTelMetricChecker: - """Utility class to capture and verify OTel counter deltas.""" +class PrometheusMetricChecker: + """Utility class to capture and verify Prometheus counter deltas.""" - _reader: InMemoryMetricReader - - def __init__(self, reader: InMemoryMetricReader): - self._reader = reader - - def _get_metric_value(self, metric_name: str, labels: dict[str, str]) -> float: + def _get_counter_value(self, counter: Counter, labels: dict[str, str]) -> float: """Get current value of a counter with specific labels. - Note: The OTel SDK's MetricsData types are not fully typed, so we cast - to Any to avoid type warnings when traversing the metrics data structure. + Note: For prometheus_client counters, we access the internal _value + of the labeled metric. This is implementation-specific but works for testing. """ - raw_data = self._reader.get_metrics_data() # pyright: ignore[reportUnknownVariableType] - if raw_data is None: + try: + labeled_counter = counter.labels(**labels) + return labeled_counter._value.get() + except Exception: return 0.0 - data = cast(Any, raw_data) - for resource_metrics in data.resource_metrics: - for scope_metrics in resource_metrics.scope_metrics: - for metric in scope_metrics.metrics: - if metric.name == metric_name and hasattr( - metric.data, "data_points" - ): - for point in metric.data.data_points: - # Check if labels match - point_attrs: dict[str, str] = ( - dict(point.attributes) if point.attributes else {} - ) - if all(point_attrs.get(k) == v for k, v in labels.items()): - return float(point.value) - return 0.0 - - def capture(self, metric_name: str, labels: dict[str, str]) -> float: + def capture(self, counter: Counter, labels: dict[str, str]) -> float: """Capture current value of a counter with specific labels.""" - return self._get_metric_value(metric_name, labels) + return self._get_counter_value(counter, labels) def get_delta( - self, metric_name: str, labels: dict[str, str], before: float + self, counter: Counter, labels: dict[str, str], before: float ) -> float: """Get the delta between a before value and current.""" - return self.capture(metric_name, labels) - before + return self.capture(counter, labels) - before def assert_delta( self, - metric_name: str, + counter: Counter, labels: dict[str, str], before: float, expected: int | float, message: str = "", ) -> None: """Assert that the delta matches expected value.""" - delta = self.get_delta(metric_name, labels, before) + delta = self.get_delta(counter, labels, before) assert ( delta == expected ), f"{message}: expected delta {expected}, got {delta}. Labels: {labels}" -def _reset_otel_metrics_singleton() -> None: - """Reset the otel_metrics singleton instance so it reinitializes with a new provider. - - This clears all instance attributes so _ensure_initialized() will create - new meters tied to the current global MeterProvider. - """ - # Reset initialization flag (instance attribute shadows class attribute) - otel_metrics._is_initialized = False # pyright: ignore[reportPrivateUsage] - - # Reset all meters - otel_metrics._api_meter = None # pyright: ignore[reportPrivateUsage] - otel_metrics._deriver_meter = None # pyright: ignore[reportPrivateUsage] - otel_metrics._dialectic_meter = None # pyright: ignore[reportPrivateUsage] - otel_metrics._dreamer_meter = None # pyright: ignore[reportPrivateUsage] - - # Reset all counters - otel_metrics._api_requests = None # pyright: ignore[reportPrivateUsage] - otel_metrics._messages_created = None # pyright: ignore[reportPrivateUsage] - otel_metrics._dialectic_calls = None # pyright: ignore[reportPrivateUsage] - otel_metrics._deriver_queue_items = None # pyright: ignore[reportPrivateUsage] - otel_metrics._deriver_tokens = None # pyright: ignore[reportPrivateUsage] - otel_metrics._dialectic_tokens = None # pyright: ignore[reportPrivateUsage] - otel_metrics._dreamer_tokens = None # pyright: ignore[reportPrivateUsage] - - @pytest.fixture -def otel_test_setup( +def prometheus_test_setup( monkeypatch: pytest.MonkeyPatch, -) -> Iterator[tuple[InMemoryMetricReader, OTelMetricChecker]]: - """Set up OTel metrics with in-memory reader for testing. - - The OTel SDK only allows set_meter_provider() to be called once per process. - To work around this for testing, we patch get_meter() to return meters - from our test provider directly. +) -> Iterator[PrometheusMetricChecker]: + """Set up Prometheus metrics for testing. Yields: - Tuple of (reader, checker) for verifying metrics + A PrometheusMetricChecker for verifying metrics """ - # Create in-memory reader - reader = InMemoryMetricReader() + # Enable METRICS in settings and set namespace for test assertions + monkeypatch.setattr("src.config.settings.METRICS.ENABLED", True) + monkeypatch.setattr("src.config.settings.METRICS.NAMESPACE", "test") - # Create a test meter provider - provider = MeterProvider(metric_readers=[reader]) + checker = PrometheusMetricChecker() - # Patch get_meter to return meters from our test provider - # This is necessary because set_meter_provider() can only be called once per process - def test_get_meter(name: str, version: str = "") -> Meter: - return provider.get_meter(name, version) - - monkeypatch.setattr("src.telemetry.otel.metrics.get_meter", test_get_meter) - - # Reset the otel_metrics singleton instance so it reinitializes with our test provider - _reset_otel_metrics_singleton() - - # Enable OTEL in settings and set namespace for test assertions - monkeypatch.setattr("src.config.settings.OTEL.ENABLED", True) - monkeypatch.setattr("src.config.settings.OTEL.SERVICE_NAMESPACE", "test") - - checker = OTelMetricChecker(reader) - - yield reader, checker - - # Cleanup: reset singleton so other tests aren't affected - _reset_otel_metrics_singleton() + yield checker @pytest.fixture def metric_checker( - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], -) -> OTelMetricChecker: + prometheus_test_setup: PrometheusMetricChecker, +) -> PrometheusMetricChecker: """Fixture providing a metric checker instance.""" - return otel_test_setup[1] + return prometheus_test_setup # ============================================================================= @@ -285,12 +219,12 @@ class TestDeriverIngestionMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify OUTPUT_TOTAL tokens match response.output_tokens from LLM.""" from src.deriver.deriver import process_representation_tasks_batch - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -309,7 +243,7 @@ class TestDeriverIngestionMetrics: "token_type": "output", "component": "output_total", } - before = metric_checker.capture("deriver_tokens_processed", labels) + before = metric_checker.capture(deriver_tokens_processed_counter, labels) # Mock the LLM call and save_representation (we're testing metrics, not DB writes) with ( @@ -327,12 +261,12 @@ class TestDeriverIngestionMetrics: message_level_configuration=create_test_configuration(), observers=[peer.name], observed=peer.name, - queue_items_count=len(messages), + queue_item_message_ids=[m.id for m in messages], ) # Verify output tokens metric metric_checker.assert_delta( - "deriver_tokens_processed", + deriver_tokens_processed_counter, labels, before, expected_output_tokens, @@ -343,13 +277,13 @@ class TestDeriverIngestionMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify PROMPT component is tracked for ingestion input.""" from src.deriver.deriver import process_representation_tasks_batch from src.deriver.prompts import estimate_minimal_deriver_prompt_tokens - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -367,7 +301,7 @@ class TestDeriverIngestionMetrics: "token_type": "input", "component": "prompt", } - before = metric_checker.capture("deriver_tokens_processed", labels) + before = metric_checker.capture(deriver_tokens_processed_counter, labels) with ( patch( @@ -384,11 +318,11 @@ class TestDeriverIngestionMetrics: message_level_configuration=create_test_configuration(), observers=[peer.name], observed=peer.name, - queue_items_count=len(messages), + queue_item_message_ids=[m.id for m in messages], ) metric_checker.assert_delta( - "deriver_tokens_processed", + deriver_tokens_processed_counter, labels, before, expected_prompt_tokens, @@ -399,12 +333,12 @@ class TestDeriverIngestionMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify MESSAGES component is tracked for ingestion input.""" from src.deriver.deriver import process_representation_tasks_batch - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -424,7 +358,7 @@ class TestDeriverIngestionMetrics: "token_type": "input", "component": "messages", } - before = metric_checker.capture("deriver_tokens_processed", labels) + before = metric_checker.capture(deriver_tokens_processed_counter, labels) with ( patch( @@ -441,11 +375,13 @@ class TestDeriverIngestionMetrics: message_level_configuration=create_test_configuration(), observers=[peer.name], observed=peer.name, - queue_items_count=len(messages), + queue_item_message_ids=[m.id for m in messages], ) # Verify messages tokens were tracked (should be > 0) - delta = metric_checker.get_delta("deriver_tokens_processed", labels, before) + delta = metric_checker.get_delta( + deriver_tokens_processed_counter, labels, before + ) assert delta > 0, f"Expected messages input tokens > 0, got {delta}" @@ -462,11 +398,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify OUTPUT_TOTAL tokens are tracked for summary.""" - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -491,7 +427,7 @@ class TestDeriverSummaryMetrics: "token_type": "output", "component": "output_total", } - before = metric_checker.capture("deriver_tokens_processed", labels) + before = metric_checker.capture(deriver_tokens_processed_counter, labels) with ( patch( @@ -517,7 +453,7 @@ class TestDeriverSummaryMetrics: # Verify output tokens match the summary token_count metric_checker.assert_delta( - "deriver_tokens_processed", + deriver_tokens_processed_counter, labels, before, expected_output_tokens, @@ -528,11 +464,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify PROMPT component is tracked for summary input.""" - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -555,7 +491,7 @@ class TestDeriverSummaryMetrics: "token_type": "input", "component": "prompt", } - before = metric_checker.capture("deriver_tokens_processed", labels) + before = metric_checker.capture(deriver_tokens_processed_counter, labels) with ( patch( @@ -580,7 +516,7 @@ class TestDeriverSummaryMetrics: ) metric_checker.assert_delta( - "deriver_tokens_processed", + deriver_tokens_processed_counter, labels, before, expected_prompt_tokens, @@ -591,11 +527,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify MESSAGES component is tracked for summary input.""" - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -627,7 +563,7 @@ class TestDeriverSummaryMetrics: "token_type": "input", "component": "messages", } - before = metric_checker.capture("deriver_tokens_processed", labels) + before = metric_checker.capture(deriver_tokens_processed_counter, labels) with ( patch( @@ -652,7 +588,9 @@ class TestDeriverSummaryMetrics: ) # Verify messages tokens match what summarizer actually computed - delta = metric_checker.get_delta("deriver_tokens_processed", labels, before) + delta = metric_checker.get_delta( + deriver_tokens_processed_counter, labels, before + ) assert ( delta == expected_messages_tokens ), f"Expected messages input tokens {expected_messages_tokens}, got {delta}" @@ -662,11 +600,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify metrics are NOT emitted when _create_summary returns is_fallback=True.""" - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -695,10 +633,10 @@ class TestDeriverSummaryMetrics: "component": "prompt", } before_output = metric_checker.capture( - "deriver_tokens_processed", output_labels + deriver_tokens_processed_counter, output_labels ) before_prompt = metric_checker.capture( - "deriver_tokens_processed", prompt_labels + deriver_tokens_processed_counter, prompt_labels ) with patch( @@ -719,10 +657,10 @@ class TestDeriverSummaryMetrics: # Verify NO change in metrics when fallback output_delta = metric_checker.get_delta( - "deriver_tokens_processed", output_labels, before_output + deriver_tokens_processed_counter, output_labels, before_output ) prompt_delta = metric_checker.get_delta( - "deriver_tokens_processed", prompt_labels, before_prompt + deriver_tokens_processed_counter, prompt_labels, before_prompt ) assert ( @@ -746,12 +684,12 @@ class TestDialecticTokenMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify INPUT tokens are tracked from LLM response.""" from src.dialectic.core import DialecticAgent - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -766,7 +704,7 @@ class TestDialecticTokenMetrics: "component": "total", "reasoning_level": "low", } - before = metric_checker.capture("dialectic_tokens_processed", labels) + before = metric_checker.capture(dialectic_tokens_processed_counter, labels) agent = DialecticAgent( db=db_session, @@ -783,7 +721,7 @@ class TestDialecticTokenMetrics: await agent.answer("What do you know about this user?") metric_checker.assert_delta( - "dialectic_tokens_processed", + dialectic_tokens_processed_counter, labels, before, expected_input_tokens, @@ -794,12 +732,12 @@ class TestDialecticTokenMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, ): """Verify OUTPUT tokens are tracked from LLM response.""" from src.dialectic.core import DialecticAgent - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -814,7 +752,7 @@ class TestDialecticTokenMetrics: "component": "total", "reasoning_level": "low", } - before = metric_checker.capture("dialectic_tokens_processed", labels) + before = metric_checker.capture(dialectic_tokens_processed_counter, labels) agent = DialecticAgent( db=db_session, @@ -831,7 +769,7 @@ class TestDialecticTokenMetrics: await agent.answer("What do you know about this user?") metric_checker.assert_delta( - "dialectic_tokens_processed", + dialectic_tokens_processed_counter, labels, before, expected_output_tokens, @@ -842,16 +780,16 @@ class TestDialecticTokenMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], + prometheus_test_setup: PrometheusMetricChecker, monkeypatch: pytest.MonkeyPatch, ): - """Verify metrics are NOT emitted when OTEL.ENABLED=False.""" + """Verify metrics are NOT emitted when PROMETHEUS.ENABLED=False.""" from src.dialectic.core import DialecticAgent - _, metric_checker = otel_test_setup + metric_checker = prometheus_test_setup - # Explicitly disable OTEL metrics - monkeypatch.setattr("src.config.settings.OTEL.ENABLED", False) + # Explicitly disable Prometheus metrics + monkeypatch.setattr("src.config.settings.METRICS.ENABLED", False) workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -874,10 +812,10 @@ class TestDialecticTokenMetrics: "reasoning_level": "low", } before_input = metric_checker.capture( - "dialectic_tokens_processed", input_labels + dialectic_tokens_processed_counter, input_labels ) before_output = metric_checker.capture( - "dialectic_tokens_processed", output_labels + dialectic_tokens_processed_counter, output_labels ) agent = DialecticAgent( @@ -896,10 +834,10 @@ class TestDialecticTokenMetrics: # Verify NO change in metrics input_delta = metric_checker.get_delta( - "dialectic_tokens_processed", input_labels, before_input + dialectic_tokens_processed_counter, input_labels, before_input ) output_delta = metric_checker.get_delta( - "dialectic_tokens_processed", output_labels, before_output + dialectic_tokens_processed_counter, output_labels, before_output ) assert ( diff --git a/tests/routes/test_conclusions.py b/tests/routes/test_conclusions.py index 58076250..a2ef56a0 100644 --- a/tests/routes/test_conclusions.py +++ b/tests/routes/test_conclusions.py @@ -1149,3 +1149,160 @@ class TestConclusionRoutes: data = list_response.json() ids = [obs["id"] for obs in data["items"]] assert created_id in ids + + @pytest.mark.asyncio + async def test_create_conclusion_without_session_id( + self, + client: TestClient, + db_session: AsyncSession, + sample_data: tuple[Workspace, Peer], + ): + """Test creating a conclusion without session_id (sessionless/global conclusion)""" + test_workspace, test_peer = sample_data + + # Create another peer + test_peer2 = models.Peer( + name=str(generate_nanoid()), workspace_name=test_workspace.name + ) + db_session.add(test_peer2) + await db_session.commit() + + # Create conclusion without session_id + response = client.post( + f"/v3/workspaces/{test_workspace.name}/conclusions", + json={ + "conclusions": [ + { + "content": "User prefers dark mode (global)", + "observer_id": test_peer.name, + "observed_id": test_peer2.name, + # No session_id - this is the key test + } + ] + }, + ) + + assert response.status_code == 201 + data = response.json() + assert len(data) == 1 + + conclusion = data[0] + assert conclusion["content"] == "User prefers dark mode (global)" + assert conclusion["observer_id"] == test_peer.name + assert conclusion["observed_id"] == test_peer2.name + assert conclusion["session_id"] is None # Should be null + assert "id" in conclusion + assert "created_at" in conclusion + + @pytest.mark.asyncio + async def test_create_conclusions_mixed_session_and_sessionless( + self, + client: TestClient, + db_session: AsyncSession, + sample_data: tuple[Workspace, Peer], + ): + """Test creating a batch with both session-scoped and sessionless conclusions""" + test_workspace, test_peer = sample_data + + # Create another peer + test_peer2 = models.Peer( + name=str(generate_nanoid()), workspace_name=test_workspace.name + ) + db_session.add(test_peer2) + await db_session.flush() + + # Create a session + test_session = models.Session( + name=str(generate_nanoid()), workspace_name=test_workspace.name + ) + db_session.add(test_session) + await db_session.commit() + + # Create mixed batch: one with session, one without + response = client.post( + f"/v3/workspaces/{test_workspace.name}/conclusions", + json={ + "conclusions": [ + { + "content": "Session-scoped conclusion", + "observer_id": test_peer.name, + "observed_id": test_peer2.name, + "session_id": test_session.name, + }, + { + "content": "Global conclusion without session", + "observer_id": test_peer.name, + "observed_id": test_peer2.name, + # No session_id + }, + ] + }, + ) + + assert response.status_code == 201 + data = response.json() + assert len(data) == 2 + + # Find conclusions by content + session_conclusion = next( + c for c in data if c["content"] == "Session-scoped conclusion" + ) + global_conclusion = next( + c for c in data if c["content"] == "Global conclusion without session" + ) + + assert session_conclusion["session_id"] == test_session.name + assert global_conclusion["session_id"] is None + + @pytest.mark.asyncio + async def test_list_sessionless_conclusions( + self, + client: TestClient, + db_session: AsyncSession, + sample_data: tuple[Workspace, Peer], + ): + """Test that sessionless conclusions can be listed without session filter""" + test_workspace, test_peer = sample_data + + # Create another peer + test_peer2 = models.Peer( + name=str(generate_nanoid()), workspace_name=test_workspace.name + ) + db_session.add(test_peer2) + await db_session.commit() + + # Create sessionless conclusion + create_response = client.post( + f"/v3/workspaces/{test_workspace.name}/conclusions", + json={ + "conclusions": [ + { + "content": "Sessionless conclusion for list test", + "observer_id": test_peer.name, + "observed_id": test_peer2.name, + } + ] + }, + ) + assert create_response.status_code == 201 + created_id = create_response.json()[0]["id"] + + # List all conclusions (no session filter) + list_response = client.post( + f"/v3/workspaces/{test_workspace.name}/conclusions/list", + json={ + "filters": { + "observer_id": test_peer.name, + "observed_id": test_peer2.name, + } + }, + ) + + assert list_response.status_code == 200 + data = list_response.json() + ids = [obs["id"] for obs in data["items"]] + assert created_id in ids + + # Verify the conclusion has null session_id + conclusion = next(c for c in data["items"] if c["id"] == created_id) + assert conclusion["session_id"] is None diff --git a/tests/routes/test_sessions.py b/tests/routes/test_sessions.py index fe8379f8..c7025253 100644 --- a/tests/routes/test_sessions.py +++ b/tests/routes/test_sessions.py @@ -1149,10 +1149,10 @@ def test_get_session_context_peer_perspective_without_target_fails( assert "peer_target" in error_detail.lower() -def test_get_session_context_with_last_message( +def test_get_session_context_with_search_query( client: TestClient, sample_data: tuple[Workspace, Peer] ): - """Test session context with last_message parameter for semantic search""" + """Test session context with search_query parameter for semantic search""" test_workspace, test_peer = sample_data session_id = str(generate_nanoid()) @@ -1162,12 +1162,12 @@ def test_get_session_context_with_last_message( json={"id": session_id, "peers": {test_peer.name: {}}}, ) - # Get context with last_message and peer_target + # Get context with search_query and peer_target response = client.get( f"/v3/workspaces/{test_workspace.name}/sessions/{session_id}/context", params={ "peer_target": test_peer.name, - "last_message": "What is my favorite color?", + "search_query": "What is my favorite color?", }, ) assert response.status_code == 200 @@ -1193,7 +1193,7 @@ def test_get_session_context_with_limit_to_session( f"/v3/workspaces/{test_workspace.name}/sessions/{session_id}/context", params={ "peer_target": test_peer.name, - "last_message": "Test query", + "search_query": "Test query", "limit_to_session": True, }, ) @@ -1220,7 +1220,7 @@ def test_get_session_context_with_search_parameters( f"/v3/workspaces/{test_workspace.name}/sessions/{session_id}/context", params={ "peer_target": test_peer.name, - "last_message": "Test query", + "search_query": "Test query", "search_top_k": 5, "search_max_distance": 0.8, # float value (semantic distance 0.0-1.0) }, @@ -1248,7 +1248,7 @@ def test_get_session_context_with_include_most_frequent( f"/v3/workspaces/{test_workspace.name}/sessions/{session_id}/context", params={ "peer_target": test_peer.name, - "last_message": "Test query", + "search_query": "Test query", "include_most_frequent": True, }, ) @@ -1275,7 +1275,7 @@ def test_get_session_context_with_max_observations( f"/v3/workspaces/{test_workspace.name}/sessions/{session_id}/context", params={ "peer_target": test_peer.name, - "last_message": "Test query", + "search_query": "Test query", "max_observations": 10, }, ) @@ -1312,7 +1312,7 @@ def test_get_session_context_with_all_representation_params( "tokens": 500, "peer_target": test_peer.name, "peer_perspective": peer2_name, - "last_message": "What do you know about me?", + "search_query": "What do you know about me?", "limit_to_session": True, "search_top_k": 10, "search_max_distance": 0.9, # float value (semantic distance 0.0-1.0) diff --git a/tests/sdk/test_conclusions.py b/tests/sdk/test_conclusions.py index 7e910404..33331de3 100644 --- a/tests/sdk/test_conclusions.py +++ b/tests/sdk/test_conclusions.py @@ -609,3 +609,163 @@ async def test_observation_scope_via_peer_string( assert len(created) == 1 assert created[0].observed_id == target.id + + +@pytest.mark.asyncio +async def test_observation_create_without_session_id( + client_fixture: tuple[Honcho, str], +): + """ + Tests creating observations without a session_id (sessionless/global conclusions). + """ + honcho_client, client_type = client_fixture + + if client_type == "async": + observer = await honcho_client.aio.peer(id="test-obs-no-session-observer") + target = await honcho_client.aio.peer(id="test-obs-no-session-target") + + # Create a session just to ensure peers exist + session = await honcho_client.aio.session(id="test-obs-no-session-session") + await session.aio.add_messages( + [ + observer.message("Hello from observer"), + target.message("Hello from target"), + ] + ) + + # Get observation scope for observer -> target + obs_scope = observer.conclusions_of(target) + + # Create observation WITHOUT session_id + created = await obs_scope.aio.create( + [ + ConclusionCreateParams( + content="Global observation without session", + # No session_id - this is the key test + ) + ] + ) + + assert len(created) == 1 + assert isinstance(created[0], Conclusion) + assert created[0].content == "Global observation without session" + assert created[0].observer_id == observer.id + assert created[0].observed_id == target.id + assert created[0].session_id is None # Should be None + assert created[0].id # Has an ID + else: + observer = honcho_client.peer(id="test-obs-no-session-observer") + target = honcho_client.peer(id="test-obs-no-session-target") + + # Create a session just to ensure peers exist + session = honcho_client.session(id="test-obs-no-session-session") + session.add_messages( + [ + observer.message("Hello from observer"), + target.message("Hello from target"), + ] + ) + + # Get observation scope for observer -> target + obs_scope = observer.conclusions_of(target) + + # Create observation WITHOUT session_id + created = obs_scope.create( + [ + ConclusionCreateParams( + content="Global observation without session", + # No session_id - this is the key test + ) + ] + ) + + assert len(created) == 1 + assert isinstance(created[0], Conclusion) + assert created[0].content == "Global observation without session" + assert created[0].observer_id == observer.id + assert created[0].observed_id == target.id + assert created[0].session_id is None # Should be None + assert created[0].id # Has an ID + + +@pytest.mark.asyncio +async def test_observation_create_mixed_session_and_sessionless( + client_fixture: tuple[Honcho, str], +): + """ + Tests creating a batch with both session-scoped and sessionless observations. + """ + honcho_client, client_type = client_fixture + + if client_type == "async": + observer = await honcho_client.aio.peer(id="test-obs-mixed-session-observer") + target = await honcho_client.aio.peer(id="test-obs-mixed-session-target") + session = await honcho_client.aio.session(id="test-obs-mixed-session-session") + + # Ensure session and both peers exist + await session.aio.add_messages( + [ + observer.message("Hello from observer"), + target.message("Hello from target"), + ] + ) + + # Get observation scope + obs_scope = observer.conclusions_of(target) + + # Create mixed batch: one with session, one without + created = await obs_scope.aio.create( + [ + {"content": "Session-scoped observation", "session_id": session.id}, + {"content": "Global observation without session"}, # No session_id + ] + ) + + assert len(created) == 2 + + # Find observations by content + session_obs = next( + c for c in created if c.content == "Session-scoped observation" + ) + global_obs = next( + c for c in created if c.content == "Global observation without session" + ) + + assert session_obs.session_id == session.id + assert global_obs.session_id is None + else: + observer = honcho_client.peer(id="test-obs-mixed-session-observer") + target = honcho_client.peer(id="test-obs-mixed-session-target") + session = honcho_client.session(id="test-obs-mixed-session-session") + + # Ensure session and both peers exist + session.add_messages( + [ + observer.message("Hello from observer"), + target.message("Hello from target"), + ] + ) + + # Get observation scope + obs_scope = observer.conclusions_of(target) + + # Create mixed batch: one with session, one without + created = obs_scope.create( + [ + {"content": "Session-scoped observation", "session_id": session.id}, + {"content": "Global observation without session"}, # No session_id + ] + ) + + assert len(created) == 2 + + # Find observations by content + session_obs = next( + c for c in created if c.content == "Session-scoped observation" + ) + global_obs = next( + c for c in created if c.content == "Global observation without session" + ) + + assert session_obs.session_id == session.id + assert global_obs.session_id is None diff --git a/tests/telemetry/conftest.py b/tests/telemetry/conftest.py index 6bce95eb..bf296ef3 100644 --- a/tests/telemetry/conftest.py +++ b/tests/telemetry/conftest.py @@ -337,7 +337,7 @@ def mock_telemetry_settings(): mock_settings.TELEMETRY.MAX_RETRIES = max_retries mock_settings.TELEMETRY.MAX_BUFFER_SIZE = max_buffer_size mock_settings.TELEMETRY.HEADERS = headers - mock_settings.OTEL.ENABLED = False + mock_settings.METRICS.ENABLED = False return patch("src.telemetry.emitter.settings", mock_settings) return _configure diff --git a/tests/telemetry/test_emit_function.py b/tests/telemetry/test_emit_function.py index 6b0d1b5e..89ddbeee 100644 --- a/tests/telemetry/test_emit_function.py +++ b/tests/telemetry/test_emit_function.py @@ -350,56 +350,6 @@ class TestShutdownTelemetryEvents: mock_shutdown.assert_called_once() -# ============================================================================= -# Tests for initialize_telemetry() function -# ============================================================================= - - -class TestInitializeTelemetry: - """Tests for initialize_telemetry() in src.telemetry.""" - - def test_initialize_otel_when_enabled(self): - """initialize_telemetry() initializes OTel metrics when enabled.""" - import src.telemetry as telemetry_module - - with ( - patch("src.config.settings") as mock_settings, - # Patch where it's used (in src.telemetry), not where it's defined - patch.object(telemetry_module, "initialize_otel_metrics") as mock_otel_init, - ): - mock_settings.OTEL.ENABLED = True - mock_settings.OTEL.ENDPOINT = "http://otel:9009/metrics" - mock_settings.OTEL.HEADERS = None - mock_settings.OTEL.EXPORT_INTERVAL_MILLIS = 60000 - mock_settings.OTEL.SERVICE_NAME = "honcho" - mock_settings.OTEL.SERVICE_NAMESPACE = "test" - - telemetry_module.initialize_telemetry() - - mock_otel_init.assert_called_once_with( - endpoint="http://otel:9009/metrics", - headers=None, - export_interval_millis=60000, - service_name="honcho", - service_namespace="test", - enabled=True, - ) - - def test_skip_otel_when_disabled(self): - """initialize_telemetry() skips OTel when disabled.""" - import src.telemetry as telemetry_module - - with ( - patch("src.config.settings") as mock_settings, - patch.object(telemetry_module, "initialize_otel_metrics") as mock_otel_init, - ): - mock_settings.OTEL.ENABLED = False - - telemetry_module.initialize_telemetry() - - mock_otel_init.assert_not_called() - - # ============================================================================= # Tests for initialize_telemetry_async() function # ============================================================================= @@ -454,17 +404,13 @@ class TestShutdownTelemetry: """Tests for shutdown_telemetry() in src.telemetry.""" @pytest.mark.asyncio - async def test_shutdown_calls_all_subsystems(self): - """shutdown_telemetry() shuts down all telemetry subsystems.""" + async def test_shutdown_calls_cloudevents(self): + """shutdown_telemetry() shuts down CloudEvents emitter.""" from src.telemetry import shutdown_telemetry - with ( - patch( - "src.telemetry.events.shutdown_telemetry_events", new_callable=AsyncMock - ) as mock_ce_shutdown, - patch("src.telemetry.shutdown_otel_metrics") as mock_otel_shutdown, - ): + with patch( + "src.telemetry.events.shutdown_telemetry_events", new_callable=AsyncMock + ) as mock_ce_shutdown: await shutdown_telemetry() mock_ce_shutdown.assert_called_once() - mock_otel_shutdown.assert_called_once() diff --git a/tests/test_schema_validations.py b/tests/test_schema_validations.py index 345fde23..281522a9 100644 --- a/tests/test_schema_validations.py +++ b/tests/test_schema_validations.py @@ -1,3 +1,5 @@ +from typing import Any + import pytest from pydantic import ValidationError @@ -6,6 +8,7 @@ from src.schemas import ( DocumentMetadata, MessageCreate, PeerCreate, + ResolvedConfiguration, SessionCreate, WorkspaceCreate, ) @@ -140,3 +143,63 @@ class TestDocumentValidations: ) error_dict = exc_info.value.errors()[0] assert error_dict["type"] == "string_too_long" + + +class TestResolvedConfigurationMigration: + """Test backward compatibility for queue items created before v3.0.0. + + In v3.0.0, the 'deriver' field was renamed to 'reasoning'. Old queue items + may still have the 'deriver' field and need to be migrated at validation time. + """ + + def _make_config(self, **overrides: dict[str, Any]): + """Helper to create a valid config dict with overrides.""" + base = { + "reasoning": {"enabled": True}, + "peer_card": {"use": True, "create": True}, + "summary": { + "enabled": True, + "messages_per_short_summary": 20, + "messages_per_long_summary": 60, + }, + "dream": {"enabled": False}, + } + base.update(overrides) + return base + + def test_old_queue_item_with_deriver_field(self): + """Old queue items with 'deriver' should be migrated to 'reasoning'.""" + old_payload = self._make_config() + del old_payload["reasoning"] + old_payload["deriver"] = {"enabled": True} + + config = ResolvedConfiguration.model_validate(old_payload) + + assert config.reasoning.enabled is True + + def test_new_queue_item_with_reasoning_field(self): + """New queue items with 'reasoning' should work normally.""" + new_payload = self._make_config(reasoning={"enabled": False}) + + config = ResolvedConfiguration.model_validate(new_payload) + + assert config.reasoning.enabled is False + + def test_migration_does_not_override_reasoning(self): + """If both 'deriver' and 'reasoning' exist, 'reasoning' takes precedence.""" + payload = self._make_config(reasoning={"enabled": False}) + payload["deriver"] = {"enabled": True} + + config = ResolvedConfiguration.model_validate(payload) + + assert config.reasoning.enabled is False + + def test_missing_reasoning_and_deriver_fails(self): + """Payload missing both 'reasoning' and 'deriver' should fail validation.""" + payload = self._make_config() + del payload["reasoning"] + + with pytest.raises(ValidationError) as exc_info: + ResolvedConfiguration.model_validate(payload) + + assert any(e["loc"] == ("reasoning",) for e in exc_info.value.errors()) diff --git a/uv.lock b/uv.lock index f3250a51..100461d3 100644 --- a/uv.lock +++ b/uv.lock @@ -1025,7 +1025,7 @@ wheels = [ [[package]] name = "honcho" -version = "3.0.0" +version = "3.0.1" source = { virtual = "." } dependencies = [ { name = "alembic" }, @@ -1042,10 +1042,9 @@ dependencies = [ { name = "langfuse" }, { name = "nanoid" }, { name = "openai" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, { name = "pdfplumber" }, { name = "pgvector" }, + { name = "prometheus-client" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -1098,10 +1097,9 @@ requires-dist = [ { name = "langfuse", specifier = ">=3.3.2" }, { name = "nanoid", specifier = ">=2.0.0" }, { name = "openai", specifier = ">=1.99.7" }, - { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.36.0" }, - { name = "opentelemetry-sdk", specifier = ">=1.36.0" }, { name = "pdfplumber", specifier = ">=0.11.7" }, { name = "pgvector", specifier = ">=0.2.5" }, + { name = "prometheus-client", specifier = ">=0.21.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.1.19" }, { name = "pydantic", specifier = ">=2.11.7" }, { name = "pydantic-settings", specifier = ">=2.10.1" }, @@ -2127,6 +2125,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/74/a88bf1b1efeae488a0c0b7bdf71429c313722d1fc0f377537fbe554e6180/pre_commit-4.2.0-py2.py3-none-any.whl", hash = "sha256:a009ca7205f1eb497d10b845e52c838a98b6cdd2102a6c8e4540e94ee75c58bd", size = 220707, upload-time = "2025-03-18T21:35:19.343Z" }, ] +[[package]] +name = "prometheus-client" +version = "0.24.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/58/a794d23feb6b00fc0c72787d7e87d872a6730dd9ed7c7b3e954637d8f280/prometheus_client-0.24.1.tar.gz", hash = "sha256:7e0ced7fbbd40f7b84962d5d2ab6f17ef88a72504dcf7c0b40737b43b2a461f9", size = 85616, upload-time = "2026-01-14T15:26:26.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/c3/24a2f845e3917201628ecaba4f18bab4d18a337834c1df2a159ee9d22a42/prometheus_client-0.24.1-py3-none-any.whl", hash = "sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055", size = 64057, upload-time = "2026-01-14T15:26:24.42Z" }, +] + [[package]] name = "propcache" version = "0.4.1"