RomanNum3ral
/
honcho
mirror of https://github.com/plastic-labs/honcho.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Telemetry Overhaul (#333) 

* chore: Refactor a top-level telemetry folder

* fix: Add OTEL push based metrics

* chore: cleanup otel to match prometheus implementation

* fix: Remove prometheus

* feat: Scaffold CloudEvent Emitter

* chore: remove dead code

* feat: PoC Cloud Events

* chore: cleanup test

* fix: Event naming conventions and OTEL Settings

* fix: Revamped Dream Event Structure

* fix: Instrument Event Code

* fix: Add Tests for CloudEvent Telemetry

* fix: Code Rabbit Nits

* fix: Dedupe Event IDs
This commit is contained in:
Vineeth Voruganti 2026-01-21 16:49:14 -05:00 committed by GitHub
parent 8c8a8c4103
commit 6b3ecef601
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
63 changed files with 7074 additions and 602 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
.env.template
View File

@ -23,6 +23,7 @@ LOG_LEVEL=INFO
# COLLECT_METRICS_LOCAL=false
# LOCAL_METRICS_FILE=metrics.jsonl
# REASONING_TRACES_FILE=traces.jsonl # Path to JSONL file for reasoning traces
# NAMESPACE="honcho"
@ -97,12 +98,14 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
# DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000 # 30 days
# DERIVER_PROVIDER=google
# DERIVER_MODEL=gemini-2.5-flash-lite
# DERIVER_TEMPERATURE=
# DERIVER_DEDUPLICATE=true
# DERIVER_MAX_OUTPUT_TOKENS=4096
# DERIVER_THINKING_BUDGET_TOKENS=1024
# DERIVER_LOG_OBSERVATIONS=false
# DERIVER_MAX_INPUT_TOKENS=23000
# DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100
# DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=4096
# DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=1024
# DERIVER_BACKUP_PROVIDER=
# DERIVER_BACKUP_MODEL=
@ -127,19 +130,22 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
# DIALECTIC_LEVELS__minimal__PROVIDER=google
# DIALECTIC_LEVELS__minimal__MODEL=gemini-2.5-flash-lite
# DIALECTIC_LEVELS__minimal__THINKING_BUDGET_TOKENS=0
# DIALECTIC_LEVELS__minimal__MAX_TOOL_ITERATIONS=2
# DIALECTIC_LEVELS__minimal__MAX_TOOL_ITERATIONS=5
# DIALECTIC_LEVELS__minimal__TOOL_CHOICE=any
# Low level
# DIALECTIC_LEVELS__low__PROVIDER=google
# DIALECTIC_LEVELS__low__MODEL=gemini-3-flash
# DIALECTIC_LEVELS__low__MODEL=gemini-3-flash-preview
# DIALECTIC_LEVELS__low__THINKING_BUDGET_TOKENS=0
# DIALECTIC_LEVELS__low__MAX_TOOL_ITERATIONS=5
# DIALECTIC_LEVELS__low__TOOL_CHOICE=any
# Medium level
# DIALECTIC_LEVELS__medium__PROVIDER=anthropic
# DIALECTIC_LEVELS__medium__MODEL=claude-haiku-4-5
# DIALECTIC_LEVELS__medium__THINKING_BUDGET_TOKENS=512
# DIALECTIC_LEVELS__medium__THINKING_BUDGET_TOKENS=1024
# DIALECTIC_LEVELS__medium__MAX_TOOL_ITERATIONS=4
# DIALECTIC_LEVELS__medium__TOOL_CHOICE=
# High level
# DIALECTIC_LEVELS__high__PROVIDER=anthropic
@ -150,7 +156,7 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
# Max level
# DIALECTIC_LEVELS__max__PROVIDER=anthropic
# DIALECTIC_LEVELS__max__MODEL=claude-opus-4-5
# DIALECTIC_LEVELS__max__THINKING_BUDGET_TOKENS=512
# DIALECTIC_LEVELS__max__THINKING_BUDGET_TOKENS=2048
# DIALECTIC_LEVELS__max__MAX_TOOL_ITERATIONS=10
# Optional backup per level (must set both or neither):
# DIALECTIC_LEVELS__max__BACKUP_PROVIDER=google
@ -179,16 +185,20 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
# DREAM_MIN_HOURS_BETWEEN_DREAMS=8
# DREAM_ENABLED_TYPES=["omni"]
# DREAM_PROVIDER=anthropic
# DREAM_MODEL=claude-haiku-4-5
# DREAM_MAX_OUTPUT_TOKENS=4000
# DREAM_THINKING_BUDGET_TOKENS=2048
# DREAM_MAX_TOOL_ITERATIONS=8
# DREAM_HISTORY_TOKEN_LIMIT=8192
# DREAM_MODEL=claude-sonnet-4-20250514
# DREAM_MAX_OUTPUT_TOKENS=16384
# DREAM_THINKING_BUDGET_TOKENS=8192
# DREAM_MAX_TOOL_ITERATIONS=20
# DREAM_HISTORY_TOKEN_LIMIT=16384
# DREAM_BACKUP_PROVIDER=
# DREAM_BACKUP_MODEL=
# Specialist models (use same provider as main model)
# DREAM_DEDUCTION_MODEL=claude-haiku-4-5
# DREAM_INDUCTION_MODEL=claude-haiku-4-5
# Dream Surprisal Settings (Tree-based observation sampling for targeted reasoning)
# DREAM_SURPRISAL__ENABLED=true
# DREAM_SURPRISAL__ENABLED=false
# DREAM_SURPRISAL__TREE_TYPE=kdtree # Options: kdtree, balltree, rptree, covertree, lsh, graph, prototype
# DREAM_SURPRISAL__TREE_K=5 # Number of neighbors for kNN-based trees
# DREAM_SURPRISAL__SAMPLING_STRATEGY=recent # Options: recent, random, all
@ -215,16 +225,33 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here
# SENTRY_PROFILES_SAMPLE_RATE=0.1
# =============================================================================
# Metrics (Optional)
# OpenTelemetry Settings (Push-based metrics via OTLP)
# =============================================================================
# METRICS_ENABLED=false
# METRICS_NAMESPACE=honcho # Inherits from NAMESPACE if not set
# OTEL_ENABLED=false
# OTEL_ENDPOINT=https://mimir.example.com/otlp/v1/metrics
# OTEL_HEADERS={"X-Scope-OrgID": "honcho"} # JSON string for auth headers
# OTEL_EXPORT_INTERVAL_MILLIS=60000
# OTEL_SERVICE_NAME=honcho
# OTEL_SERVICE_NAMESPACE=honcho # Inherits from NAMESPACE if not set
# =============================================================================
# CloudEvents Telemetry Settings (Analytics events)
# =============================================================================
# TELEMETRY_ENABLED=false
# TELEMETRY_ENDPOINT=https://telemetry.honcho.dev/v1/events
# TELEMETRY_HEADERS={"Authorization": "Bearer your-token"} # JSON string for auth headers
# TELEMETRY_BATCH_SIZE=100
# TELEMETRY_FLUSH_INTERVAL_SECONDS=1.0
# TELEMETRY_FLUSH_THRESHOLD=50
# TELEMETRY_MAX_RETRIES=3
# TELEMETRY_MAX_BUFFER_SIZE=10000
# TELEMETRY_NAMESPACE=honcho # Inherits from NAMESPACE if not set
# =============================================================================
# Cache
# =============================================================================
# CACHE_ENABLED=false
# CACHE_URL="redis://localhost:6379/0?suppress=false"
# CACHE_URL="redis://localhost:6379/0?suppress=true"
# CACHE_NAMESPACE="honcho" # Inherits from NAMESPACE if not set
# CACHE_DEFAULT_TTL_SECONDS=300
# CACHE_DEFAULT_LOCK_TTL_SECONDS=5
@ -239,9 +266,10 @@ VECTOR_STORE_TYPE=pgvector
VECTOR_STORE_MIGRATED=false
# Global namespace prefix for all vector namespaces
# Namespaces follow the pattern:
# - Documents: {NAMESPACE}.{workspace}.{observer}.{observed}
# - Messages: {NAMESPACE}.{workspace}.messages
# Namespaces follow the pattern: {NAMESPACE}.{type}.{hash}
# where hash is a base64url-encoded SHA-256 of the workspace/peer names
# - Documents: {NAMESPACE}.doc.{hash(workspace, observer, observed)}
# - Messages: {NAMESPACE}.msg.{hash(workspace)}
# VECTOR_STORE_NAMESPACE=honcho # Inherits from NAMESPACE if not set
# Embedding dimensions (default: 1536 for OpenAI text-embedding-3-small)
@ -253,4 +281,6 @@ VECTOR_STORE_MIGRATED=false
# LanceDB-specific settings (local embedded mode)
# VECTOR_STORE_LANCEDB_PATH=./lancedb_data
#VECTOR_STORE_RECONCILIATION_INTERVAL_SECONDS
# Reconciliation interval for background sync (default: 5 minutes)
# VECTOR_STORE_RECONCILIATION_INTERVAL_SECONDS=300

3
.gitignore vendored
View File

@ -190,3 +190,6 @@ CRUSH.md
metrics.jsonl
AGENTS.md
lancedb_data/
mimir-data/
grafana-data/

15
README.md
View File

@ -411,18 +411,20 @@ cp config.toml.example config.toml
Then modify the values as needed. The TOML file is organized into sections:
- `[app]` - Application-level settings (log level, session limits, embedding settings, metrics)
- `[app]` - Application-level settings (log level, session limits, embedding settings, namespace)
- `[db]` - Database connection and pool settings
- `[auth]` - Authentication configuration
- `[cache]` - Redis cache configuration
- `[llm]` - LLM provider API keys and general settings
- `[dialectic]` - Dialectic API configuration (provider, model, search settings)
- `[deriver]` - Background worker settings and representation configuration
- `[peer_card]` - Peer card generation settings
- `[dialectic]` - Dialectic API configuration with per-level reasoning settings
- `[summary]` - Session summarization settings
- `[dream]` - Dream processing configuration
- `[dream]` - Dream processing configuration (including specialist models and surprisal settings)
- `[webhook]` - Webhook configuration
- `[metrics]` - Metrics collection settings
- `[otel]` - OpenTelemetry push-based metrics via OTLP
- `[telemetry]` - CloudEvents telemetry for analytics
- `[vector_store]` - Vector store configuration (pgvector, turbopuffer, or lancedb)
- `[sentry]` - Error tracking and monitoring settings
### Using Environment Variables
@ -436,9 +438,12 @@ Examples:
- `DB_CONNECTION_URI` - Database connection string
- `AUTH_JWT_SECRET` - JWT secret key
- `DIALECTIC_MODEL` - Dialectic API model
- `DIALECTIC_LEVELS__low__MODEL` - Model for low reasoning level
- `DERIVER_PROVIDER` - Provider for background deriver
- `SUMMARY_PROVIDER` - Summary generation provider
- `LOG_LEVEL` - Application log level
- `OTEL_ENABLED` - Enable OpenTelemetry metrics
- `TELEMETRY_ENABLED` - Enable CloudEvents telemetry
### Configuration Priority

62
config.toml.example
View File

@ -17,7 +17,8 @@ MAX_EMBEDDING_TOKENS_PER_REQUEST = 300000
# LANGFUSE_PUBLIC_KEY = "your-public-key-here"
# COLLECT_METRICS_LOCAL = false
# LOCAL_METRICS_FILE = "metrics.jsonl"
NAMESPACE="honcho"
# REASONING_TRACES_FILE = "traces.jsonl" # Path to JSONL file for reasoning traces
NAMESPACE = "honcho"
# Database settings
[db]
@ -75,14 +76,16 @@ STALE_SESSION_TIMEOUT_MINUTES = 5
# QUEUE_ERROR_RETENTION_SECONDS = 2592000 # 30 days
PROVIDER = "google"
MODEL = "gemini-2.5-flash-lite"
# TEMPERATURE = 0.0
# BACKUP_PROVIDER = "anthropic"
# BACKUP_MODEL = "claude-haiku-4-5"
DEDUPLICATE = true
MAX_OUTPUT_TOKENS = 4096
THINKING_BUDGET_TOKENS = 1024
LOG_OBSERVATIONS = false
MAX_INPUT_TOKENS = 23000
WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100
REPRESENTATION_BATCH_MAX_TOKENS = 4096
REPRESENTATION_BATCH_MAX_TOKENS = 1024
# Peer card settings
[peer_card]
@ -100,19 +103,22 @@ SESSION_HISTORY_MAX_TOKENS = 16384
PROVIDER = "google"
MODEL = "gemini-2.5-flash-lite"
THINKING_BUDGET_TOKENS = 0
MAX_TOOL_ITERATIONS = 2
MAX_TOOL_ITERATIONS = 5
TOOL_CHOICE = "any"
[dialectic.levels.low]
PROVIDER = "google"
MODEL = "gemini-3-flash"
MODEL = "gemini-3-flash-preview"
THINKING_BUDGET_TOKENS = 0
MAX_TOOL_ITERATIONS = 5
TOOL_CHOICE = "any"
[dialectic.levels.medium]
PROVIDER = "anthropic"
MODEL = "claude-haiku-4-5"
THINKING_BUDGET_TOKENS = 512
THINKING_BUDGET_TOKENS = 1024
MAX_TOOL_ITERATIONS = 4
# TOOL_CHOICE = "any" # Optional: None/auto lets model decide
[dialectic.levels.high]
PROVIDER = "anthropic"
@ -123,7 +129,7 @@ MAX_TOOL_ITERATIONS = 4
[dialectic.levels.max]
PROVIDER = "anthropic"
MODEL = "claude-opus-4-5"
THINKING_BUDGET_TOKENS = 512
THINKING_BUDGET_TOKENS = 2048
MAX_TOOL_ITERATIONS = 10
# Backup provider example (optional, must set both or neither):
# BACKUP_PROVIDER = "google"
@ -150,25 +156,59 @@ IDLE_TIMEOUT_MINUTES = 60
MIN_HOURS_BETWEEN_DREAMS = 8
ENABLED_TYPES = ["omni"]
PROVIDER = "anthropic"
MODEL = "claude-haiku-4-5"
MAX_OUTPUT_TOKENS = 4000
MODEL = "claude-sonnet-4-20250514"
MAX_OUTPUT_TOKENS = 16384
THINKING_BUDGET_TOKENS = 8192
MAX_TOOL_ITERATIONS = 20
HISTORY_TOKEN_LIMIT = 16384
# BACKUP_PROVIDER = "google"
# BACKUP_MODEL = "gemini-2.5-flash"
# Specialist models (use same provider as main model)
DEDUCTION_MODEL = "claude-haiku-4-5"
INDUCTION_MODEL = "claude-haiku-4-5"
# Surprisal-based sampling subsystem
[dream.surprisal]
ENABLED = false
TREE_TYPE = "kdtree" # Options: kdtree, balltree, rptree, covertree, lsh, graph, prototype
TREE_K = 5 # k for kNN-based trees
SAMPLING_STRATEGY = "recent" # Options: recent, random, all
SAMPLE_SIZE = 200
TOP_PERCENT_SURPRISAL = 0.10 # Top 10% of observations
MIN_HIGH_SURPRISAL_FOR_REPLACE = 10
INCLUDE_LEVELS = ["explicit", "deductive"]
# Webhook settings
[webhook]
SECRET = ""
MAX_WORKSPACE_LIMIT = 10
# Metrics settings
[metrics]
# OpenTelemetry settings (push-based metrics via OTLP)
[otel]
ENABLED = false
# ENDPOINT = "https://mimir.example.com/otlp/v1/metrics"
# HEADERS = '{"X-Scope-OrgID": "honcho"}' # JSON string for auth headers
EXPORT_INTERVAL_MILLIS = 60000
SERVICE_NAME = "honcho"
# SERVICE_NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set
# CloudEvents telemetry settings (analytics events)
[telemetry]
ENABLED = false
# ENDPOINT = "https://telemetry.honcho.dev/v1/events"
# HEADERS = '{"Authorization": "Bearer your-token"}' # JSON string for auth headers
BATCH_SIZE = 100
FLUSH_INTERVAL_SECONDS = 1.0
FLUSH_THRESHOLD = 50
MAX_RETRIES = 3
MAX_BUFFER_SIZE = 10000
# NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set
# Cache settings
[cache]
ENABLED = false
URL = "redis://localhost:6379/0?suppress=false"
URL = "redis://localhost:6379/0?suppress=true"
# NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set
DEFAULT_TTL_SECONDS = 300
DEFAULT_LOCK_TTL_SECONDS = 5

91
docker-compose.yml.example
View File

@ -11,6 +11,7 @@ services:
- 8000:8000
volumes:
- .:/app
- venv:/app/.venv
env_file:
- .env
deriver:
@ -23,6 +24,7 @@ services:
condition: service_healthy
volumes:
- .:/app
- venv:/app/.venv
env_file:
- .env
database:
@ -38,8 +40,8 @@ services:
- POSTGRES_HOST_AUTH_METHOD=trust
- PGDATA=/var/lib/postgresql/data/pgdata
volumes:
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
- ./data:/var/lib/postgresql/data/
- ./database/init.sql:/docker-entrypoint-initdb.d/init.sql
- pgdata:/var/lib/postgresql/data/
healthcheck:
test: ["CMD-SHELL", "pg_isready -U testuser -d honcho"]
interval: 5s
@ -57,3 +59,88 @@ services:
interval: 5s
timeout: 5s
retries: 5
mimir:
image: grafana/mimir:2.14.0
command:
- -config.file=/etc/mimir/mimir.yaml
ports:
- 9009:9009
volumes:
- ./mimir-data:/data
configs:
- source: mimir_config
target: /etc/mimir/mimir.yaml
grafana:
image: grafana/grafana:11.4.0
ports:
- 3000:3000
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
volumes:
- ./grafana-data:/var/lib/grafana
depends_on:
- mimir
volumes:
pgdata:
venv:
configs:
mimir_config:
content: |
multitenancy_enabled: false
server:
http_listen_port: 9009
log_level: warn
common:
storage:
backend: filesystem
filesystem:
dir: /data
blocks_storage:
backend: filesystem
filesystem:
dir: /data/blocks
bucket_store:
sync_dir: /data/tsdb-sync
tsdb:
dir: /data/tsdb
compactor:
data_dir: /data/compactor
sharding_ring:
kvstore:
store: memberlist
distributor:
ring:
instance_addr: 127.0.0.1
kvstore:
store: memberlist
ingester:
ring:
instance_addr: 127.0.0.1
kvstore:
store: memberlist
replication_factor: 1
ruler_storage:
backend: filesystem
filesystem:
dir: /data/rules
alertmanager_storage:
backend: filesystem
filesystem:
dir: /data/alertmanager
store_gateway:
sharding_ring:
replication_factor: 1

6
pyproject.toml
View File

@ -31,13 +31,15 @@ dependencies = [
"google-genai>=1.32.0",
"pdfplumber>=0.11.7",
"typing-extensions>=4.11.0",
"prometheus-client>=0.20.0",
"json-repair>=0.49.0",
"turbopuffer>=1.8.1",
"lancedb>=0.25.3",
"redis>=7.0.0,<8.0.0",
"cashews[redis]==7.4.4",
"scikit-learn>=1.6.0",
"opentelemetry-sdk>=1.36.0",
"opentelemetry-exporter-otlp-proto-http>=1.36.0",
"cloudevents>=1.12.0",
]
[dependency-groups]
dev = [
@ -114,7 +116,7 @@ reportUnusedCallResult = false
reportCallInDefaultInitializer = false
reportAny = false
reportExplicitAny = false
allowedUntypedLibraries = ["langfuse", "lancedb", "pyarrow"]
allowedUntypedLibraries = ["langfuse", "lancedb", "pyarrow", "opentelemetry"]
reportImplicitOverride = false
reportImportCycles = false

76
src/config.py
View File

@ -68,6 +68,8 @@ class TomlConfigSettingsSource(PydanticBaseSettingsSource):
"WEBHOOK": "webhook",
"DREAM": "dream",
"VECTOR_STORE": "vector_store",
"OTEL": "otel",
"TELEMETRY": "telemetry",
"": "app", # For AppSettings with no prefix
}
@ -439,10 +441,65 @@ class WebhookSettings(HonchoSettings):
MAX_WORKSPACE_LIMIT: int = 10
class MetricsSettings(HonchoSettings):
model_config = SettingsConfigDict(env_prefix="METRICS_", extra="ignore") # pyright: ignore
class OpenTelemetrySettings(HonchoSettings):
"""OpenTelemetry settings for push-based metrics via OTLP.
These settings configure the OTel SDK to push metrics via OTLP HTTP
to any compatible backend (Mimir, Grafana Cloud, etc.).
"""
model_config = SettingsConfigDict(env_prefix="OTEL_", extra="ignore") # pyright: ignore
# Master toggle for OTel metrics
ENABLED: bool = False
# OTLP HTTP endpoint for metrics (e.g., "https://mimir.example.com/otlp/v1/metrics")
# For Mimir, the endpoint is typically: <mimir-url>/otlp/v1/metrics
# For Grafana Cloud: https://otlp-gateway-<region>.grafana.net/otlp/v1/metrics
ENDPOINT: str | None = None
HEADERS: dict[str, str] | None = None
# Export interval in milliseconds (default: 60 seconds)
EXPORT_INTERVAL_MILLIS: int = 60000
# Service name for resource attributes (identifies what this service is)
SERVICE_NAME: str = "honcho"
# Service namespace for resource attributes (defaults to top-level NAMESPACE if not set)
SERVICE_NAMESPACE: str | None = None
class TelemetrySettings(HonchoSettings):
"""CloudEvents telemetry settings for analytics.
These settings configure the CloudEvents emitter for pushing
structured events to an analytics backend.
"""
model_config = SettingsConfigDict(env_prefix="TELEMETRY_", extra="ignore") # pyright: ignore
# Master toggle for CloudEvents emission
ENABLED: bool = False
# CloudEvents HTTP endpoint (e.g., "https://telemetry.honcho.dev/v1/events")
ENDPOINT: str | None = None
# Optional headers for authentication
HEADERS: dict[str, str] | None = None
# Batching configuration
BATCH_SIZE: Annotated[int, Field(default=100, gt=0, le=1000)] = 100
FLUSH_INTERVAL_SECONDS: Annotated[float, Field(default=1.0, gt=0.0, le=60.0)] = 1.0
FLUSH_THRESHOLD: Annotated[int, Field(default=50, gt=0, le=1000)] = 50
# Retry configuration
MAX_RETRIES: Annotated[int, Field(default=3, gt=0, le=10)] = 3
# Buffer configuration
MAX_BUFFER_SIZE: Annotated[int, Field(default=10000, gt=0, le=100000)] = 10000
# Namespace for instance identification (propagated from top-level NAMESPACE if not set)
NAMESPACE: str | None = None
@ -615,7 +672,8 @@ class AppSettings(HonchoSettings):
PEER_CARD: PeerCardSettings = Field(default_factory=PeerCardSettings)
SUMMARY: SummarySettings = Field(default_factory=SummarySettings)
WEBHOOK: WebhookSettings = Field(default_factory=WebhookSettings)
METRICS: MetricsSettings = Field(default_factory=MetricsSettings)
OTEL: OpenTelemetrySettings = Field(default_factory=OpenTelemetrySettings)
TELEMETRY: TelemetrySettings = Field(default_factory=TelemetrySettings)
CACHE: CacheSettings = Field(default_factory=CacheSettings)
DREAM: DreamSettings = Field(default_factory=DreamSettings)
VECTOR_STORE: VectorStoreSettings = Field(default_factory=VectorStoreSettings)
@ -631,16 +689,18 @@ class AppSettings(HonchoSettings):
def propagate_namespace(self) -> "AppSettings":
"""Propagate top-level NAMESPACE to nested settings if not explicitly set.
After this validator runs, CACHE.NAMESPACE, METRICS.NAMESPACE, and
VECTOR_STORE.NAMESPACE are guaranteed to exist. Explicitly provided
nested namespaces are preserved.
After this validator runs, CACHE.NAMESPACE,
VECTOR_STORE.NAMESPACE, TELEMETRY.NAMESPACE, and OTEL.SERVICE_NAMESPACE
are guaranteed to exist. Explicitly provided nested namespaces are preserved.
"""
if "NAMESPACE" not in self.CACHE.model_fields_set:
self.CACHE.NAMESPACE = self.NAMESPACE
if "NAMESPACE" not in self.METRICS.model_fields_set:
self.METRICS.NAMESPACE = self.NAMESPACE
if "NAMESPACE" not in self.VECTOR_STORE.model_fields_set:
self.VECTOR_STORE.NAMESPACE = self.NAMESPACE
if "NAMESPACE" not in self.TELEMETRY.model_fields_set:
self.TELEMETRY.NAMESPACE = self.NAMESPACE
if "SERVICE_NAMESPACE" not in self.OTEL.model_fields_set:
self.OTEL.SERVICE_NAMESPACE = self.NAMESPACE
return self

4
src/crud/__init__.py
View File

@ -38,6 +38,7 @@ from .representation import (
get_working_representation,
)
from .session import (
SessionDeletionResult,
clone_session,
delete_session,
get_or_create_session,
@ -57,6 +58,7 @@ from .webhook import (
list_webhook_endpoints,
)
from .workspace import (
WorkspaceDeletionResult,
delete_workspace,
get_all_workspaces,
get_or_create_workspace,
@ -107,6 +109,7 @@ __all__ = [
# Representation
"get_working_representation",
# Session
"SessionDeletionResult",
"get_sessions",
"get_or_create_session",
"get_session",
@ -124,6 +127,7 @@ __all__ = [
"delete_webhook_endpoint",
"list_webhook_endpoints",
# Workspace
"WorkspaceDeletionResult",
"delete_workspace",
"get_or_create_workspace",
"get_workspace",

2
src/crud/representation.py
View File

@ -14,8 +14,8 @@ from src.dependencies import tracked_db
from src.dreamer.dream_scheduler import check_and_schedule_dream
from src.embedding_client import embedding_client
from src.schemas import ResolvedConfiguration
from src.telemetry.logging import accumulate_metric
from src.utils.formatting import format_datetime_utc
from src.utils.logging import accumulate_metric
from src.utils.representation import (
DeductiveObservation,
ExplicitObservation,

23
src/crud/session.py
View File

@ -1,3 +1,4 @@
from dataclasses import dataclass
from logging import getLogger
from typing import Any
@ -26,6 +27,15 @@ from .workspace import get_or_create_workspace
logger = getLogger(__name__)
@dataclass
class SessionDeletionResult:
"""Result of a session deletion including cascade counts."""
messages_deleted: int
conclusions_deleted: int
SESSION_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}:session:{session_name}"
SESSION_LOCK_PREFIX = f"{get_cache_namespace()}:lock"
@ -375,7 +385,7 @@ async def _batch_delete_matching(
async def delete_session(
db: AsyncSession, workspace_name: str, session_name: str
) -> bool:
) -> SessionDeletionResult:
"""
Delete a session and all associated data (hard delete).
@ -394,7 +404,7 @@ async def delete_session(
session_name: Name of the session
Returns:
True if the session was deleted successfully
SessionDeletionResult containing cascade counts
Raises:
ResourceNotFoundException: If the session does not exist
@ -518,7 +528,7 @@ async def delete_session(
)
# Delete Document entries associated with this session in batches
await _batch_delete_matching(
conclusions_deleted = await _batch_delete_matching(
db,
models.Document,
[
@ -529,7 +539,7 @@ async def delete_session(
)
# Delete Message entries in batches
await _batch_delete_matching(
messages_deleted = await _batch_delete_matching(
db,
models.Message,
[
@ -556,7 +566,10 @@ async def delete_session(
await db.rollback()
raise e
return True
return SessionDeletionResult(
messages_deleted=messages_deleted,
conclusions_deleted=conclusions_deleted,
)
async def clone_session(

62
src/crud/workspace.py
View File

@ -1,3 +1,4 @@
from dataclasses import dataclass
from logging import getLogger
from typing import Any
@ -16,6 +17,18 @@ from src.vector_store import get_external_vector_store
logger = getLogger(__name__)
@dataclass
class WorkspaceDeletionResult:
"""Result of a workspace deletion including cascade counts."""
workspace: schemas.Workspace
peers_deleted: int
sessions_deleted: int
messages_deleted: int
conclusions_deleted: int
WORKSPACE_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}"
WORKSPACE_LOCK_PREFIX = f"{get_cache_namespace()}:lock"
@ -214,7 +227,9 @@ async def update_workspace(
return honcho_workspace
async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Workspace:
async def delete_workspace(
db: AsyncSession, workspace_name: str
) -> WorkspaceDeletionResult:
"""
Delete a workspace.
@ -223,7 +238,8 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor
workspace_name: Name of the workspace
Returns:
A snapshot of the deleted workspace as a Pydantic schema
WorkspaceDeletionResult containing a snapshot of the deleted workspace
and cascade counts for deleted resources
"""
logger.warning("Deleting workspace %s", workspace_name)
stmt = select(models.Workspace).where(models.Workspace.name == workspace_name)
@ -242,6 +258,40 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor
created_at=honcho_workspace.created_at,
)
# Count resources before deletion for telemetry
peers_count = int(
await db.scalar(
select(func.count(models.Peer.id)).where(
models.Peer.workspace_name == workspace_name
)
)
or 0
)
sessions_count = int(
await db.scalar(
select(func.count(models.Session.id)).where(
models.Session.workspace_name == workspace_name
)
)
or 0
)
messages_count = int(
await db.scalar(
select(func.count(models.Message.id)).where(
models.Message.workspace_name == workspace_name
)
)
or 0
)
conclusions_count = int(
await db.scalar(
select(func.count(models.Document.id)).where(
models.Document.workspace_name == workspace_name
)
)
or 0
)
# order is important here.
# delete all active queue sessions referencing this workspace first (using work_unit_key parsing)
# then queue items referencing this workspace
@ -383,4 +433,10 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor
await db.rollback()
raise
return workspace_snapshot
return WorkspaceDeletionResult(
workspace=workspace_snapshot,
peers_deleted=peers_count,
sessions_deleted=sessions_count,
messages_deleted=messages_count,
conclusions_deleted=conclusions_count,
)

27
src/deriver/__main__.py
View File

@ -3,9 +3,13 @@ import logging
import os
import uvloop
from prometheus_client import start_http_server
from src.config import settings
from src.telemetry import (
initialize_telemetry,
initialize_telemetry_async,
shutdown_telemetry,
)
from .queue_manager import main
@ -44,14 +48,15 @@ def setup_logging():
logging.getLogger("groq._base_client").setLevel(logging.WARNING)
def start_metrics_server() -> None:
"""Start Prometheus metrics HTTP server on port 9090."""
async def run_deriver():
"""Run the deriver with proper telemetry lifecycle management."""
# Initialize async telemetry (CloudEvents emitter)
await initialize_telemetry_async()
try:
# Uses default REGISTRY from prometheus_client
start_http_server(9090, addr="0.0.0.0") # nosec B104
print("[DERIVER] Starting Prometheus metrics server on port 9090")
except Exception as e:
print(f"[DERIVER] Failed to start Prometheus metrics server: {str(e)}")
await main()
finally:
# Shutdown telemetry (flush CloudEvents buffer, shutdown OTel metrics)
await shutdown_telemetry()
if __name__ == "__main__":
@ -62,10 +67,10 @@ if __name__ == "__main__":
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
try:
# Initialize sync telemetry (OTel metrics)
initialize_telemetry()
print("[DERIVER] Running main loop")
if settings.METRICS.ENABLED:
start_metrics_server()
asyncio.run(main())
asyncio.run(run_deriver())
except KeyboardInterrupt:
print("[DERIVER] Shutdown initiated via KeyboardInterrupt")
except Exception as e:

68
src/deriver/consumer.py
View File

@ -1,4 +1,5 @@
import logging
import time
import sentry_sdk
from pydantic import ValidationError
@ -7,14 +8,20 @@ from sqlalchemy import select
from src import crud, models
from src.dependencies import tracked_db
from src.deriver.deriver import process_representation_tasks_batch
from src.dreamer.dreamer import process_dream
from src.dreamer import process_dream
from src.exceptions import ResourceNotFoundException
from src.models import Message
from src.reconciler.queue_cleanup import cleanup_queue_items
from src.reconciler.sync_vectors import run_vector_reconciliation_cycle
from src.schemas import ReconcilerType, ResolvedConfiguration
from src.telemetry.events import (
CleanupStaleItemsCompletedEvent,
DeletionCompletedEvent,
SyncVectorsCompletedEvent,
emit,
)
from src.telemetry.logging import log_performance_metrics
from src.utils import summarizer
from src.utils.logging import log_performance_metrics
from src.utils.queue_payload import (
DeletionPayload,
DreamPayload,
@ -151,6 +158,7 @@ async def process_representation_batch(
*,
observer: str | None,
observed: str | None,
queue_items_count: int,
) -> None:
"""
Prepares and processes a batch of messages for representation tasks.
@ -173,6 +181,7 @@ async def process_representation_batch(
message_level_configuration,
observer=observer,
observed=observed,
queue_items_count=queue_items_count,
)
@ -195,6 +204,10 @@ async def process_deletion(
"""
deletion_type = payload.deletion_type
resource_id = payload.resource_id
success = True
error_message: str | None = None
messages_deleted = 0
conclusions_deleted = 0
logger.info(
"Processing deletion task: type=%s, resource_id=%s, workspace=%s",
@ -206,13 +219,18 @@ async def process_deletion(
async with tracked_db("process_deletion") as db:
if deletion_type == "session":
try:
await crud.delete_session(
result = await crud.delete_session(
db, workspace_name=workspace_name, session_name=resource_id
)
messages_deleted = result.messages_deleted
conclusions_deleted = result.conclusions_deleted
logger.info(
"Successfully deleted session %s in workspace %s",
"Successfully deleted session %s in workspace %s "
+ "(messages=%d, conclusions=%d)",
resource_id,
workspace_name,
messages_deleted,
conclusions_deleted,
)
except ResourceNotFoundException as e:
# Session not found - may have already been deleted, treat as success
@ -227,6 +245,7 @@ async def process_deletion(
await crud.delete_document_by_id(
db, workspace_name=workspace_name, document_id=resource_id
)
conclusions_deleted = 1 # Single observation deleted
logger.info(
"Successfully deleted observation %s in workspace %s",
resource_id,
@ -241,7 +260,22 @@ async def process_deletion(
)
else:
raise ValueError(f"Unsupported deletion type: {deletion_type}")
success = False
error_message = f"Unsupported deletion type: {deletion_type}"
raise ValueError(error_message)
# Emit telemetry event
emit(
DeletionCompletedEvent(
workspace_name=workspace_name,
deletion_type=deletion_type,
resource_id=resource_id,
success=success,
messages_deleted=messages_deleted,
conclusions_deleted=conclusions_deleted,
error_message=error_message,
)
)
async def process_reconciler(payload: ReconcilerPayload) -> None:
@ -257,11 +291,14 @@ async def process_reconciler(payload: ReconcilerPayload) -> None:
payload: The reconciler payload containing the reconciler type
"""
reconciler_type = payload.reconciler_type
start_time = time.perf_counter()
if reconciler_type == ReconcilerType.SYNC_VECTORS:
logger.debug("Processing sync_vectors task")
metrics = await run_vector_reconciliation_cycle()
duration_ms = (time.perf_counter() - start_time) * 1000
if (
metrics.total_synced > 0
or metrics.total_failed > 0
@ -276,9 +313,30 @@ async def process_reconciler(payload: ReconcilerPayload) -> None:
metrics.documents_cleaned,
)
# Emit telemetry event
emit(
SyncVectorsCompletedEvent(
documents_synced=metrics.documents_synced,
documents_failed=metrics.documents_failed,
documents_cleaned=metrics.documents_cleaned,
message_embeddings_synced=metrics.message_embeddings_synced,
message_embeddings_failed=metrics.message_embeddings_failed,
total_duration_ms=duration_ms,
)
)
elif reconciler_type == ReconcilerType.CLEANUP_QUEUE:
logger.debug("Processing cleanup_queue task")
await cleanup_queue_items()
duration_ms = (time.perf_counter() - start_time) * 1000
# Emit telemetry event for cleanup stale items
emit(
CleanupStaleItemsCompletedEvent(
total_duration_ms=duration_ms,
)
)
else:
raise ValueError(f"Unsupported reconciler type: {reconciler_type}")

49
src/deriver/deriver.py
View File

@ -1,19 +1,22 @@
import logging
import time
from src import crud, prometheus
from src import crud
from src.config import settings
from src.crud.representation import RepresentationManager
from src.dependencies import tracked_db
from src.models import Message
from src.schemas import ResolvedConfiguration
from src.telemetry import otel_metrics
from src.telemetry.events import RepresentationCompletedEvent, emit
from src.telemetry.logging import accumulate_metric, log_performance_metrics
from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes
from src.telemetry.sentry import with_sentry_transaction
from src.utils.clients import honcho_llm_call
from src.utils.config_helpers import get_configuration
from src.utils.formatting import format_new_turn_with_timestamp
from src.utils.logging import accumulate_metric, log_performance_metrics
from src.utils.representation import PromptRepresentation, Representation
from src.utils.tokens import estimate_tokens, track_deriver_input_tokens
from src.utils.tracing import with_sentry_transaction
from .prompts import estimate_minimal_deriver_prompt_tokens, minimal_deriver_prompt
@ -27,6 +30,7 @@ async def process_representation_tasks_batch(
*,
observer: str,
observed: str,
queue_items_count: int,
) -> None:
"""
Process messages with minimal overhead - single LLM call, no peer card.
@ -36,6 +40,7 @@ async def process_representation_tasks_batch(
message_level_configuration: Optional configuration override.
observer: The observer peer ID.
observed: The observed peer ID.
queue_items_count: Number of QueueItem records being processed in this batch.
"""
if not messages:
return
@ -88,10 +93,10 @@ async def process_representation_tasks_batch(
prompt_tokens = estimate_minimal_deriver_prompt_tokens()
messages_tokens = estimate_tokens(formatted_messages)
track_deriver_input_tokens(
task_type=prometheus.DeriverTaskTypes.INGESTION,
task_type=DeriverTaskTypes.INGESTION,
components={
prometheus.DeriverComponents.PROMPT: prompt_tokens,
prometheus.DeriverComponents.MESSAGES: messages_tokens,
DeriverComponents.PROMPT: prompt_tokens,
DeriverComponents.MESSAGES: messages_tokens,
},
)
@ -136,11 +141,14 @@ async def process_representation_tasks_batch(
"ms",
)
prometheus.DERIVER_TOKENS_PROCESSED.labels(
task_type=prometheus.DeriverTaskTypes.INGESTION.value,
token_type=prometheus.TokenTypes.OUTPUT.value,
component=prometheus.DeriverComponents.OUTPUT_TOTAL.value,
).inc(response.output_tokens)
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_deriver_tokens(
count=response.output_tokens,
task_type=DeriverTaskTypes.INGESTION.value,
token_type=TokenTypes.OUTPUT.value,
component=DeriverComponents.OUTPUT_TOTAL.value,
)
message_ids = [m.id for m in messages if m.peer_name == observed]
@ -209,3 +217,22 @@ async def process_representation_tasks_batch(
)
log_performance_metrics("minimal_deriver", f"{latest_message.id}_{observer}")
# Emit telemetry event
emit(
RepresentationCompletedEvent(
workspace_name=latest_message.workspace_name,
session_name=latest_message.session_name,
observed=observed,
queue_items_processed=queue_items_count,
earliest_message_id=earliest_message.public_id,
latest_message_id=latest_message.public_id,
message_count=len(messages),
explicit_conclusion_count=len(observations.explicit),
context_preparation_ms=context_prep_duration,
llm_call_ms=llm_duration,
total_duration_ms=overall_duration,
input_tokens=response.input_tokens,
output_tokens=response.output_tokens,
)
)

12
src/deriver/queue_manager.py
View File

@ -15,7 +15,7 @@ from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql import func
from src import models, prometheus
from src import models
from src.cache.client import close_cache, init_cache
from src.config import settings
from src.dependencies import tracked_db
@ -35,7 +35,8 @@ from src.reconciler import (
set_reconciler_scheduler,
)
from src.schemas import ResolvedConfiguration
from src.sentry import initialize_sentry
from src.telemetry import otel_metrics
from src.telemetry.sentry import initialize_sentry
from src.utils.work_unit import parse_work_unit_key
from src.webhooks.events import (
QueueEmptyEvent,
@ -439,6 +440,7 @@ class QueueManager:
message_level_configuration,
observer=work_unit.observer,
observed=work_unit.observed,
queue_items_count=len(items_to_process),
)
await self.mark_queue_items_as_processed(
items_to_process, work_unit_key
@ -774,11 +776,13 @@ class QueueManager:
if (
work_unit.task_type in ["representation", "summary"]
and work_unit.workspace_name is not None
and settings.OTEL.ENABLED
):
prometheus.DERIVER_QUEUE_ITEMS_PROCESSED.labels(
otel_metrics.record_deriver_queue_item(
count=len(items),
workspace_name=work_unit.workspace_name,
task_type=work_unit.task_type,
).inc(len(items))
)
async def mark_queue_item_as_errored(
self, item: QueueItem, work_unit_key: str, error: str

71
src/dialectic/core.py
View File

@ -13,9 +13,17 @@ from typing import Any, cast
from sqlalchemy.ext.asyncio import AsyncSession
from src import crud, prometheus
from src import crud
from src.config import ReasoningLevel, settings
from src.dialectic import prompts
from src.telemetry import otel_metrics
from src.telemetry.events import DialecticCompletedEvent, emit
from src.telemetry.logging import (
accumulate_metric,
log_performance_metrics,
log_token_usage_metrics,
)
from src.telemetry.otel.metrics import DialecticComponents, TokenTypes
from src.utils.agent_tools import DIALECTIC_TOOLS, create_tool_executor, search_memory
from src.utils.clients import (
HonchoLLMCallResponse,
@ -23,11 +31,6 @@ from src.utils.clients import (
honcho_llm_call,
)
from src.utils.formatting import format_new_turn_with_timestamp
from src.utils.logging import (
accumulate_metric,
log_performance_metrics,
log_token_usage_metrics,
)
logger = logging.getLogger(__name__)
@ -87,6 +90,10 @@ class DialecticAgent:
}
]
self._session_history_initialized: bool = False
self._prefetched_conclusion_count: int = 0
self._run_id: str = str(uuid.uuid4())[
:8
] # Always generate for event correlation
async def _initialize_session_history(self) -> None:
"""Fetch and inject session history into the system prompt if configured."""
@ -174,6 +181,11 @@ class DialecticAgent:
if explicit_repr.is_empty() and derived_repr.is_empty():
return None
# Count prefetched conclusions for telemetry
explicit_count = len(explicit_repr.explicit) + len(explicit_repr.deductive)
derived_count = len(derived_repr.explicit) + len(derived_repr.deductive)
self._prefetched_conclusion_count = explicit_count + derived_count
# Format as two separate sections
parts: list[str] = []
@ -256,6 +268,9 @@ class DialecticAgent:
observer=self.observer,
observed=self.observed,
history_token_limit=settings.DIALECTIC.HISTORY_TOKEN_LIMIT,
run_id=self._run_id,
agent_type="dialectic",
parent_category="dialectic",
)
return tool_executor, task_name, run_id, start_time
@ -272,6 +287,7 @@ class DialecticAgent:
cache_creation_input_tokens: int | None,
tool_calls_count: int,
thinking_content: str | None,
iterations: int,
) -> None:
"""
Log metrics common to both streaming and non-streaming responses.
@ -287,6 +303,7 @@ class DialecticAgent:
cache_creation_input_tokens: Cache creation tokens (if any)
tool_calls_count: Number of tool calls made
thinking_content: Thinking trace content (if any)
iterations: Number of iterations in the tool execution loop
"""
accumulate_metric(task_name, "tool_calls", tool_calls_count, "count")
@ -308,19 +325,39 @@ class DialecticAgent:
if not self.metric_key and run_id is not None:
log_performance_metrics("dialectic_chat", run_id)
# Track prometheus metrics - actual token counts from API
if prometheus.METRICS_ENABLED:
prometheus.DIALECTIC_TOKENS_PROCESSED.labels(
token_type=prometheus.TokenTypes.INPUT.value,
component=prometheus.DialecticComponents.TOTAL.value,
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_dialectic_tokens(
count=input_tokens,
token_type=TokenTypes.INPUT.value,
component=DialecticComponents.TOTAL.value,
reasoning_level=self.reasoning_level,
).inc(input_tokens)
)
otel_metrics.record_dialectic_tokens(
count=output_tokens,
token_type=TokenTypes.OUTPUT.value,
component=DialecticComponents.TOTAL.value,
reasoning_level=self.reasoning_level,
)
prometheus.DIALECTIC_TOKENS_PROCESSED.labels(
token_type=prometheus.TokenTypes.OUTPUT.value,
component=prometheus.DialecticComponents.TOTAL.value,
# Emit telemetry event
emit(
DialecticCompletedEvent(
run_id=self._run_id,
workspace_name=self.workspace_name,
peer_name=self.observed,
session_name=self.session_name,
reasoning_level=self.reasoning_level,
).inc(output_tokens)
total_iterations=iterations,
prefetched_conclusion_count=self._prefetched_conclusion_count,
tool_calls_count=tool_calls_count,
total_duration_ms=elapsed_ms,
input_tokens=input_tokens,
output_tokens=output_tokens,
cache_read_tokens=cache_read_input_tokens or 0,
cache_creation_tokens=cache_creation_input_tokens or 0,
)
)
async def answer(self, query: str) -> str:
"""
@ -368,6 +405,7 @@ class DialecticAgent:
cache_creation_input_tokens=response.cache_creation_input_tokens,
tool_calls_count=len(response.tool_calls_made),
thinking_content=response.thinking_content,
iterations=response.iterations,
)
return response.content
@ -429,4 +467,5 @@ class DialecticAgent:
cache_creation_input_tokens=response.cache_creation_input_tokens,
tool_calls_count=len(response.tool_calls_made),
thinking_content=response.thinking_content,
iterations=response.iterations,
)

2
src/dreamer/__init__.py
View File

@ -2,7 +2,7 @@ from .dream_scheduler import (
check_and_schedule_dream,
get_dream_scheduler,
)
from .dreamer import process_dream
from .orchestrator import process_dream
__all__ = [
"get_dream_scheduler",

51
src/dreamer/dreamer.py
View File

@ -1,51 +0,0 @@
import logging
import sentry_sdk
from src.config import settings
from src.dependencies import tracked_db
from src.dreamer.orchestrator import run_dream
from src.schemas import DreamType
from src.utils.queue_payload import DreamPayload
logger = logging.getLogger(__name__)
@sentry_sdk.trace
async def process_dream(
payload: DreamPayload,
workspace_name: str,
) -> None:
"""
Process a dream task by performing collection maintenance operations.
Args:
payload: The dream task payload containing workspace, peer, and dream type information
"""
logger.info(
f"""
(っ- ‸ - ς)ᶻ z 𐰁 ᶻ z 𐰁 ᶻ z 𐰁\n
DREAM: {payload.dream_type} documents for {workspace_name}/{payload.observer}/{payload.observed}\n
𐰁 z ᶻ 𐰁 z ᶻ 𐰁 z ᶻ(っ- ‸ - ς)"""
)
try:
match payload.dream_type:
case DreamType.OMNI:
async with tracked_db("dream_orchestrator") as db:
await run_dream(
db=db,
workspace_name=workspace_name,
observer=payload.observer,
observed=payload.observed,
session_name=payload.session_name,
)
except Exception as e:
logger.error(
f"Error processing dream task {payload.dream_type} for {payload.observer}/{payload.observed}: {str(e)}",
exc_info=True,
)
if settings.SENTRY.ENABLED:
sentry_sdk.capture_exception(e)
# Don't re-raise - we want to mark the dream task as processed even if it fails

162
src/dreamer/orchestrator.py
View File

@ -13,23 +13,53 @@ from __future__ import annotations
import logging
import time
import uuid
from dataclasses import dataclass
from typing import Any
import sentry_sdk
from sqlalchemy.ext.asyncio import AsyncSession
from src import crud
from src.config import settings
from src.dreamer.specialists import SPECIALISTS
from src.dependencies import tracked_db
from src.dreamer.specialists import SPECIALISTS, SpecialistResult
from src.dreamer.surprisal import SurprisalScore # type: ignore
from src.exceptions import SpecialistExecutionError, SurprisalError
from src.utils.config_helpers import get_configuration
from src.utils.logging import (
from src.schemas import DreamType
from src.telemetry.events import DreamRunEvent, emit
from src.telemetry.logging import (
accumulate_metric,
log_performance_metrics,
)
from src.utils.config_helpers import get_configuration
from src.utils.queue_payload import DreamPayload
logger = logging.getLogger(__name__)
@dataclass
class DreamResult:
"""Result of a dream cycle for telemetry reporting."""
# Run identification
run_id: str
specialists_run: list[str]
# Specialist outcomes
deduction_success: bool
induction_success: bool
# Surprisal sampling
surprisal_enabled: bool
surprisal_conclusion_count: int
# Aggregate metrics
total_iterations: int
total_duration_ms: float
input_tokens: int
output_tokens: int
# Predefined probing questions to guide the specialists
# These serve as semantic entry points for searching observations
PROBING_QUESTIONS: list[str] = [
@ -57,7 +87,7 @@ async def run_dream(
observer: str,
observed: str,
session_name: str,
) -> None:
) -> DreamResult | None:
"""
Run a full dream cycle with optional surprisal-based sampling.
@ -74,7 +104,7 @@ async def run_dream(
session_name: Session identifier
"""
if not settings.DREAM.ENABLED:
return
return None
run_id = str(uuid.uuid4())[:8]
task_name = f"dream_orchestrator_{run_id}"
@ -94,7 +124,14 @@ async def run_dream(
logger.info(
f"[{run_id}] Dreams disabled for {workspace_name}/{session_name}, skipping dream"
)
return
return None
# Track specialist outcomes
deduction_success = False
induction_success = False
surprisal_observation_count = 0
deduction_result: SpecialistResult | None = None
induction_result: SpecialistResult | None = None
# Phase 0: Surprisal-based sampling (if enabled)
probing_questions = PROBING_QUESTIONS # Default
@ -114,6 +151,7 @@ async def run_dream(
logger.info(
f"[{run_id}] Surprisal: Found {len(high_surprisal_obs)} high-surprisal observations"
)
surprisal_observation_count = len(high_surprisal_obs)
accumulate_metric(
task_name, "surprisal_observations", len(high_surprisal_obs), "count"
)
@ -166,9 +204,15 @@ async def run_dream(
session_name=session_name,
probing_questions=probing_questions,
configuration=configuration,
parent_run_id=run_id,
)
logger.info(f"[{run_id}] Deduction completed: {deduction_result[:200]}...")
accumulate_metric(task_name, "deduction_result", deduction_result, "blob")
logger.info(
f"[{run_id}] Deduction completed: {deduction_result.content[:200]}..."
)
accumulate_metric(
task_name, "deduction_result", deduction_result.content, "blob"
)
deduction_success = deduction_result.success
except SpecialistExecutionError as e:
logger.error(f"[{run_id}] Deduction specialist failed: {e}", exc_info=True)
accumulate_metric(task_name, "deduction_error", str(e), "blob")
@ -185,9 +229,15 @@ async def run_dream(
session_name=session_name,
probing_questions=probing_questions,
configuration=configuration,
parent_run_id=run_id,
)
logger.info(f"[{run_id}] Induction completed: {induction_result[:200]}...")
accumulate_metric(task_name, "induction_result", induction_result, "blob")
logger.info(
f"[{run_id}] Induction completed: {induction_result.content[:200]}..."
)
accumulate_metric(
task_name, "induction_result", induction_result.content, "blob"
)
induction_success = induction_result.success
except SpecialistExecutionError as e:
logger.error(f"[{run_id}] Induction specialist failed: {e}", exc_info=True)
accumulate_metric(task_name, "induction_error", str(e), "blob")
@ -199,6 +249,50 @@ async def run_dream(
logger.info(f"[{run_id}] Dream cycle completed in {duration_ms:.0f}ms")
log_performance_metrics("dream_orchestrator", run_id)
# Aggregate metrics from specialist results
total_iterations = (deduction_result.iterations if deduction_result else 0) + (
induction_result.iterations if induction_result else 0
)
total_input_tokens = (deduction_result.input_tokens if deduction_result else 0) + (
induction_result.input_tokens if induction_result else 0
)
total_output_tokens = (
deduction_result.output_tokens if deduction_result else 0
) + (induction_result.output_tokens if induction_result else 0)
# Emit DreamRunEvent with aggregated metrics
emit(
DreamRunEvent(
run_id=run_id,
workspace_name=workspace_name,
session_name=session_name,
observer=observer,
observed=observed,
specialists_run=["deduction", "induction"],
deduction_success=deduction_success,
induction_success=induction_success,
surprisal_enabled=settings.DREAM.SURPRISAL.ENABLED,
surprisal_conclusion_count=surprisal_observation_count,
total_iterations=total_iterations,
total_input_tokens=total_input_tokens,
total_output_tokens=total_output_tokens,
total_duration_ms=duration_ms,
)
)
return DreamResult(
run_id=run_id,
specialists_run=["deduction", "induction"],
deduction_success=deduction_success,
induction_success=induction_success,
surprisal_enabled=settings.DREAM.SURPRISAL.ENABLED,
surprisal_conclusion_count=surprisal_observation_count,
total_iterations=total_iterations,
total_duration_ms=duration_ms,
input_tokens=total_input_tokens,
output_tokens=total_output_tokens,
)
def _create_queries_from_surprisal(
high_surprisal_obs: list[SurprisalScore],
@ -222,3 +316,51 @@ def _create_queries_from_surprisal(
content = content[:200] + "..."
queries.append(content)
return queries[:10] # Limit to 10 queries
@sentry_sdk.trace
async def process_dream(
payload: DreamPayload,
workspace_name: str,
) -> None:
"""
Process a dream task by performing collection maintenance operations.
Args:
payload: The dream task payload containing workspace, peer, and dream type information
"""
logger.info(
f"""
(っ- ‸ - ς)ᶻ z 𐰁 ᶻ z 𐰁 ᶻ z 𐰁\n
DREAM: {payload.dream_type} documents for {workspace_name}/{payload.observer}/{payload.observed}\n
𐰁 z ᶻ 𐰁 z ᶻ 𐰁 z ᶻ(っ- ‸ - ς)"""
)
try:
match payload.dream_type:
case DreamType.OMNI:
async with tracked_db("dream_orchestrator") as db:
result = await run_dream(
db=db,
workspace_name=workspace_name,
observer=payload.observer,
observed=payload.observed,
session_name=payload.session_name,
)
# Log completion (telemetry event already emitted in run_dream)
if result is not None:
logger.info(
f"Dream completed: run_id={result.run_id}, "
+ f"iterations={result.total_iterations}, "
+ f"duration={result.total_duration_ms:.0f}ms"
)
except Exception as e:
logger.error(
f"Error processing dream task {payload.dream_type} for {payload.observer}/{payload.observed}: {str(e)}",
exc_info=True,
)
if settings.SENTRY.ENABLED:
sentry_sdk.capture_exception(e)
# Don't re-raise - we want to mark the dream task as processed even if it fails

90
src/dreamer/specialists.py
View File

@ -15,24 +15,42 @@ import time
import uuid
from abc import ABC, abstractmethod
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from src import prometheus
from src.config import settings
from src.schemas import ResolvedConfiguration
from src.telemetry import otel_metrics
from src.telemetry.events import DreamSpecialistEvent, emit
from src.telemetry.logging import accumulate_metric, log_performance_metrics
from src.telemetry.otel.metrics import TokenTypes
from src.utils.agent_tools import (
DEDUCTION_SPECIALIST_TOOLS,
INDUCTION_SPECIALIST_TOOLS,
create_tool_executor,
)
from src.utils.clients import HonchoLLMCallResponse, honcho_llm_call
from src.utils.logging import accumulate_metric, log_performance_metrics
logger = logging.getLogger(__name__)
@dataclass
class SpecialistResult:
"""Result of a specialist run for telemetry and aggregation."""
run_id: str
specialist_type: str
iterations: int
tool_calls_count: int
input_tokens: int
output_tokens: int
duration_ms: float
success: bool
content: str
# Tool names to exclude when peer card creation is disabled
PEER_CARD_TOOL_NAMES = {"update_peer_card", "get_peer_card"}
@ -81,7 +99,8 @@ class BaseSpecialist(ABC):
session_name: str,
probing_questions: list[str],
configuration: ResolvedConfiguration | None = None,
) -> str:
parent_run_id: str | None = None,
) -> SpecialistResult:
"""
Run the specialist agent.
@ -93,11 +112,12 @@ class BaseSpecialist(ABC):
session_name: Session identifier
probing_questions: Entry point questions to guide exploration
configuration: Resolved configuration for checking feature flags (optional)
parent_run_id: Optional run_id from orchestrator for correlation
Returns:
Summary of work done
SpecialistResult with metrics and content
"""
run_id = str(uuid.uuid4())[:8]
run_id = parent_run_id or str(uuid.uuid4())[:8]
task_name = f"dreamer_{self.name}_{run_id}"
start_time = time.perf_counter()
@ -115,7 +135,7 @@ class BaseSpecialist(ABC):
{"role": "user", "content": self.build_user_prompt(probing_questions)},
]
# Create tool executor
# Create tool executor with telemetry context
tool_executor: Callable[
[str, dict[str, Any]], Any
] = await create_tool_executor(
@ -127,12 +147,22 @@ class BaseSpecialist(ABC):
include_observation_ids=True,
history_token_limit=settings.DREAM.HISTORY_TOKEN_LIMIT,
configuration=configuration,
run_id=run_id,
agent_type=self.name,
parent_category="dream",
)
# Get model with potential override
model = self.get_model()
llm_settings = settings.DREAM.model_copy(update={"MODEL": model})
# Track iterations via callback
iteration_count = 0
def iteration_callback(data: Any) -> None:
nonlocal iteration_count
iteration_count = data.iteration
# Run the agent loop
response: HonchoLLMCallResponse[str] = await honcho_llm_call(
llm_settings=llm_settings,
@ -144,6 +174,7 @@ class BaseSpecialist(ABC):
max_tool_iterations=self.get_max_iterations(),
messages=messages,
track_name=f"Dreamer/{self.name}",
iteration_callback=iteration_callback,
)
# Log metrics
@ -155,16 +186,18 @@ class BaseSpecialist(ABC):
accumulate_metric(task_name, "input_tokens", response.input_tokens, "count")
accumulate_metric(task_name, "output_tokens", response.output_tokens, "count")
if prometheus.METRICS_ENABLED:
prometheus.DREAMER_TOKENS_PROCESSED.labels(
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_dreamer_tokens(
count=response.input_tokens,
specialist_name=self.name,
token_type=prometheus.TokenTypes.INPUT.value,
).inc(response.input_tokens)
prometheus.DREAMER_TOKENS_PROCESSED.labels(
token_type=TokenTypes.INPUT.value,
)
otel_metrics.record_dreamer_tokens(
count=response.output_tokens,
specialist_name=self.name,
token_type=prometheus.TokenTypes.OUTPUT.value,
).inc(response.output_tokens)
token_type=TokenTypes.OUTPUT.value,
)
logger.info(
f"{self.name}: Completed in {duration_ms:.0f}ms, "
@ -174,7 +207,34 @@ class BaseSpecialist(ABC):
log_performance_metrics(f"dreamer_{self.name}", run_id)
return response.content
# Emit telemetry event
emit(
DreamSpecialistEvent(
run_id=run_id,
specialist_type=self.name,
workspace_name=workspace_name,
observer=observer,
observed=observed,
iterations=iteration_count,
tool_calls_count=len(response.tool_calls_made),
input_tokens=response.input_tokens,
output_tokens=response.output_tokens,
duration_ms=duration_ms,
success=True,
)
)
return SpecialistResult(
run_id=run_id,
specialist_type=self.name,
iterations=iteration_count,
tool_calls_count=len(response.tool_calls_made),
input_tokens=response.input_tokens,
output_tokens=response.output_tokens,
duration_ms=duration_ms,
success=True,
content=response.content,
)
class DeductionSpecialist(BaseSpecialist):

47
src/main.py
View File

@ -15,7 +15,6 @@ from pydantic import ValidationError
from sentry_sdk.integrations.fastapi import FastApiIntegration
from sentry_sdk.integrations.starlette import StarletteIntegration
from src import prometheus
from src.cache.client import close_cache, init_cache
from src.config import settings
from src.db import engine, request_context
@ -30,8 +29,14 @@ from src.routers import (
workspaces,
)
from src.security import create_admin_jwt
from src.sentry import initialize_sentry
from src.utils.logging import get_route_template
from src.telemetry import (
initialize_telemetry,
initialize_telemetry_async,
otel_metrics,
shutdown_telemetry,
)
from src.telemetry.logging import get_route_template
from src.telemetry.sentry import initialize_sentry
if TYPE_CHECKING:
from sentry_sdk._types import Event, Hint
@ -115,6 +120,10 @@ if SENTRY_ENABLED:
@asynccontextmanager
async def lifespan(_: FastAPI):
# Initialize telemetry (OTel metrics + CloudEvents emitter)
initialize_telemetry()
await initialize_telemetry_async()
try:
await init_cache()
except Exception as e:
@ -131,6 +140,8 @@ async def lifespan(_: FastAPI):
await close_external_vector_store()
await close_cache()
await engine.dispose()
# Shutdown telemetry (flush CloudEvents buffer, shutdown OTel metrics)
await shutdown_telemetry()
app = FastAPI(
@ -180,23 +191,13 @@ app.include_router(conclusions.router, prefix="/v2")
app.include_router(keys.router, prefix="/v2")
app.include_router(webhooks.router, prefix="/v2")
app.add_api_route("/metrics", prometheus.metrics, methods=["GET"])
# Global exception handlers
@app.exception_handler(HonchoException)
async def honcho_exception_handler(request: Request, exc: HonchoException):
async def honcho_exception_handler(_request: Request, exc: HonchoException):
"""Handle all Honcho-specific exceptions."""
logger.error(f"{exc.__class__.__name__}: {exc.detail}", exc_info=exc)
if prometheus.METRICS_ENABLED and request.url.path != "/metrics":
template = get_route_template(request)
prometheus.API_REQUESTS.labels(
method=request.method,
endpoint=template,
status_code=str(exc.status_code),
).inc()
return JSONResponse(
status_code=exc.status_code,
content={"detail": exc.detail},
@ -204,18 +205,10 @@ async def honcho_exception_handler(request: Request, exc: HonchoException):
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
async def global_exception_handler(_request: Request, exc: Exception):
"""Handle all unhandled exceptions."""
logger.error(f"Unhandled exception: {str(exc)}", exc_info=True)
if prometheus.METRICS_ENABLED and request.url.path != "/metrics":
template = get_route_template(request)
prometheus.API_REQUESTS.labels(
method=request.method,
endpoint=template,
status_code="500",
).inc()
if SENTRY_ENABLED:
sentry_sdk.capture_exception(exc)
return JSONResponse(
@ -239,14 +232,14 @@ async def track_request(
try:
response = await call_next(request)
# Track Prometheus metrics if enabled
if prometheus.METRICS_ENABLED and request.url.path != "/metrics":
# Track metrics if enabled
if settings.OTEL.ENABLED:
template = get_route_template(request)
prometheus.API_REQUESTS.labels(
otel_metrics.record_api_request(
method=request.method,
endpoint=template,
status_code=str(response.status_code),
).inc()
)
return response
finally:

186
src/prometheus.py
View File

@ -1,186 +0,0 @@
"""
Prometheus metrics for Honcho.
This module defines all Prometheus metrics for all Honcho processes and exposes them via the /metrics endpoint.
"""
import logging
from enum import Enum
from typing import cast
from prometheus_client import (
CONTENT_TYPE_LATEST,
REGISTRY,
Counter,
disable_created_metrics,
generate_latest,
)
from starlette.responses import Response as StarletteResponse
from src.config import settings
METRICS_ENABLED = settings.METRICS.ENABLED
disable_created_metrics() # Disables _created metrics on counters, histograms, and summaries
logger = logging.getLogger(__name__)
class NamespacedCounter(Counter):
"""Counter that automatically includes namespace label"""
def labels(self, **kwargs: str) -> "NamespacedCounter":
"""Override labels to automatically appends namespace label"""
# METRICS.NAMESPACE is guaranteed to be non-None by AppSettings.propagate_namespace validator
kwargs["namespace"] = cast(str, settings.METRICS.NAMESPACE)
return super().labels(**kwargs)
# Tracks all requests to the Honcho API.
#
# Incremented in: src/main.py middleware for every request
# Labels:
# - method: HTTP method (GET, POST, PUT, DELETE, etc.)
# - endpoint: FastAPI route template (e.g., "/v2/workspaces/{workspace_id}/sessions/{session_id}/messages")
# - status_code: HTTP response status code (200, 404, 500, etc.)
API_REQUESTS = NamespacedCounter(
"api_requests_total",
"Total API requests",
[
"namespace",
"method",
"endpoint",
"status_code",
],
)
# Tracks the total number of honcho messages created.
#
# Incremented in: src/routers/messages.py when messages are successfully created
# Labels:
# - workspace_name: The workspace where messages were created
MESSAGES_CREATED = NamespacedCounter(
"messages_created_total",
"Total messages created",
[
"namespace",
"workspace_name",
],
)
# Tracks the total number of dialectic calls made.
#
# Incremented in: src/routers/peers.py when successful dialectic calls are made
# Labels:
# - workspace_name: The workspace where the dialectic call was made
# - reasoning_level: The reasoning level used for the call
DIALECTIC_CALLS = NamespacedCounter(
"dialectic_calls_total",
"Total dialectic calls",
[
"namespace",
"workspace_name",
"reasoning_level",
],
)
# Tracks the total number of queue items successfully processed by the deriver.
#
# Incremented in: src/deriver/queue_manager.py when queue items are processed
# Labels:
# - workspace_name: The workspace where items were processed
# - task_type: The type of task processed (e.g., "representation", "summary", "peer_card")
DERIVER_QUEUE_ITEMS_PROCESSED = NamespacedCounter(
"deriver_queue_items_processed_total",
"Total deriver queue items processed",
["namespace", "workspace_name", "task_type"],
)
class TokenTypes(Enum):
INPUT = "input"
OUTPUT = "output"
# Tracks the total number of input and output tokens processed by the deriver.
#
# Incremented in: src/deriver/deriver.py after the critical analysis call is made
# Labels:
# - task_type: The type of task that processed the tokens
# - token_type: The type of tokens ("input" or "output")
# - component: The component of the input
DERIVER_TOKENS_PROCESSED = NamespacedCounter(
"deriver_tokens_processed_total",
"Total tokens processed by the deriver",
[
"namespace",
"task_type",
"token_type",
"component",
],
)
class DeriverTaskTypes(Enum):
INGESTION = "ingestion"
SUMMARY = "summary"
class DeriverComponents(Enum):
PROMPT = "prompt" # used in ingestion and summary
MESSAGES = "messages" # used in ingestion and summary
PREVIOUS_SUMMARY = "previous_summary" # only used for summary
OUTPUT_TOTAL = "output_total"
# Tracks the total number of input and output tokens processed by the dialectic.
#
# Incremented in: src/dialectic/core.py after the dialectic call is made
# Labels:
# - token_type: The type of tokens
# - component: The component of the input
# - reasoning_level: The reasoning level used for the call
DIALECTIC_TOKENS_PROCESSED = NamespacedCounter(
"dialectic_tokens_processed_total",
"Total tokens processed by the dialectic",
[
"namespace",
"token_type",
"component",
"reasoning_level",
],
)
class DialecticComponents(Enum):
TOTAL = "total"
# Tracks the total number of input and output tokens processed by the dreamer.
#
# Incremented in: src/dreamer/specialists.py after the specialist LLM call is made
# Labels:
# - specialist_name: The name of the specialist ("deduction" or "induction")
# - token_type: The type of tokens ("input" or "output")
DREAMER_TOKENS_PROCESSED = NamespacedCounter(
"dreamer_tokens_processed_total",
"Total tokens processed by the dreamer",
[
"namespace",
"specialist_name",
"token_type",
],
)
async def metrics() -> StarletteResponse:
"""Prometheus metrics endpoint"""
if not settings.METRICS.ENABLED:
return StarletteResponse("Metrics are disabled", status_code=404)
try:
return StarletteResponse(
generate_latest(REGISTRY), media_type=CONTENT_TYPE_LATEST
)
except Exception as e:
logger.error(f"Failed to generate metrics: {e}", exc_info=True)
return StarletteResponse("Failed to generate metrics", status_code=500)

22
src/routers/messages.py
View File

@ -16,12 +16,13 @@ from fastapi_pagination.ext.sqlalchemy import apaginate
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm.attributes import flag_modified
from src import crud, prometheus, schemas
from src import crud, schemas
from src.config import settings
from src.dependencies import db
from src.deriver import enqueue
from src.exceptions import FileTooLargeError, ResourceNotFoundException
from src.security import require_auth
from src.telemetry import otel_metrics
from src.utils.files import process_file_uploads_for_messages
logger = logging.getLogger(__name__)
@ -99,9 +100,12 @@ async def create_messages_for_session(
session_name=session_id,
)
prometheus.MESSAGES_CREATED.labels(
workspace_name=workspace_id,
).inc(len(created_messages))
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_messages_created(
count=len(created_messages),
workspace_name=workspace_id,
)
# Enqueue for processing (existing logic)
payloads = [
@ -194,9 +198,13 @@ async def create_messages_with_file(
"Batch of %s messages created from file uploads and queued for processing",
len(created_messages),
)
prometheus.MESSAGES_CREATED.labels(
workspace_name=workspace_id,
).inc(len(created_messages))
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_messages_created(
count=len(created_messages),
workspace_name=workspace_id,
)
return created_messages

17
src/routers/peers.py
View File

@ -8,12 +8,13 @@ from fastapi_pagination import Page
from fastapi_pagination.ext.sqlalchemy import apaginate
from sqlalchemy.ext.asyncio import AsyncSession
from src import crud, prometheus, schemas
from src import crud, schemas
from src.config import settings
from src.dependencies import db, tracked_db
from src.dialectic.chat import agentic_chat, agentic_chat_stream
from src.exceptions import AuthenticationException, ResourceNotFoundException
from src.security import JWTParams, require_auth
from src.telemetry import otel_metrics
from src.utils.search import search
logger = logging.getLogger(__name__)
@ -183,11 +184,12 @@ async def chat(
yield f"data: {json.dumps({'delta': {'content': chunk}, 'done': False})}\n\n"
yield f"data: {json.dumps({'done': True})}\n\n"
if prometheus.METRICS_ENABLED:
prometheus.DIALECTIC_CALLS.labels(
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_dialectic_call(
workspace_name=workspace_id,
reasoning_level=options.reasoning_level,
).inc()
)
return StreamingResponse(
format_sse_stream(
@ -214,11 +216,12 @@ async def chat(
reasoning_level=options.reasoning_level,
)
if prometheus.METRICS_ENABLED:
prometheus.DIALECTIC_CALLS.labels(
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_dialectic_call(
workspace_name=workspace_id,
reasoning_level=options.reasoning_level,
).inc()
)
return schemas.DialecticResponse(content=str(response))

17
src/routers/workspaces.py
View File

@ -12,6 +12,7 @@ from src.dependencies import db
from src.deriver.enqueue import enqueue_dream
from src.exceptions import AuthenticationException
from src.security import JWTParams, require_auth
from src.telemetry.events import DeletionCompletedEvent, emit
from src.utils.search import search
logger = logging.getLogger(__name__)
@ -114,7 +115,21 @@ async def delete_workspace(
This action cannot be undone.
"""
await crud.delete_workspace(db, workspace_name=workspace_id)
result = await crud.delete_workspace(db, workspace_name=workspace_id)
# Emit telemetry event with cascade counts
emit(
DeletionCompletedEvent(
workspace_name=workspace_id,
deletion_type="workspace",
resource_id=workspace_id,
success=True,
peers_deleted=result.peers_deleted,
sessions_deleted=result.sessions_deleted,
messages_deleted=result.messages_deleted,
conclusions_deleted=result.conclusions_deleted,
)
)
@router.post(

46
src/sentry.py
View File

@ -1,46 +0,0 @@
"""Sentry initialization and configuration."""
from __future__ import annotations
import logging
from collections.abc import Sequence
from typing import TYPE_CHECKING
import sentry_sdk
from src.config import settings
if TYPE_CHECKING:
from sentry_sdk._types import EventProcessor
from sentry_sdk.integrations import Integration
logger = logging.getLogger(__name__)
# Sentry SDK's default behavior:
# - Captures INFO+ level logs as breadcrumbs
# - Captures ERROR+ level logs as Sentry events
#
# For custom log levels, use the LoggingIntegration class:
# sentry_sdk.init(..., integrations=[LoggingIntegration(level=logging.INFO, event_level=logging.ERROR)])
def initialize_sentry(
*,
integrations: Sequence[Integration],
before_send: EventProcessor | None = None,
) -> None:
"""Initialize Sentry SDK with project settings.
Args:
integrations: Sentry SDK integrations to enable (e.g., Starlette, FastAPI).
before_send: Optional event filter callback to suppress specific exceptions.
"""
sentry_sdk.init(
dsn=settings.SENTRY.DSN,
enable_tracing=True,
release=settings.SENTRY.RELEASE,
environment=settings.SENTRY.ENVIRONMENT,
traces_sample_rate=settings.SENTRY.TRACES_SAMPLE_RATE,
profiles_sample_rate=settings.SENTRY.PROFILES_SAMPLE_RATE,
before_send=before_send,
integrations=integrations,
)

85
src/telemetry/__init__.py Normal file
View File

@ -0,0 +1,85 @@
"""
Telemetry module for Honcho.
This module consolidates all telemetry, metrics, and observability functionality:
- Sentry: Error tracking and performance tracing
- OTel: Push-based metrics via OTLP to any compatible backend (e.g., Mimir)
- CloudEvents: Structured events for analytics (push-based)
- Logging: Langfuse integration, Rich console output, metric accumulation
- Tracing: Sentry transaction decorators
- Metrics Collector: JSON file-based benchmark aggregation
- Reasoning Traces: JSONL logging of LLM inputs/outputs
"""
from src.telemetry.events import emit
from src.telemetry.otel import get_meter, initialize_otel_metrics, shutdown_otel_metrics
from src.telemetry.otel.metrics import otel_metrics
__all__ = [
"emit",
"get_meter",
"initialize_otel_metrics",
"initialize_telemetry",
"initialize_telemetry_async",
"otel_metrics",
"shutdown_otel_metrics",
"shutdown_telemetry",
]
def initialize_telemetry() -> None:
"""
Initialize all telemetry systems based on configuration.
This should be called once at application startup (in main.py lifespan).
It reads configuration from settings and initializes:
- OTel metrics (if OTEL_ENABLED=true)
Note: CloudEvents telemetry requires async initialization and should be
initialized separately using initialize_telemetry_async().
Sentry is initialized separately in sentry.py as it has its own lifecycle.
"""
from src.config import settings
if settings.OTEL.ENABLED:
initialize_otel_metrics(
endpoint=settings.OTEL.ENDPOINT,
headers=settings.OTEL.HEADERS,
export_interval_millis=settings.OTEL.EXPORT_INTERVAL_MILLIS,
service_name=settings.OTEL.SERVICE_NAME,
service_namespace=settings.OTEL.SERVICE_NAMESPACE,
enabled=True,
)
async def initialize_telemetry_async() -> None:
"""
Initialize async telemetry systems based on configuration.
This should be called once at application startup (in main.py lifespan),
after initialize_telemetry(). It initializes:
- CloudEvents emitter (if TELEMETRY_ENABLED=true)
"""
from src.config import settings
from src.telemetry.events import initialize_telemetry_events
if settings.TELEMETRY.ENABLED:
await initialize_telemetry_events()
async def shutdown_telemetry() -> None:
"""
Shutdown all telemetry systems gracefully.
This should be called during application shutdown to ensure:
- OTel metrics are flushed
- CloudEvents buffer is flushed
"""
from src.telemetry.events import shutdown_telemetry_events
# Shutdown CloudEvents emitter (flushes buffer)
await shutdown_telemetry_events()
# Shutdown OTel metrics
shutdown_otel_metrics()

379
src/telemetry/emitter.py Normal file
View File

@ -0,0 +1,379 @@
"""
Buffered CloudEvents HTTP emitter with retry logic.
This module provides a TelemetryEmitter class that:
- Buffers events in memory for efficient batching
- Sends events as CloudEvents to a configurable HTTP endpoint
- Implements exponential backoff retry logic
- Supports graceful shutdown with flush
"""
import asyncio
import contextlib
import json
import logging
from collections import deque
from typing import TYPE_CHECKING
import httpx
from cloudevents.conversion import to_json # pyright: ignore[reportUnknownVariableType]
from cloudevents.http import CloudEvent
if TYPE_CHECKING:
from src.telemetry.events.base import BaseEvent
logger = logging.getLogger(__name__)
class TelemetryEmitter:
"""Buffered, async CloudEvents emitter with retry logic.
This emitter queues events and sends them in batches to reduce
network overhead. It supports automatic periodic flushing and
exponential backoff retry on failure.
Usage:
emitter = TelemetryEmitter(
endpoint="https://telemetry.example.com/v1/events",
headers={"X-API-Key": "..."},
)
await emitter.start()
# Queue events (non-blocking)
emitter.emit(my_event)
# Graceful shutdown
await emitter.shutdown()
"""
endpoint: str | None
headers: dict[str, str]
batch_size: int
flush_interval: float
flush_threshold: int
max_retries: int
max_buffer_size: int
enabled: bool
_buffer: deque[CloudEvent]
_flush_task: asyncio.Task[None] | None
_client: httpx.AsyncClient | None
_running: bool
_lock: asyncio.Lock
def __init__(
self,
endpoint: str | None = None,
headers: dict[str, str] | None = None,
batch_size: int = 100,
flush_interval_seconds: float = 1.0,
flush_threshold: int = 50,
max_retries: int = 3,
max_buffer_size: int = 10000,
enabled: bool = True,
):
"""Initialize the telemetry emitter.
Args:
endpoint: CloudEvents HTTP endpoint URL
headers: Optional HTTP headers for authentication
batch_size: Maximum events per batch
flush_interval_seconds: How often to flush the buffer
flush_threshold: Trigger flush when buffer reaches this size
max_retries: Maximum retry attempts on failure
max_buffer_size: Maximum events to buffer (oldest dropped if exceeded)
enabled: Whether emission is enabled
"""
self.endpoint = endpoint
self.headers = headers or {}
self.batch_size = batch_size
self.flush_interval = flush_interval_seconds
self.flush_threshold = flush_threshold
self.max_retries = max_retries
self.max_buffer_size = max_buffer_size
self.enabled = enabled and endpoint is not None
self._buffer = deque(maxlen=max_buffer_size)
self._flush_task = None
self._client = None
self._running = False
self._lock = asyncio.Lock()
async def start(self) -> None:
"""Start the emitter background tasks.
Creates the HTTP client and starts the periodic flush task.
"""
if not self.enabled:
logger.info("Telemetry emitter disabled (no endpoint or disabled)")
return
self._client = httpx.AsyncClient(
timeout=30.0,
headers={
"Content-Type": "application/cloudevents+json",
**self.headers,
},
)
self._running = True
self._flush_task = asyncio.create_task(self._periodic_flush())
logger.info("Telemetry emitter started, endpoint: %s", self.endpoint)
async def shutdown(self) -> None:
"""Gracefully shutdown the emitter.
Stops the periodic flush task, flushes remaining events,
and closes the HTTP client.
"""
if not self.enabled:
return
self._running = False
# Cancel the periodic flush task
if self._flush_task is not None:
self._flush_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._flush_task
# Final flush of remaining events
await self.flush()
# Close HTTP client
if self._client is not None:
await self._client.aclose()
self._client = None
logger.info("Telemetry emitter shutdown complete")
def emit(self, event: "BaseEvent") -> None:
"""Queue an event for emission. Non-blocking.
The event is converted to a CloudEvent and added to the buffer.
If the buffer is full, the oldest events are dropped.
Triggers an immediate flush when buffer reaches flush_threshold.
Args:
event: The Honcho event to emit
"""
if not self.enabled:
return
from src.config import settings
# Generate deterministic event ID
event_id = event.generate_id()
# Build source with namespace for tenant routing
# Format: /honcho/{namespace}/{category} or /honcho/{category}
namespace = settings.TELEMETRY.NAMESPACE
source = f"/honcho/{event.category()}"
if namespace:
source = f"/honcho/{namespace}/{event.category()}"
# Build CloudEvent attributes
attributes: dict[str, str] = {
"id": event_id,
"source": source,
"type": event.event_type(),
"time": event.timestamp.isoformat(),
"dataschema": f"https://honcho.dev/schemas/{event.event_type()}/v{event.schema_version()}",
}
# Create CloudEvent
cloud_event = CloudEvent(attributes, event.model_dump(mode="json"))
self._buffer.append(cloud_event)
buffer_size = len(self._buffer)
logger.debug("Queued event %s (buffer size: %d)", event_id, buffer_size)
# Warning logs as buffer approaches max capacity
capacity_ratio = buffer_size / self.max_buffer_size
if capacity_ratio >= 0.8:
logger.warning(
"Telemetry buffer at %.0f%% capacity (%d/%d events)",
capacity_ratio * 100,
buffer_size,
self.max_buffer_size,
)
logger.debug("Event added to emitter (buffer size: %d)", buffer_size)
# Threshold-based flush trigger
if buffer_size >= self.flush_threshold and self._running:
logger.debug("Triggering flush (buffer size: %d)", buffer_size)
asyncio.create_task(self.flush())
async def flush(self) -> None:
"""Flush buffered events to the endpoint.
Sends events in batches up to batch_size. Uses exponential
backoff retry on failure. Events are returned to the buffer
on permanent failure.
"""
if not self.enabled or not self._buffer or self._client is None:
return
async with self._lock:
while self._buffer:
# Extract a batch
batch: list[CloudEvent] = []
while self._buffer and len(batch) < self.batch_size:
batch.append(self._buffer.popleft())
if not batch:
break
# Try to send the batch
success = await self._send_batch(batch)
if not success:
# Put events back at the front of the buffer
for event in reversed(batch):
self._buffer.appendleft(event)
logger.warning(
"Failed to send batch of %d events, returned to buffer",
len(batch),
)
break
async def _send_batch(self, batch: list[CloudEvent]) -> bool:
"""Send a batch of events to the endpoint with retry logic.
Args:
batch: List of CloudEvents to send
Returns:
True if successful, False if all retries failed
"""
if self._client is None or self.endpoint is None:
return False
for attempt in range(self.max_retries):
try:
# Convert batch to JSON
# For a single event, use structured format
# For multiple events, use batch format (array)
if len(batch) == 1:
body = to_json(batch[0])
else:
# CloudEvents batch format is a JSON array
events_json = [json.loads(to_json(e)) for e in batch]
body = json.dumps(events_json).encode()
response = await self._client.post(
self.endpoint,
content=body,
headers=(
{"Content-Type": "application/cloudevents-batch+json"}
if len(batch) > 1
else None
),
)
response.raise_for_status()
logger.debug(
"Sent batch of %d events (status: %d)",
len(batch),
response.status_code,
)
return True
except httpx.HTTPStatusError as e:
logger.warning(
"HTTP error sending events (attempt %d/%d): %s",
attempt + 1,
self.max_retries,
e,
)
except httpx.RequestError as e:
logger.warning(
"Request error sending events (attempt %d/%d): %s",
attempt + 1,
self.max_retries,
e,
)
# Exponential backoff
if attempt < self.max_retries - 1:
delay = 2**attempt
await asyncio.sleep(delay)
return False
async def _periodic_flush(self) -> None:
"""Background task that periodically flushes the buffer."""
while self._running:
try:
await asyncio.sleep(self.flush_interval)
if self._buffer:
await self.flush()
except asyncio.CancelledError:
break
except Exception as e:
logger.exception("Error in periodic flush: %s", e)
@property
def buffer_size(self) -> int:
"""Return the current number of buffered events."""
return len(self._buffer)
@property
def is_running(self) -> bool:
"""Return whether the emitter is running."""
return self._running
# Global emitter instance (initialized by initialize_telemetry_events)
_emitter: TelemetryEmitter | None = None
def get_emitter() -> TelemetryEmitter | None:
"""Get the global telemetry emitter instance."""
return _emitter
async def initialize_emitter(
endpoint: str | None = None,
headers: dict[str, str] | None = None,
batch_size: int = 100,
flush_interval_seconds: float = 1.0,
flush_threshold: int = 50,
max_retries: int = 3,
max_buffer_size: int = 10000,
enabled: bool = True,
) -> TelemetryEmitter:
"""Initialize and start the global telemetry emitter.
Args:
endpoint: CloudEvents HTTP endpoint URL
headers: Optional HTTP headers for authentication
batch_size: Maximum events per batch
flush_interval_seconds: How often to flush the buffer
flush_threshold: Trigger flush when buffer reaches this size
max_retries: Maximum retry attempts on failure
max_buffer_size: Maximum events to buffer
enabled: Whether emission is enabled
Returns:
The initialized emitter instance
"""
global _emitter
_emitter = TelemetryEmitter(
endpoint=endpoint,
headers=headers,
batch_size=batch_size,
flush_interval_seconds=flush_interval_seconds,
flush_threshold=flush_threshold,
max_retries=max_retries,
max_buffer_size=max_buffer_size,
enabled=enabled,
)
await _emitter.start()
return _emitter
async def shutdown_emitter() -> None:
"""Shutdown the global telemetry emitter."""
global _emitter
if _emitter is not None:
await _emitter.shutdown()
_emitter = None

189
src/telemetry/events/__init__.py Normal file
View File

@ -0,0 +1,189 @@
"""
CloudEvents telemetry event system for Honcho.
This module provides:
- emit(): Queue an event for emission to the telemetry backend
- BaseEvent: Base class for all event types
- Concrete event types for all Honcho operations
Event Categories:
representation: Message processing and conclusion extraction
- RepresentationCompletedEvent: Message batch processed, conclusions extracted
dream: Memory consolidation operations
- DreamRunEvent: Full dream orchestration completed
- DreamSpecialistEvent: Individual specialist (deduction/induction) completed
dialectic: User-initiated queries
- DialecticCompletedEvent: Chat query completed
agent: Agentic loop tracking (correlates via run_id)
- AgentIterationEvent: Per-LLM-call metrics
- AgentToolConclusionsCreatedEvent: Conclusions created by agent
- AgentToolConclusionsDeletedEvent: Conclusions deleted by agent
- AgentToolPeerCardUpdatedEvent: Peer card updated by agent
- AgentToolSummaryCreatedEvent: Summary created
deletion: Resource removal
- DeletionCompletedEvent: Resource deletion completed (with cascade counts)
reconciliation: Maintenance operations
- SyncVectorsCompletedEvent: Vector store sync completed
- CleanupStaleItemsCompletedEvent: Stale items cleanup completed
Usage:
from src.telemetry.events import emit, RepresentationCompletedEvent
emit(RepresentationCompletedEvent(
workspace_name="my_workspace",
session_name="my_session",
observed="assistant_peer",
queue_items_processed=3,
earliest_message_id="msg_001",
latest_message_id="msg_010",
message_count=10,
explicit_conclusion_count=5,
context_preparation_ms=50.0,
llm_call_ms=1200.0,
total_duration_ms=1300.0,
input_tokens=5000,
output_tokens=500,
))
"""
import logging
from src.telemetry.events.agent import (
AgentIterationEvent,
AgentToolConclusionsCreatedEvent,
AgentToolConclusionsDeletedEvent,
AgentToolPeerCardUpdatedEvent,
AgentToolSummaryCreatedEvent,
)
from src.telemetry.events.base import BaseEvent, generate_event_id
from src.telemetry.events.deletion import DeletionCompletedEvent
from src.telemetry.events.dialectic import DialecticCompletedEvent
from src.telemetry.events.dream import (
DreamRunEvent,
DreamSpecialistEvent,
)
from src.telemetry.events.reconciliation import (
CleanupStaleItemsCompletedEvent,
SyncVectorsCompletedEvent,
)
from src.telemetry.events.representation import RepresentationCompletedEvent
logger = logging.getLogger(__name__)
__all__ = [
# Core
"emit",
"BaseEvent",
"generate_event_id",
# Representation events
"RepresentationCompletedEvent",
# Dream events
"DreamRunEvent",
"DreamSpecialistEvent",
# Dialectic events
"DialecticCompletedEvent",
# Agent events
"AgentIterationEvent",
"AgentToolConclusionsCreatedEvent",
"AgentToolConclusionsDeletedEvent",
"AgentToolPeerCardUpdatedEvent",
"AgentToolSummaryCreatedEvent",
# Reconciliation events
"SyncVectorsCompletedEvent",
"CleanupStaleItemsCompletedEvent",
# Deletion events
"DeletionCompletedEvent",
# Lifecycle
"initialize_telemetry_events",
"shutdown_telemetry_events",
]
def emit(event: BaseEvent) -> None:
"""Queue an event for emission to the telemetry backend.
This is the main entry point for emitting telemetry events.
Events are buffered and sent asynchronously to the configured endpoint.
If telemetry is disabled or not initialized, this is a no-op.
Telemetry failures are caught and logged to Sentry to avoid blocking operations.
Args:
event: The event to emit (must be a BaseEvent subclass instance)
Example:
from src.telemetry.events import emit
emit(MyEvent(
my_field="some_value",
...
))
"""
try:
from src.telemetry.emitter import get_emitter
emitter = get_emitter()
if emitter is None:
logger.debug("Telemetry emitter not initialized, dropping event")
return
emitter.emit(event)
except Exception as e:
# Log to Sentry but don't block the main operation
import sentry_sdk
sentry_sdk.capture_exception(e)
logger.warning(
"Failed to emit telemetry event %s: %s",
type(event).__name__,
str(e),
)
async def initialize_telemetry_events() -> None:
"""Initialize the telemetry events system based on configuration.
This should be called once at application startup. It reads
configuration from settings and initializes the CloudEvents emitter.
This is typically called from initialize_telemetry() in the main
telemetry module.
"""
from src.config import settings
from src.telemetry.emitter import initialize_emitter
if not settings.TELEMETRY.ENABLED:
logger.info("CloudEvents telemetry disabled")
return
await initialize_emitter(
endpoint=settings.TELEMETRY.ENDPOINT,
headers=settings.TELEMETRY.HEADERS,
batch_size=settings.TELEMETRY.BATCH_SIZE,
flush_interval_seconds=settings.TELEMETRY.FLUSH_INTERVAL_SECONDS,
flush_threshold=settings.TELEMETRY.FLUSH_THRESHOLD,
max_retries=settings.TELEMETRY.MAX_RETRIES,
max_buffer_size=settings.TELEMETRY.MAX_BUFFER_SIZE,
enabled=True,
)
logger.info(
"CloudEvents telemetry initialized, endpoint: %s", settings.TELEMETRY.ENDPOINT
)
async def shutdown_telemetry_events() -> None:
"""Shutdown the telemetry events system.
This should be called during application shutdown to ensure
all buffered events are flushed before exit.
"""
from src.telemetry.emitter import shutdown_emitter
await shutdown_emitter()
logger.info("CloudEvents telemetry shutdown complete")

233
src/telemetry/events/agent.py Normal file
View File

@ -0,0 +1,233 @@
"""
Agent events for Honcho telemetry.
Agent events track per-iteration metrics and state-changing tool executions
within agentic loops (dream specialists, dialectic agents). These events
correlate with parent events via run_id for detailed analytics.
Events in this module:
- AgentIterationEvent: Per-LLM-call metrics within an agent run
- AgentToolConclusionsCreatedEvent: Conclusions created by agent tool
- AgentToolConclusionsDeletedEvent: Conclusions deleted by agent tool
- AgentToolPeerCardUpdatedEvent: Peer card updated by agent tool
- AgentToolSummaryCreatedEvent: Summary created by agent tool (future)
"""
from typing import ClassVar
from pydantic import Field
from src.telemetry.events.base import BaseEvent
class AgentIterationEvent(BaseEvent):
"""Emitted after each LLM call within an agentic loop.
Tracks per-iteration token usage and tool calls for detailed cost analysis.
Correlates with parent events (dream.run, dream.specialist, dialectic.completed)
via run_id.
"""
_event_type: ClassVar[str] = "agent.iteration"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "agent"
# Run identification
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
# Context
parent_category: str = Field(
..., description="Parent category: 'dream' or 'dialectic'"
)
agent_type: str = Field(
..., description="Agent type: 'deduction', 'induction', or 'dialectic'"
)
workspace_name: str = Field(..., description="Workspace name")
# Peer context (for dream agents)
observer: str | None = Field(default=None, description="Observer peer name")
observed: str | None = Field(default=None, description="Observed peer name")
# Peer context (for dialectic agent)
peer_name: str | None = Field(default=None, description="Peer name being queried")
# Iteration info
iteration: int = Field(..., description="Iteration number (1-indexed)")
# What happened in this iteration
tool_calls: list[str] = Field(
default_factory=list,
description="Tool names called in this iteration (can be empty or multiple)",
)
# Token usage for this single LLM call
input_tokens: int = Field(..., description="Input tokens for this iteration")
output_tokens: int = Field(..., description="Output tokens for this iteration")
cache_read_tokens: int = Field(
default=0, description="Tokens read from prompt cache"
)
cache_creation_tokens: int = Field(
default=0, description="Tokens written to prompt cache"
)
def get_resource_id(self) -> str:
"""Resource ID includes run_id and iteration for uniqueness."""
return f"{self.run_id}:{self.iteration}"
class AgentToolConclusionsCreatedEvent(BaseEvent):
"""Emitted when the create_conclusions tool is executed.
Tracks conclusion creation with level breakdown for analytics.
"""
_event_type: ClassVar[str] = "agent.tool.conclusions.created"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "agent"
# Run identification
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
iteration: int = Field(..., description="Iteration number when this occurred")
# Context
parent_category: str = Field(
..., description="Parent category: 'dream' or 'dialectic'"
)
agent_type: str = Field(
..., description="Agent type: 'deduction', 'induction', or 'dialectic'"
)
workspace_name: str = Field(..., description="Workspace name")
# Peer context
observer: str = Field(..., description="Observer peer name")
observed: str = Field(..., description="Observed peer name")
# What was created
conclusion_count: int = Field(..., description="Number of conclusions created")
levels: list[str] = Field(
default_factory=list,
description="Level of each conclusion (e.g., ['explicit', 'deductive', 'deductive'])",
)
def get_resource_id(self) -> str:
"""Resource ID includes run_id and iteration for uniqueness."""
return f"{self.run_id}:{self.iteration}:conclusions_created"
class AgentToolConclusionsDeletedEvent(BaseEvent):
"""Emitted when the delete_conclusions tool is executed.
Tracks conclusion deletion for memory consolidation analytics.
"""
_event_type: ClassVar[str] = "agent.tool.conclusions.deleted"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "agent"
# Run identification
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
iteration: int = Field(..., description="Iteration number when this occurred")
# Context
parent_category: str = Field(..., description="Parent category (typically 'dream')")
agent_type: str = Field(
..., description="Agent type (typically 'deduction' for deletions)"
)
workspace_name: str = Field(..., description="Workspace name")
# Peer context
observer: str = Field(..., description="Observer peer name")
observed: str = Field(..., description="Observed peer name")
# What was deleted
conclusion_count: int = Field(..., description="Number of conclusions deleted")
def get_resource_id(self) -> str:
"""Resource ID includes run_id and iteration for uniqueness."""
return f"{self.run_id}:{self.iteration}:conclusions_deleted"
class AgentToolPeerCardUpdatedEvent(BaseEvent):
"""Emitted when the update_peer_card tool is executed.
Tracks peer card updates during dream consolidation.
"""
_event_type: ClassVar[str] = "agent.tool.peer_card.updated"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "agent"
# Run identification
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
iteration: int = Field(..., description="Iteration number when this occurred")
# Context
parent_category: str = Field(..., description="Parent category (typically 'dream')")
agent_type: str = Field(..., description="Agent type: 'deduction' or 'induction'")
workspace_name: str = Field(..., description="Workspace name")
# Peer context
observer: str = Field(..., description="Observer peer name")
observed: str = Field(..., description="Observed peer name")
# What was updated
facts_count: int = Field(..., description="Number of facts in the peer card")
def get_resource_id(self) -> str:
"""Resource ID includes run_id and iteration for uniqueness."""
return f"{self.run_id}:{self.iteration}:peer_card_updated"
class AgentToolSummaryCreatedEvent(BaseEvent):
"""Emitted when a summary is created.
Tracks summary creation with full context about what was summarized
and the resources consumed.
"""
_event_type: ClassVar[str] = "agent.tool.summary.created"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "agent"
# Run identification (may be placeholder if not from an agentic loop)
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
iteration: int = Field(..., description="Iteration number when this occurred")
# Context
parent_category: str = Field(..., description="Parent category")
agent_type: str = Field(..., description="Agent type")
workspace_name: str = Field(..., description="Workspace name")
# Session context
session_name: str = Field(..., description="Session name")
# Message context - what was summarized
message_id: str = Field(..., description="Message ID the summary covers up to")
message_count: int = Field(
..., description="Number of messages included in summary"
)
message_seq_in_session: int = Field(
..., description="Sequence number of the base message in session"
)
# Summary details
summary_type: str = Field(..., description="Summary type: 'short' or 'long'")
# Token usage
input_tokens: int = Field(
..., description="Input tokens used for summary generation"
)
output_tokens: int = Field(..., description="Output tokens (summary token count)")
def get_resource_id(self) -> str:
"""Resource ID includes run_id and iteration for uniqueness."""
return f"{self.run_id}:{self.iteration}:summary_created"
__all__ = [
"AgentIterationEvent",
"AgentToolConclusionsCreatedEvent",
"AgentToolConclusionsDeletedEvent",
"AgentToolPeerCardUpdatedEvent",
"AgentToolSummaryCreatedEvent",
]

97
src/telemetry/events/base.py Normal file
View File

@ -0,0 +1,97 @@
"""
Base event classes and idempotency key generation for CloudEvents telemetry.
This module provides:
- BaseEvent: Abstract base class for all Honcho telemetry events
- generate_event_id(): Deterministic event ID generation for idempotency
"""
import base64
import hashlib
from datetime import UTC, datetime
from typing import ClassVar
from pydantic import BaseModel, Field
def generate_event_id(
event_type: str,
timestamp: datetime,
resource_id: str,
) -> str:
"""Generate a deterministic event ID for idempotency.
Same inputs always produce the same ID, so retries are automatically
deduplicated on the receiving end.
Args:
event_type: The CloudEvents type (e.g., "honcho.work.representation.completed")
timestamp: When the event occurred
resource_id: A unique identifier for the resource (can include workspace_id if relevant)
Returns:
A deterministic event ID in the format "evt_{base64_hash}"
"""
payload = f"{event_type}:{resource_id}:{timestamp.isoformat()}"
hash_bytes = hashlib.sha256(payload.encode()).digest()[:16]
# Use URL-safe base64 encoding, strip padding
encoded = base64.urlsafe_b64encode(hash_bytes).decode().rstrip("=")
return f"evt_{encoded}"
class BaseEvent(BaseModel):
"""Base class for all Honcho telemetry events.
All events inherit from this class and define:
- _event_type: The CloudEvents type string
- _schema_version: Integer version for schema evolution
- _category: Event category (work, activity, resource)
Subclasses must implement the abstract class methods by setting class variables
and providing get_resource_id(). Subclasses define their own context fields
(e.g., workspace_id, workspace_name) as needed - not all events have workspace context.
"""
# Class variables for event metadata (set by subclasses)
_event_type: ClassVar[str]
_schema_version: ClassVar[int]
_category: ClassVar[str] # "work", "activity", or "resource"
# Common timestamp field present in all events
timestamp: datetime = Field(
default_factory=lambda: datetime.now(UTC),
description="When the event occurred (UTC)",
)
@classmethod
def event_type(cls) -> str:
"""Return the CloudEvents type string for this event."""
return cls._event_type
@classmethod
def schema_version(cls) -> int:
"""Return the schema version for this event."""
return cls._schema_version
@classmethod
def category(cls) -> str:
"""Return the event category (work, activity, or resource)."""
return cls._category
def get_resource_id(self) -> str:
"""Return the resource ID for idempotency key generation.
This should be a unique identifier for the specific operation,
such as a task_id or a hash of message IDs.
Must be implemented by subclasses.
"""
raise NotImplementedError("Subclasses must implement get_resource_id()")
def generate_id(self) -> str:
"""Generate a deterministic event ID for this event instance."""
return generate_event_id(
event_type=self.event_type(),
timestamp=self.timestamp,
resource_id=self.get_resource_id(),
)

65
src/telemetry/events/deletion.py Normal file
View File

@ -0,0 +1,65 @@
"""
Deletion events for Honcho telemetry.
Deletion tasks handle async removal of resources like workspaces, sessions,
and conclusions, ensuring proper cleanup across all storage layers.
"""
from typing import ClassVar
from pydantic import Field
from src.telemetry.events.base import BaseEvent
class DeletionCompletedEvent(BaseEvent):
"""Emitted when a deletion task completes.
Deletion tasks handle async removal of workspaces, sessions, and conclusions,
ensuring proper cleanup across all storage layers. For workspace deletions,
cascade counts track how many child resources were also deleted.
"""
_event_type: ClassVar[str] = "deletion.completed"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "deletion"
# Workspace context
workspace_name: str = Field(..., description="Workspace name")
# Deletion details
deletion_type: str = Field(
..., description="Type of deletion: 'workspace', 'session', or 'conclusions'"
)
resource_id: str = Field(..., description="ID of the deleted resource")
# Outcome
success: bool = Field(..., description="Whether deletion succeeded")
# Cascade counts (populated for workspace and session deletions)
peers_deleted: int = Field(
default=0, description="Number of peers deleted (workspace deletion only)"
)
sessions_deleted: int = Field(
default=0, description="Number of sessions deleted (workspace deletion only)"
)
messages_deleted: int = Field(
default=0,
description="Number of messages deleted (workspace or session deletion)",
)
conclusions_deleted: int = Field(
default=0,
description="Number of conclusions/documents deleted (workspace or session deletion)",
)
# Optional error info
error_message: str | None = Field(
default=None, description="Error message if deletion failed"
)
def get_resource_id(self) -> str:
"""Resource ID includes workspace, type, and resource for uniqueness."""
return f"{self.workspace_name}:{self.deletion_type}:{self.resource_id}"
__all__ = ["DeletionCompletedEvent"]

75
src/telemetry/events/dialectic.py Normal file
View File

@ -0,0 +1,75 @@
"""
Dialectic events for Honcho telemetry.
Dialectic tasks answer questions about peers by gathering context from memory.
These are user-initiated operations that query the representation system.
The run_id field enables correlation with agent.iteration events.
"""
from typing import ClassVar
from pydantic import Field
from src.telemetry.events.base import BaseEvent
class DialecticCompletedEvent(BaseEvent):
"""Emitted when a dialectic (chat) query completes.
Dialectic queries answer questions about peers by gathering context
from memory. This event captures the full context of the query and
its execution metrics.
The run_id correlates with AgentIterationEvent and AgentTool* events
for detailed analytics.
"""
_event_type: ClassVar[str] = "dialectic.completed"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "dialectic"
# Run identification (for correlating with iteration/tool events)
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
# Workspace context
workspace_name: str = Field(..., description="Workspace name")
# Peer context
peer_name: str = Field(..., description="Peer name being queried about")
# Session context (optional - dialectic can run without session)
session_name: str | None = Field(
default=None, description="Session name if provided"
)
# Query configuration
reasoning_level: str = Field(
..., description="Reasoning level: minimal, low, medium, high, max"
)
# Execution metrics
total_iterations: int = Field(default=1, description="Number of LLM iterations")
prefetched_conclusion_count: int = Field(
default=0, description="Number of conclusions prefetched"
)
tool_calls_count: int = Field(default=0, description="Number of tool calls made")
# Timing metrics (milliseconds)
total_duration_ms: float = Field(..., description="Total processing time")
# Token usage with cache breakdown
input_tokens: int = Field(..., description="Total input tokens")
output_tokens: int = Field(..., description="Output tokens generated")
cache_read_tokens: int = Field(
default=0, description="Tokens read from prompt cache"
)
cache_creation_tokens: int = Field(
default=0, description="Tokens written to prompt cache"
)
def get_resource_id(self) -> str:
"""Resource ID is the run_id for uniqueness."""
return self.run_id
__all__ = ["DialecticCompletedEvent"]

119
src/telemetry/events/dream.py Normal file
View File

@ -0,0 +1,119 @@
"""
Dream events for Honcho telemetry.
Dream tasks handle memory consolidation through specialist agents (deduction, induction).
Events track both the overall dream run and individual specialist executions,
with run_id correlation for analytics queries.
"""
from typing import ClassVar
from pydantic import Field
from src.telemetry.events.base import BaseEvent
class DreamRunEvent(BaseEvent):
"""Emitted when a full dream orchestration completes.
This is the top-level event for a dream run, providing aggregate metrics
across all specialists. Individual specialist details are captured in
DreamSpecialistEvent with the same run_id for correlation.
"""
_event_type: ClassVar[str] = "dream.run"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "dream"
# Run identification (for correlating with specialist/iteration/tool events)
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
# Workspace context
workspace_name: str = Field(..., description="Workspace name")
# Session context
session_name: str = Field(..., description="Most recent session name")
# Peer context
observer: str = Field(..., description="Observer peer name")
observed: str = Field(..., description="Observed peer name")
# What ran
specialists_run: list[str] = Field(
..., description="Specialists executed (e.g., ['deduction', 'induction'])"
)
deduction_success: bool = Field(
..., description="Whether deduction specialist succeeded"
)
induction_success: bool = Field(
..., description="Whether induction specialist succeeded"
)
# Surprisal sampling (optional phase)
surprisal_enabled: bool = Field(
default=False, description="Whether surprisal sampling was enabled"
)
surprisal_conclusion_count: int = Field(
default=0, description="High-surprisal conclusions found"
)
# Aggregated metrics across all specialists
total_iterations: int = Field(
..., description="Total LLM iterations across all specialists"
)
total_input_tokens: int = Field(
..., description="Total input tokens across all specialists"
)
total_output_tokens: int = Field(
..., description="Total output tokens across all specialists"
)
total_duration_ms: float = Field(..., description="Total processing time")
def get_resource_id(self) -> str:
"""Resource ID is the run_id for uniqueness."""
return self.run_id
class DreamSpecialistEvent(BaseEvent):
"""Emitted when an individual dream specialist completes.
Each specialist (deduction, induction) emits its own event with
the same run_id as the parent DreamRunEvent for correlation.
"""
_event_type: ClassVar[str] = "dream.specialist"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "dream"
# Run identification (correlates with parent dream.run)
run_id: str = Field(..., description="8-char UUID prefix for run correlation")
# Specialist info
specialist_type: str = Field(
..., description="Specialist type: 'deduction' or 'induction'"
)
# Workspace context
workspace_name: str = Field(..., description="Workspace name")
# Peer context
observer: str = Field(..., description="Observer peer name")
observed: str = Field(..., description="Observed peer name")
# Execution metrics
iterations: int = Field(..., description="Number of LLM iterations")
tool_calls_count: int = Field(..., description="Total tool calls made")
input_tokens: int = Field(..., description="Input tokens used")
output_tokens: int = Field(..., description="Output tokens generated")
duration_ms: float = Field(..., description="Processing time")
success: bool = Field(..., description="Whether the specialist succeeded")
def get_resource_id(self) -> str:
"""Resource ID includes run_id and specialist type for uniqueness."""
return f"{self.run_id}:{self.specialist_type}"
__all__ = [
"DreamRunEvent",
"DreamSpecialistEvent",
]

88
src/telemetry/events/reconciliation.py Normal file
View File

@ -0,0 +1,88 @@
"""
Reconciliation events for Honcho telemetry.
Reconciliation tasks handle maintenance operations:
- Sync vectors: Synchronize documents and message embeddings to external vector stores
- Cleanup stale items: Clean up soft-deleted records and expired queue items
"""
from typing import ClassVar
from pydantic import Field
from src.telemetry.events.base import BaseEvent
class SyncVectorsCompletedEvent(BaseEvent):
"""Emitted when a vector sync cycle completes.
Vector sync tasks synchronize documents and message embeddings to external
vector stores. These run periodically and operate across all workspaces.
Note: This event has no workspace context as it operates globally.
"""
_event_type: ClassVar[str] = "reconciliation.sync_vectors.completed"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "reconciliation"
# Document metrics
documents_synced: int = Field(
default=0, description="Documents successfully synced to vector store"
)
documents_failed: int = Field(
default=0, description="Documents that failed to sync"
)
documents_cleaned: int = Field(
default=0, description="Soft-deleted documents cleaned up during sync"
)
# Message embedding metrics
message_embeddings_synced: int = Field(
default=0, description="Message embeddings successfully synced"
)
message_embeddings_failed: int = Field(
default=0, description="Message embeddings that failed to sync"
)
# Timing metrics (milliseconds)
total_duration_ms: float = Field(..., description="Total processing time")
def get_resource_id(self) -> str:
"""Resource ID is fixed for this global operation."""
return "sync_vectors"
class CleanupStaleItemsCompletedEvent(BaseEvent):
"""Emitted when a stale items cleanup cycle completes.
Cleanup tasks remove soft-deleted documents and expired queue items.
These run periodically and operate across all workspaces.
Note: This event has no workspace context as it operates globally.
"""
_event_type: ClassVar[str] = "reconciliation.cleanup_stale_items.completed"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "reconciliation"
# Cleanup metrics
documents_cleaned: int = Field(
default=0, description="Soft-deleted documents cleaned up"
)
queue_items_cleaned: int = Field(
default=0, description="Expired queue items cleaned up"
)
# Timing metrics (milliseconds)
total_duration_ms: float = Field(..., description="Total processing time")
def get_resource_id(self) -> str:
"""Resource ID is fixed for this global operation."""
return "cleanup_stale_items"
__all__ = [
"SyncVectorsCompletedEvent",
"CleanupStaleItemsCompletedEvent",
]

68
src/telemetry/events/representation.py Normal file
View File

@ -0,0 +1,68 @@
"""
Representation events for Honcho telemetry.
Representation tasks extract conclusions from messages to build peer
representations. These events track the processing of message batches
and the conclusions extracted from them.
"""
from typing import ClassVar
from pydantic import Field
from src.telemetry.events.base import BaseEvent
class RepresentationCompletedEvent(BaseEvent):
"""Emitted when a representation task completes processing a message batch.
Representation tasks extract conclusions from messages to build peer
representations. This event captures the full context of what was processed
and what was extracted.
"""
_event_type: ClassVar[str] = "representation.completed"
_schema_version: ClassVar[int] = 1
_category: ClassVar[str] = "representation"
# Workspace context
workspace_name: str = Field(..., description="Workspace name")
# Session context
session_name: str = Field(..., description="Session name")
# Peer context
observed: str = Field(..., description="Observed peer name")
# Queue processing info
queue_items_processed: int = Field(
..., description="Number of QueueItem records dequeued and processed"
)
# Message batch info
earliest_message_id: str = Field(..., description="First message ID in batch")
latest_message_id: str = Field(..., description="Last message ID in batch")
message_count: int = Field(..., description="Number of messages processed")
# Conclusion counts
explicit_conclusion_count: int = Field(
..., description="Number of explicit conclusions extracted"
)
# Timing metrics (milliseconds)
context_preparation_ms: float = Field(
..., description="Time spent preparing context"
)
llm_call_ms: float = Field(..., description="Time spent in LLM call")
total_duration_ms: float = Field(..., description="Total processing time")
# Token usage
input_tokens: int = Field(..., description="Input tokens used")
output_tokens: int = Field(..., description="Output tokens generated")
def get_resource_id(self) -> str:
"""Resource ID includes workspace, session, and latest message for uniqueness."""
return f"{self.workspace_name}:{self.session_name}:{self.latest_message_id}"
__all__ = ["RepresentationCompletedEvent"]

35
src/utils/logging.py → src/telemetry/logging.py
View File

@ -5,6 +5,7 @@ and a conditional observe decorator that only applies when Langfuse is configure
"""
import datetime
from collections import OrderedDict
from collections.abc import Callable
from typing import ParamSpec, TypeVar, overload
@ -18,7 +19,7 @@ from rich.text import Text
from rich.tree import Tree
from src.config import settings
from src.utils.metrics_collector import append_metrics_to_file
from src.telemetry.metrics_collector import append_metrics_to_file
from src.utils.representation import (
Representation,
)
@ -80,8 +81,16 @@ def conditional_observe(
return decorator
# dict[task_name, list[tuple[metric_name, metric_value, metric_unit]]]
accumulated_metrics: dict[str, list[tuple[str, str | int | float, str]]] = {}
# Bounded OrderedDict for accumulated metrics to prevent memory leaks.
# If an exception occurs between accumulate_metric() and log_performance_metrics(),
# the metrics would stay in memory forever. Using OrderedDict allows us to evict
# the oldest entries (FIFO) when exceeding MAX_ACCUMULATED_TASKS.
MAX_ACCUMULATED_TASKS = 1000
# OrderedDict[task_name, list[tuple[metric_name, metric_value, metric_unit]]]
accumulated_metrics: OrderedDict[str, list[tuple[str, str | int | float, str]]] = (
OrderedDict()
)
def format_reasoning_inputs_as_markdown(
@ -157,11 +166,25 @@ def accumulate_metric(
) -> None:
"""
Accumulate a metric value to be printed the next time log_performance_metrics is called.
This function uses a bounded OrderedDict to prevent memory leaks. If the number of
tracked tasks exceeds MAX_ACCUMULATED_TASKS, the oldest entries are evicted (FIFO).
Args:
task_name: Task identifier
label: Metric label
value: Metric value
unit: Metric unit
"""
# Evict oldest entries if we've exceeded the maximum to prevent memory leaks.
# This handles the case where exceptions occur before log_performance_metrics is called.
while (
len(accumulated_metrics) >= MAX_ACCUMULATED_TASKS
and task_name not in accumulated_metrics
):
# popitem(last=False) removes the oldest (first inserted) entry
accumulated_metrics.popitem(last=False)
accumulated_metrics.setdefault(task_name, []).append((label, value, unit))
@ -225,12 +248,14 @@ def log_performance_metrics(
title: Table title
"""
task_name = f"{task_slug}_{task_name}"
# No-op if metrics were evicted (due to MAX_ACCUMULATED_TASKS limit) and no
# additional metrics are passed. This handles the case where an exception
# caused old metrics to be evicted before log_performance_metrics was called.
if not accumulated_metrics.get(task_name) and not metrics:
return
if metrics is None:
metrics = []
metrics = accumulated_metrics.get(task_name, []) + metrics
accumulated_metrics[task_name].clear()
metrics = accumulated_metrics.pop(task_name, []) + metrics
if COLLECT_METRICS_LOCAL:
append_metrics_to_file(task_slug, task_name, metrics)

0
src/utils/metrics_collector.py → src/telemetry/metrics_collector.py
View File

26
src/telemetry/otel/__init__.py Normal file
View File

@ -0,0 +1,26 @@
"""
OpenTelemetry metrics module for Honcho.
This module provides push-based metrics using OpenTelemetry SDK
with OTLP HTTP export to any compatible backend (Mimir, Grafana Cloud, etc.).
"""
from src.telemetry.otel.metrics import (
DeriverComponents,
DeriverTaskTypes,
DialecticComponents,
TokenTypes,
get_meter,
initialize_otel_metrics,
shutdown_otel_metrics,
)
__all__ = [
"DeriverComponents",
"DeriverTaskTypes",
"DialecticComponents",
"TokenTypes",
"get_meter",
"initialize_otel_metrics",
"shutdown_otel_metrics",
]

481
src/telemetry/otel/metrics.py Normal file
View File

@ -0,0 +1,481 @@
"""
OpenTelemetry metrics implementation with OTLP export.
This module provides push-based metrics that replace the pull-based Prometheus
/metrics endpoint. Metrics are pushed via OTLP to any compatible backend
(Mimir, Grafana Cloud, etc.) at configurable intervals.
Usage:
from src.telemetry.otel import get_meter, initialize_otel_metrics
# Initialize once at startup
initialize_otel_metrics()
# Get a meter for your component
meter = get_meter("honcho.deriver")
# Create instruments
counter = meter.create_counter("tokens_processed", unit="tokens")
counter.add(100, {"task_type": "ingestion"})
"""
from __future__ import annotations
import atexit
import logging
from enum import Enum
from typing import TYPE_CHECKING, final
from opentelemetry import metrics
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
if TYPE_CHECKING:
from opentelemetry.metrics import Counter, Meter
logger = logging.getLogger(__name__)
# =============================================================================
# Metric label enums
# =============================================================================
class TokenTypes(Enum):
INPUT = "input"
OUTPUT = "output"
class DeriverTaskTypes(Enum):
INGESTION = "ingestion"
SUMMARY = "summary"
class DeriverComponents(Enum):
PROMPT = "prompt" # used in ingestion and summary
MESSAGES = "messages" # used in ingestion and summary
PREVIOUS_SUMMARY = "previous_summary" # only used for summary
OUTPUT_TOTAL = "output_total"
class DialecticComponents(Enum):
TOTAL = "total"
# =============================================================================
# OTel metrics infrastructure
# =============================================================================
# Global state
_meter_provider: MeterProvider | None = None
_initialized: bool = False
def initialize_otel_metrics(
*,
endpoint: str | None = None,
headers: dict[str, str] | None = None,
export_interval_millis: int = 60000,
service_name: str = "honcho",
service_namespace: str | None = None,
enabled: bool = True,
) -> None:
"""
Initialize OpenTelemetry metrics with OTLP export.
This should be called once at application startup. If already initialized,
subsequent calls are no-ops.
Args:
endpoint: OTLP HTTP endpoint URL (e.g., "https://mimir.example.com/otlp/v1/metrics").
If None, metrics are collected but not exported (useful for testing).
headers: Optional headers to include in requests (e.g., {"X-Scope-OrgID": "tenant"}).
export_interval_millis: How often to export metrics (default: 60 seconds).
service_name: Service name for resource attributes (default: "honcho").
service_namespace: Optional namespace for the service.
enabled: If False, metrics are no-ops (default: True).
"""
global _meter_provider, _initialized
if _initialized:
logger.debug("OTel metrics already initialized, skipping")
return
if not enabled:
logger.info("OTel metrics disabled")
_initialized = True
return
# Build resource attributes
resource_attributes = {
"service.name": service_name,
}
if service_namespace:
resource_attributes["service.namespace"] = service_namespace
resource = Resource.create(resource_attributes)
# Create metric reader
readers: list[PeriodicExportingMetricReader] = []
if endpoint:
try:
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
OTLPMetricExporter,
)
exporter = OTLPMetricExporter(
endpoint=endpoint,
headers=headers or {},
)
reader = PeriodicExportingMetricReader(
exporter,
export_interval_millis=export_interval_millis,
)
readers.append(reader)
logger.info(f"OTel metrics configured to push via OTLP to {endpoint}")
except Exception as e:
logger.error(f"Failed to configure OTLP metrics exporter: {e}")
# Continue without exporter - metrics still work locally
else:
logger.info(
"OTel metrics initialized without remote export (no endpoint configured)"
)
# Create and set the meter provider
# Note: empty list is valid for metric_readers (metrics still work, just not exported)
_meter_provider = MeterProvider(
resource=resource,
metric_readers=readers,
)
metrics.set_meter_provider(_meter_provider)
# Register shutdown handler
atexit.register(shutdown_otel_metrics)
_initialized = True
logger.info("OTel metrics initialized successfully")
def shutdown_otel_metrics() -> None:
"""
Shutdown the OTel metrics provider, flushing any pending metrics.
This is automatically called at process exit via atexit, but can be
called manually for graceful shutdown.
"""
global _meter_provider, _initialized
if _meter_provider is not None:
try:
_meter_provider.shutdown()
logger.info("OTel metrics shutdown complete")
except Exception as e:
logger.error(f"Error during OTel metrics shutdown: {e}")
finally:
_meter_provider = None
_initialized = False
def get_meter(name: str, version: str = "") -> Meter:
"""
Get an OTel Meter for creating instruments.
Args:
name: The name of the instrumentation scope (e.g., "honcho.deriver").
version: Optional version of the instrumentation scope.
Returns:
An OTel Meter instance for creating counters, histograms, etc.
Example:
meter = get_meter("honcho.deriver")
counter = meter.create_counter("tokens_processed", unit="tokens")
counter.add(100, {"task_type": "ingestion"})
"""
return metrics.get_meter(name, version)
# =============================================================================
# Pre-defined metrics that mirror existing Prometheus counters
# =============================================================================
# These are created lazily on first use to avoid issues with initialization order
@final
class OTelMetrics:
"""
Container for OTel metrics that mirror existing Prometheus counters.
This class provides a bridge during migration - the same metrics are
available via both Prometheus (pull) and OTel (push).
Namespace is managed at the instance level (from settings), not per-call.
"""
_instance: OTelMetrics | None = None
_is_initialized: bool = False
_namespace: str = "honcho"
# Meters (lazily initialized)
_api_meter: Meter | None = None
_deriver_meter: Meter | None = None
_dialectic_meter: Meter | None = None
_dreamer_meter: Meter | None = None
# Counters (lazily initialized)
_api_requests: Counter | None = None
_messages_created: Counter | None = None
_dialectic_calls: Counter | None = None
_deriver_queue_items: Counter | None = None
_deriver_tokens: Counter | None = None
_dialectic_tokens: Counter | None = None
_dreamer_tokens: Counter | None = None
def __new__(cls) -> OTelMetrics:
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def _ensure_initialized(self) -> None:
"""Lazily initialize meters and instruments."""
if self._is_initialized:
return
# Get namespace from settings (same as Prometheus)
from src.config import settings
self._namespace = settings.OTEL.SERVICE_NAMESPACE or "honcho"
# Get meters for different components
self._api_meter = get_meter("honcho.api")
self._deriver_meter = get_meter("honcho.deriver")
self._dialectic_meter = get_meter("honcho.dialectic")
self._dreamer_meter = get_meter("honcho.dreamer")
# Create counters that mirror Prometheus metrics
# API requests
self._api_requests = self._api_meter.create_counter(
name="api_requests",
unit="requests",
description="Total API requests",
)
# Messages created
self._messages_created = self._api_meter.create_counter(
name="messages_created",
unit="messages",
description="Total messages created",
)
# Dialectic calls
self._dialectic_calls = self._dialectic_meter.create_counter(
name="dialectic_calls",
unit="calls",
description="Total dialectic calls",
)
# Deriver queue items processed
self._deriver_queue_items = self._deriver_meter.create_counter(
name="deriver_queue_items_processed",
unit="items",
description="Total deriver queue items processed",
)
# Token counters
self._deriver_tokens = self._deriver_meter.create_counter(
name="deriver_tokens_processed",
unit="tokens",
description="Total tokens processed by the deriver",
)
self._dialectic_tokens = self._dialectic_meter.create_counter(
name="dialectic_tokens_processed",
unit="tokens",
description="Total tokens processed by the dialectic",
)
self._dreamer_tokens = self._dreamer_meter.create_counter(
name="dreamer_tokens_processed",
unit="tokens",
description="Total tokens processed by the dreamer",
)
self._is_initialized = True
def _handle_metric_error(self, method_name: str, error: Exception) -> None:
"""Handle errors from metric recording by logging to Sentry."""
import sentry_sdk
sentry_sdk.capture_exception(error)
logger.warning(
"Failed to record OTel metric in %s: %s", method_name, str(error)
)
def record_api_request(
self,
*,
method: str,
endpoint: str,
status_code: str,
) -> None:
"""Record an API request metric."""
try:
self._ensure_initialized()
if self._api_requests is None:
return # Not initialized, skip silently
self._api_requests.add(
1,
{
"method": method,
"endpoint": endpoint,
"status_code": status_code,
"namespace": self._namespace,
},
)
except Exception as e:
self._handle_metric_error("record_api_request", e)
def record_messages_created(
self,
*,
count: int,
workspace_name: str,
) -> None:
"""Record messages created metric."""
try:
self._ensure_initialized()
if self._messages_created is None:
return # Not initialized, skip silently
self._messages_created.add(
count,
{
"workspace_name": workspace_name,
"namespace": self._namespace,
},
)
except Exception as e:
self._handle_metric_error("record_messages_created", e)
def record_dialectic_call(
self,
*,
workspace_name: str,
reasoning_level: str,
) -> None:
"""Record a dialectic call metric."""
try:
self._ensure_initialized()
if self._dialectic_calls is None:
return # Not initialized, skip silently
self._dialectic_calls.add(
1,
{
"workspace_name": workspace_name,
"reasoning_level": reasoning_level,
"namespace": self._namespace,
},
)
except Exception as e:
self._handle_metric_error("record_dialectic_call", e)
def record_deriver_queue_item(
self,
*,
count: int,
workspace_name: str,
task_type: str,
) -> None:
"""Record deriver queue items processed metric."""
try:
self._ensure_initialized()
if self._deriver_queue_items is None:
return # Not initialized, skip silently
self._deriver_queue_items.add(
count,
{
"workspace_name": workspace_name,
"task_type": task_type,
"namespace": self._namespace,
},
)
except Exception as e:
self._handle_metric_error("record_deriver_queue_item", e)
def record_deriver_tokens(
self,
*,
count: int,
task_type: str,
token_type: str,
component: str,
) -> None:
"""Record deriver token usage metric."""
try:
self._ensure_initialized()
if self._deriver_tokens is None:
return # Not initialized, skip silently
self._deriver_tokens.add(
count,
{
"task_type": task_type,
"token_type": token_type,
"component": component,
"namespace": self._namespace,
},
)
except Exception as e:
self._handle_metric_error("record_deriver_tokens", e)
def record_dialectic_tokens(
self,
*,
count: int,
token_type: str,
component: str,
reasoning_level: str,
) -> None:
"""Record dialectic token usage metric."""
try:
self._ensure_initialized()
if self._dialectic_tokens is None:
return # Not initialized, skip silently
self._dialectic_tokens.add(
count,
{
"token_type": token_type,
"component": component,
"reasoning_level": reasoning_level,
"namespace": self._namespace,
},
)
except Exception as e:
self._handle_metric_error("record_dialectic_tokens", e)
def record_dreamer_tokens(
self,
*,
count: int,
specialist_name: str,
token_type: str,
) -> None:
"""Record dreamer token usage metric."""
try:
self._ensure_initialized()
if self._dreamer_tokens is None:
return # Not initialized, skip silently
self._dreamer_tokens.add(
count,
{
"specialist_name": specialist_name,
"token_type": token_type,
"namespace": self._namespace,
},
)
except Exception as e:
self._handle_metric_error("record_dreamer_tokens", e)
# Singleton instance
otel_metrics = OTelMetrics()

0
src/utils/reasoning_traces.py → src/telemetry/reasoning_traces.py
View File

90
src/telemetry/sentry.py Normal file
View File

@ -0,0 +1,90 @@
"""Sentry initialization and configuration."""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Callable, Sequence
from functools import wraps
from typing import TYPE_CHECKING, ParamSpec, TypeVar, cast
import sentry_sdk
from src.config import settings
P = ParamSpec("P")
T = TypeVar("T")
if TYPE_CHECKING:
from sentry_sdk._types import EventProcessor
from sentry_sdk.integrations import Integration
logger = logging.getLogger(__name__)
# Sentry SDK's default behavior:
# - Captures INFO+ level logs as breadcrumbs
# - Captures ERROR+ level logs as Sentry events
#
# For custom log levels, use the LoggingIntegration class:
# sentry_sdk.init(..., integrations=[LoggingIntegration(level=logging.INFO, event_level=logging.ERROR)])
def initialize_sentry(
*,
integrations: Sequence[Integration],
before_send: EventProcessor | None = None,
) -> None:
"""Initialize Sentry SDK with project settings.
Args:
integrations: Sentry SDK integrations to enable (e.g., Starlette, FastAPI).
before_send: Optional event filter callback to suppress specific exceptions.
"""
sentry_sdk.init(
dsn=settings.SENTRY.DSN,
enable_tracing=True,
release=settings.SENTRY.RELEASE,
environment=settings.SENTRY.ENVIRONMENT,
traces_sample_rate=settings.SENTRY.TRACES_SAMPLE_RATE,
profiles_sample_rate=settings.SENTRY.PROFILES_SAMPLE_RATE,
before_send=before_send,
integrations=integrations,
)
def with_sentry_transaction(
name: str, op: str
) -> Callable[[Callable[P, T]], Callable[P, T]]:
"""Decorator to wrap a function in a Sentry transaction.
Args:
name: The name of the transaction.
op: The operation type (e.g., "deriver", "http.server").
Returns:
A decorator that wraps the target function in a Sentry transaction.
Example:
@with_sentry_transaction("process_task", op="deriver")
def process_task(data):
...
"""
def decorator(func: Callable[P, T]) -> Callable[P, T]:
if asyncio.iscoroutinefunction(func):
@wraps(func)
async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
with sentry_sdk.start_transaction(name=name, op=op):
return await func(*args, **kwargs)
return cast(Callable[P, T], async_wrapper)
else:
@wraps(func)
def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
with sentry_sdk.start_transaction(name=name, op=op):
return func(*args, **kwargs)
return sync_wrapper
return decorator

86
src/utils/agent_tools.py
View File

@ -3,7 +3,7 @@ import logging
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from typing import Any, cast
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@ -13,13 +13,20 @@ from src.config import settings
from src.embedding_client import embedding_client
from src.models import Document
from src.schemas import ResolvedConfiguration
from src.telemetry.events import (
AgentToolConclusionsCreatedEvent,
AgentToolConclusionsDeletedEvent,
AgentToolPeerCardUpdatedEvent,
emit,
)
from src.utils import summarizer
from src.utils.formatting import format_new_turn_with_timestamp, utc_now_iso
from src.utils.representation import Representation
from src.utils.types import DocumentLevel
from src.utils.types import DocumentLevel, get_current_iteration
logger = logging.getLogger(__name__)
# Module-level lock registry for thread-safe observation creation.
# Keyed by (workspace_name, observer, observed) to ensure all tool executors
# operating on the same data share the same lock.
@ -887,6 +894,10 @@ class ToolContext:
db_lock: asyncio.Lock
# Optional resolved configuration for checking feature flags
configuration: ResolvedConfiguration | None = None
# Telemetry context fields
run_id: str | None = None
agent_type: str | None = None # "dialectic", "deriver", "dreamer"
parent_category: str | None = None # Parent category for CloudEvents
async def _handle_create_observations(
@ -996,6 +1007,23 @@ async def _handle_create_observations(
contradiction_count = sum(
1 for o in observations if o.get("level") == "contradiction"
)
# Emit telemetry event if context is available
if ctx.run_id and ctx.agent_type and ctx.parent_category:
emit(
AgentToolConclusionsCreatedEvent(
run_id=ctx.run_id,
iteration=get_current_iteration(),
parent_category=ctx.parent_category,
agent_type=ctx.agent_type,
workspace_name=ctx.workspace_name,
observer=ctx.observer,
observed=ctx.observed,
conclusion_count=len(observations),
levels=[o.get("level", "explicit") for o in observations],
)
)
return f"Created {len(observations)} observations for {ctx.observed} by {ctx.observer} ({explicit_count} explicit, {deductive_count} deductive, {inductive_count} inductive, {contradiction_count} contradiction)"
@ -1010,17 +1038,44 @@ async def _handle_update_peer_card(ctx: ToolContext, tool_input: dict[str, Any])
"Peer card creation is disabled for this workspace/session configuration."
)
peer_card_content = tool_input["content"]
async with ctx.db_lock:
await crud.set_peer_card(
ctx.db,
workspace_name=ctx.workspace_name,
peer_card=tool_input["content"],
peer_card=peer_card_content,
observer=ctx.observer,
observed=ctx.observed,
)
logger.info(
f"Updated peer card for {ctx.workspace_name}/{ctx.observer}/{ctx.observed}"
)
# Emit telemetry event if context is available
if ctx.run_id and ctx.agent_type and ctx.parent_category:
# Count facts in peer card (content is a list of strings per tool schema)
facts_count: int
if isinstance(peer_card_content, list):
content_list = cast(list[str], peer_card_content)
facts_count = len([line for line in content_list if line.strip()])
else:
# Fallback for string (defensive)
facts_count = len(
[line for line in str(peer_card_content).split("\n") if line.strip()]
)
emit(
AgentToolPeerCardUpdatedEvent(
run_id=ctx.run_id,
iteration=get_current_iteration(),
parent_category=ctx.parent_category,
agent_type=ctx.agent_type,
workspace_name=ctx.workspace_name,
observer=ctx.observer,
observed=ctx.observed,
facts_count=facts_count,
)
)
return f"Updated peer card for {ctx.observed} by {ctx.observer}"
@ -1370,6 +1425,22 @@ async def _handle_delete_observations(
deleted_count += 1
except Exception as e:
logger.warning(f"Failed to delete observation {obs_id}: {e}")
# Emit telemetry event if context is available
if deleted_count > 0 and ctx.run_id and ctx.agent_type and ctx.parent_category:
emit(
AgentToolConclusionsDeletedEvent(
run_id=ctx.run_id,
iteration=get_current_iteration(),
parent_category=ctx.parent_category,
agent_type=ctx.agent_type,
workspace_name=ctx.workspace_name,
observer=ctx.observer,
observed=ctx.observed,
conclusion_count=deleted_count,
)
)
return f"Deleted {deleted_count} observations"
@ -1557,6 +1628,9 @@ async def create_tool_executor(
include_observation_ids: bool = False,
history_token_limit: int = 8192,
configuration: ResolvedConfiguration | None = None,
run_id: str | None = None,
agent_type: str | None = None,
parent_category: str | None = None,
) -> Callable[[str, dict[str, Any]], Any]:
"""
Create a unified tool executor function for all agent operations.
@ -1574,6 +1648,9 @@ async def create_tool_executor(
include_observation_ids: If True, include observation IDs in output (for dreamer agent)
history_token_limit: Maximum tokens for get_recent_history (default: 8192)
configuration: Resolved configuration for checking feature flags (optional)
run_id: Optional run ID for telemetry correlation
agent_type: Optional agent type for telemetry (dialectic, deriver, dreamer)
parent_category: Optional parent category for CloudEvents
Returns:
An async callable that executes tools with the captured context
@ -1593,6 +1670,9 @@ async def create_tool_executor(
history_token_limit=history_token_limit,
db_lock=shared_lock,
configuration=configuration,
run_id=run_id,
agent_type=agent_type,
parent_category=parent_category,
)
async def execute_tool(tool_name: str, tool_input: dict[str, Any]) -> str:

64
src/utils/clients.py
View File

@ -2,6 +2,7 @@ import json
import logging
from collections.abc import AsyncIterator, Callable
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Any, Generic, Literal, TypeVar, cast, overload
from anthropic import AsyncAnthropic
@ -22,15 +23,37 @@ from sentry_sdk.ai.monitoring import ai_track
from tenacity import retry, stop_after_attempt, wait_exponential
from src.config import LLMComponentSettings, settings
from src.telemetry.logging import conditional_observe
from src.telemetry.reasoning_traces import log_reasoning_trace
from src.utils.json_parser import validate_and_repair_json
from src.utils.logging import conditional_observe
from src.utils.reasoning_traces import log_reasoning_trace
from src.utils.representation import PromptRepresentation
from src.utils.tokens import estimate_tokens
from src.utils.types import SupportedProviders
from src.utils.types import SupportedProviders, set_current_iteration
logger = logging.getLogger(__name__)
@dataclass
class IterationData:
"""Data passed to iteration callbacks after each tool execution loop iteration."""
iteration: int
"""1-indexed iteration number."""
tool_calls: list[str]
"""List of tool names called in this iteration."""
input_tokens: int
"""Input tokens used in this iteration's LLM call."""
output_tokens: int
"""Output tokens generated in this iteration's LLM call."""
cache_read_tokens: int = 0
"""Tokens read from cache in this iteration."""
cache_creation_tokens: int = 0
"""Tokens written to cache in this iteration."""
# Type alias for iteration callback
IterationCallback = Callable[[IterationData], None]
T = TypeVar("T")
# Type aliases for OpenAI GPT-5 specific parameters
@ -467,6 +490,8 @@ class HonchoLLMCallResponse(BaseModel, Generic[T]):
cache_read_input_tokens: int = 0
finish_reasons: list[str]
tool_calls_made: list[dict[str, Any]] = Field(default_factory=list)
iterations: int = 0
"""Number of LLM calls made in the tool execution loop (1 = single response, 2+ = tool use iterations plus final synthesis)."""
thinking_content: str | None = None
# Full thinking blocks with signatures for multi-turn conversation replay (Anthropic only)
thinking_blocks: list[dict[str, Any]] = Field(default_factory=list)
@ -506,6 +531,7 @@ class StreamingResponseWithMetadata:
cache_creation_input_tokens: int
cache_read_input_tokens: int
thinking_content: str | None
iterations: int
def __init__(
self,
@ -516,6 +542,7 @@ class StreamingResponseWithMetadata:
cache_creation_input_tokens: int,
cache_read_input_tokens: int,
thinking_content: str | None = None,
iterations: int = 0,
):
self._stream = stream
self.tool_calls_made = tool_calls_made
@ -524,6 +551,7 @@ class StreamingResponseWithMetadata:
self.cache_creation_input_tokens = cache_creation_input_tokens
self.cache_read_input_tokens = cache_read_input_tokens
self.thinking_content = thinking_content
self.iterations = iterations
def __aiter__(self) -> AsyncIterator[HonchoLLMCallStreamChunk]:
return self._stream.__aiter__()
@ -628,6 +656,7 @@ async def _execute_tool_loop(
],
before_retry_callback: Callable[[Any], None],
stream_final: bool = False,
iteration_callback: IterationCallback | None = None,
) -> HonchoLLMCallResponse[Any] | StreamingResponseWithMetadata:
"""
Execute the tool calling loop for agentic LLM interactions.
@ -660,6 +689,7 @@ async def _execute_tool_loop(
get_provider_and_model: Function to get current provider/model based on attempt
before_retry_callback: Callback for retry events
stream_final: If True, stream the final response instead of returning it synchronously
iteration_callback: Optional callback invoked after each iteration with IterationData
Returns:
Final HonchoLLMCallResponse with accumulated token counts and tool call history,
@ -771,6 +801,7 @@ async def _execute_tool_loop(
cache_creation_input_tokens=total_cache_creation_tokens,
cache_read_input_tokens=total_cache_read_tokens,
thinking_content=response.thinking_content,
iterations=iteration + 1,
)
response.tool_calls_made = all_tool_calls
@ -778,6 +809,7 @@ async def _execute_tool_loop(
response.output_tokens = total_output_tokens
response.cache_creation_input_tokens = total_cache_creation_tokens
response.cache_read_input_tokens = total_cache_read_tokens
response.iterations = iteration + 1
return response
# Determine which provider we're using (reuse the helper)
@ -793,6 +825,9 @@ async def _execute_tool_loop(
)
conversation_messages.append(assistant_message)
# Set current iteration for telemetry context (1-indexed)
set_current_iteration(iteration + 1)
# Execute tools and add results
tool_results: list[dict[str, Any]] = []
for tool_call in response.tool_calls_made:
@ -837,6 +872,21 @@ async def _execute_tool_loop(
# Add tool result message in provider-specific format
_append_tool_results(current_provider, tool_results, conversation_messages)
# Call iteration callback if provided
if iteration_callback is not None:
try:
iteration_data = IterationData(
iteration=iteration + 1, # 1-indexed
tool_calls=[tc["name"] for tc in response.tool_calls_made],
input_tokens=response.input_tokens,
output_tokens=response.output_tokens,
cache_read_tokens=response.cache_read_input_tokens or 0,
cache_creation_tokens=response.cache_creation_input_tokens or 0,
)
iteration_callback(iteration_data)
except Exception:
logger.warning("iteration_callback failed", exc_info=True)
# After first iteration, switch from "required" to "auto" to allow model to stop
if iteration == 0 and effective_tool_choice in ("required", "any"):
effective_tool_choice = "auto"
@ -884,6 +934,7 @@ async def _execute_tool_loop(
cache_creation_input_tokens=total_cache_creation_tokens,
cache_read_input_tokens=total_cache_read_tokens,
thinking_content=None, # No thinking content at max iterations
iterations=iteration + 1, # +1 for the synthesis call
)
# Make one final call to get a text response
@ -927,6 +978,7 @@ async def _execute_tool_loop(
final_response = await final_call_func()
final_response.tool_calls_made = all_tool_calls
final_response.iterations = iteration + 1 # +1 for the synthesis call
# Include accumulated tokens from all iterations plus the final call
final_response.input_tokens = total_input_tokens + final_response.input_tokens
final_response.output_tokens = total_output_tokens + final_response.output_tokens
@ -1126,6 +1178,7 @@ async def honcho_llm_call(
messages: list[dict[str, Any]] | None = None,
max_input_tokens: int | None = None,
trace_name: str | None = None,
iteration_callback: IterationCallback | None = None,
) -> HonchoLLMCallResponse[M]: ...
@ -1154,6 +1207,7 @@ async def honcho_llm_call(
messages: list[dict[str, Any]] | None = None,
max_input_tokens: int | None = None,
trace_name: str | None = None,
iteration_callback: IterationCallback | None = None,
) -> HonchoLLMCallResponse[str]: ...
@ -1182,6 +1236,7 @@ async def honcho_llm_call(
messages: list[dict[str, Any]] | None = None,
max_input_tokens: int | None = None,
trace_name: str | None = None,
iteration_callback: IterationCallback | None = None,
) -> AsyncIterator[HonchoLLMCallStreamChunk] | StreamingResponseWithMetadata: ...
@ -1210,6 +1265,7 @@ async def honcho_llm_call(
messages: list[dict[str, Any]] | None = None,
max_input_tokens: int | None = None,
trace_name: str | None = None,
iteration_callback: IterationCallback | None = None,
) -> (
HonchoLLMCallResponse[Any]
| AsyncIterator[HonchoLLMCallStreamChunk]
@ -1241,6 +1297,7 @@ async def honcho_llm_call(
tool_executor: Async callable to execute tools, receives (tool_name, tool_input)
max_tool_iterations: Maximum number of tool execution loops
messages: Optional message list for multi-turn conversations (overrides prompt)
iteration_callback: Optional callback invoked after each tool iteration with IterationData
Returns:
HonchoLLMCallResponse or AsyncIterator depending on stream parameter
@ -1455,6 +1512,7 @@ async def honcho_llm_call(
get_provider_and_model=_get_provider_and_model,
before_retry_callback=before_retry_callback,
stream_final=stream_final_only,
iteration_callback=iteration_callback,
)
if trace_name and isinstance(result, HonchoLLMCallResponse):
log_reasoning_trace(

69
src/utils/summarizer.py
View File

@ -9,16 +9,19 @@ from typing import TypedDict
from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncSession
from src import prometheus, schemas
from src import schemas
from src.cache.client import cache as cache_client
from src.config import settings
from src.crud.session import session_cache_key
from src.dependencies import tracked_db
from src.exceptions import ResourceNotFoundException
from src.models import Message
from src.telemetry import otel_metrics
from src.telemetry.events import AgentToolSummaryCreatedEvent, emit
from src.telemetry.logging import accumulate_metric, conditional_observe
from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes
from src.utils.clients import HonchoLLMCallResponse, honcho_llm_call
from src.utils.formatting import utc_now_iso
from src.utils.logging import accumulate_metric, conditional_observe
from src.utils.tokens import estimate_tokens, track_deriver_input_tokens
from .. import crud, models
@ -412,7 +415,12 @@ async def _create_and_save_summary(
previous_summary_tokens = latest_summary["token_count"] if latest_summary else 0
input_tokens = messages_tokens + previous_summary_tokens
new_summary, is_fallback = await _create_summary(
(
new_summary,
is_fallback,
llm_input_tokens,
llm_output_tokens,
) = await _create_summary(
formatted_messages=formatted_messages,
previous_summary_text=previous_summary_text,
summary_type=summary_type,
@ -432,20 +440,22 @@ async def _create_and_save_summary(
prompt_tokens = estimate_long_summary_prompt_tokens()
track_deriver_input_tokens(
task_type=prometheus.DeriverTaskTypes.SUMMARY,
task_type=DeriverTaskTypes.SUMMARY,
components={
prometheus.DeriverComponents.PROMPT: prompt_tokens,
prometheus.DeriverComponents.MESSAGES: messages_tokens,
prometheus.DeriverComponents.PREVIOUS_SUMMARY: previous_summary_tokens,
DeriverComponents.PROMPT: prompt_tokens,
DeriverComponents.MESSAGES: messages_tokens,
DeriverComponents.PREVIOUS_SUMMARY: previous_summary_tokens,
},
)
# Track output tokens
prometheus.DERIVER_TOKENS_PROCESSED.labels(
task_type=prometheus.DeriverTaskTypes.SUMMARY.value,
token_type=prometheus.TokenTypes.OUTPUT.value,
component=prometheus.DeriverComponents.OUTPUT_TOTAL.value,
).inc(new_summary["token_count"])
if settings.OTEL.ENABLED:
otel_metrics.record_deriver_tokens(
count=new_summary["token_count"],
task_type=DeriverTaskTypes.SUMMARY.value,
token_type=TokenTypes.OUTPUT.value,
component=DeriverComponents.OUTPUT_TOTAL.value,
)
# Save summary to database with new transaction
async with tracked_db("summary.save") as db:
@ -477,6 +487,27 @@ async def _create_and_save_summary(
"ms",
)
# Emit telemetry event (only for non-fallback summaries)
# Note: Using AgentToolSummaryCreatedEvent with dummy run_id/iteration since
# this is called from the deriver, not from an agentic loop
if not is_fallback:
emit(
AgentToolSummaryCreatedEvent(
run_id="deriver", # Placeholder - not from an agentic run
iteration=0, # Placeholder - not from an agentic loop
parent_category="deriver",
agent_type="summarizer",
workspace_name=workspace_name,
session_name=session_name,
message_id=message_public_id,
message_count=len(messages),
message_seq_in_session=message_seq_in_session,
summary_type="short" if summary_type == SummaryType.SHORT else "long",
input_tokens=llm_input_tokens,
output_tokens=llm_output_tokens,
)
)
async def _create_summary(
formatted_messages: str,
@ -487,7 +518,7 @@ async def _create_summary(
last_message_id: int,
last_message_content_preview: str,
message_count: int,
) -> tuple[Summary, bool]:
) -> tuple[Summary, bool, int, int]:
"""
Generate a summary of the provided messages using an LLM.
@ -502,12 +533,16 @@ async def _create_summary(
message_count: Number of messages for fallback
Returns:
A tuple of (Summary, is_fallback) where is_fallback indicates if
the summary was generated using a fallback instead of an LLM call
A tuple of (Summary, is_fallback, llm_input_tokens, llm_output_tokens)
where is_fallback indicates if the summary was generated using a
fallback instead of an LLM call, and the token counts are from the LLM call
(0 if fallback was used)
"""
response: HonchoLLMCallResponse[str] | None = None
is_fallback = False
llm_input_tokens = 0
llm_output_tokens = 0
try:
if summary_type == SummaryType.SHORT:
response = await create_short_summary(
@ -520,6 +555,8 @@ async def _create_summary(
summary_text = response.content
summary_tokens = response.output_tokens
llm_input_tokens = response.input_tokens
llm_output_tokens = response.output_tokens
# Detect potential issues with the summary
if not summary_text.strip():
@ -547,6 +584,8 @@ async def _create_summary(
message_public_id=message_public_id,
),
is_fallback,
llm_input_tokens,
llm_output_tokens,
)

21
src/utils/tokens.py
View File

@ -1,6 +1,8 @@
import tiktoken
from src import prometheus
from src.config import settings
from src.telemetry import otel_metrics
from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes
tokenizer = tiktoken.get_encoding("o200k_base")
@ -18,8 +20,8 @@ def estimate_tokens(text: str | list[str] | None) -> int:
def track_deriver_input_tokens(
task_type: prometheus.DeriverTaskTypes,
components: dict[prometheus.DeriverComponents, int],
task_type: DeriverTaskTypes,
components: dict[DeriverComponents, int],
) -> None:
"""
Helper method to track input token components for a given task type.
@ -29,8 +31,11 @@ def track_deriver_input_tokens(
components: Dict mapping component names to token counts
"""
for component, token_count in components.items():
prometheus.DERIVER_TOKENS_PROCESSED.labels(
task_type=task_type.value,
token_type=prometheus.TokenTypes.INPUT.value,
component=component.value,
).inc(token_count)
# OTel metrics (push-based)
if settings.OTEL.ENABLED:
otel_metrics.record_deriver_tokens(
count=token_count,
task_type=task_type.value,
token_type=TokenTypes.INPUT.value,
component=component.value,
)

48
src/utils/tracing.py
View File

@ -1,48 +0,0 @@
import asyncio
from collections.abc import Callable
from functools import wraps
from typing import ParamSpec, TypeVar, cast
import sentry_sdk
P = ParamSpec("P")
T = TypeVar("T")
def with_sentry_transaction(
name: str, op: str
) -> Callable[[Callable[P, T]], Callable[P, T]]:
"""Decorator to wrap a function in a Sentry transaction.
Args:
name: The name of the transaction.
op: The operation type (e.g., "deriver", "http.server").
Returns:
A decorator that wraps the target function in a Sentry transaction.
Example:
@with_sentry_transaction("process_task", op="deriver")
def process_task(data):
...
"""
def decorator(func: Callable[P, T]) -> Callable[P, T]:
if asyncio.iscoroutinefunction(func):
@wraps(func)
async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
with sentry_sdk.start_transaction(name=name, op=op):
return await func(*args, **kwargs)
return cast(Callable[P, T], async_wrapper)
else:
@wraps(func)
def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
with sentry_sdk.start_transaction(name=name, op=op):
return func(*args, **kwargs)
return sync_wrapper
return decorator

15
src/utils/types.py
View File

@ -1,7 +1,22 @@
from contextvars import ContextVar
from typing import Generic, Literal, NamedTuple, TypeVar
T = TypeVar("T")
# Context variable for tracking current iteration in tool execution loop
# This is used for telemetry to associate tool calls with their iteration
_current_iteration: ContextVar[int] = ContextVar("current_iteration", default=0)
def set_current_iteration(iteration: int) -> None:
"""Set the current iteration number for telemetry context."""
_current_iteration.set(iteration)
def get_current_iteration() -> int:
"""Get the current iteration number from telemetry context."""
return _current_iteration.get()
class GetOrCreateResult(NamedTuple, Generic[T]):
"""Result of a get_or_create operation indicating whether the resource was created."""

2
tests/bench/beam.py
View File

@ -81,7 +81,7 @@ from honcho_core.types.workspaces.sessions.message_create_param import (
from openai import AsyncOpenAI
from src.config import settings
from src.utils.metrics_collector import MetricsCollector
from src.telemetry.metrics_collector import MetricsCollector
from .beam_common import (
ConversationResult,

2
tests/bench/locomo.py
View File

@ -78,7 +78,7 @@ from honcho_core.types.workspaces.sessions.message_create_param import (
from openai import AsyncOpenAI
from src.config import settings
from src.utils.metrics_collector import MetricsCollector
from src.telemetry.metrics_collector import MetricsCollector
from .locomo_common import (
CATEGORY_NAMES,

2
tests/bench/longmem.py
View File

@ -77,7 +77,7 @@ from openai import AsyncOpenAI
from typing_extensions import TypedDict
from src.config import settings
from src.utils.metrics_collector import MetricsCollector
from src.telemetry.metrics_collector import MetricsCollector
from .longmem_common import (
calculate_timing_statistics,

4
tests/bench/run_tests.py
View File

@ -452,12 +452,12 @@ Evaluate whether the actual response contains the core correct information from
if session_name and observed:
response_text = await query_peer.chat(
query,
session_id=session_name,
session=session_name,
target=peers[observed],
)
elif session_name:
response_text = await query_peer.chat(
query, session_id=session_name
query, session=session_name
)
elif observed:
response_text = await query_peer.chat(

2
tests/conftest.py
View File

@ -661,7 +661,7 @@ def mock_tracked_db(db_session: AsyncSession):
patch("src.routers.sessions.tracked_db", mock_tracked_db_context),
patch("src.routers.peers.tracked_db", mock_tracked_db_context),
patch("src.crud.representation.tracked_db", mock_tracked_db_context),
patch("src.dreamer.dreamer.tracked_db", mock_tracked_db_context),
patch("src.dreamer.orchestrator.tracked_db", mock_tracked_db_context),
patch("src.dreamer.dream_scheduler.tracked_db", mock_tracked_db_context),
patch("src.dialectic.chat.tracked_db", mock_tracked_db_context),
patch("src.utils.summarizer.tracked_db", mock_tracked_db_context),

17
tests/crud/test_workspace.py
View File

@ -380,18 +380,25 @@ class TestWorkspaceCRUD:
db_session: AsyncSession,
sample_data: tuple[models.Workspace, models.Peer],
):
"""Test that delete_workspace returns the deleted workspace object"""
"""Test that delete_workspace returns WorkspaceDeletionResult with cascade counts"""
test_workspace, _test_peer = sample_data
# Store workspace details before deletion
workspace_name = test_workspace.name
# Delete workspace
deleted_workspace = await crud.delete_workspace(db_session, test_workspace.name)
result = await crud.delete_workspace(db_session, test_workspace.name)
# Verify returned workspace matches the deleted workspace
assert deleted_workspace.name == workspace_name
assert isinstance(deleted_workspace, schemas.Workspace)
# Verify returned result is WorkspaceDeletionResult with correct workspace
assert isinstance(result, crud.WorkspaceDeletionResult)
assert result.workspace.name == workspace_name
assert isinstance(result.workspace, schemas.Workspace)
# Verify cascade counts are present (should have 1 peer from sample_data)
assert result.peers_deleted == 1
assert result.sessions_deleted >= 0
assert result.messages_deleted >= 0
assert result.conclusions_deleted >= 0
@pytest.mark.asyncio
async def test_delete_workspace_complex_cascade(

3
tests/deriver/test_queue_processing.py
View File

@ -347,6 +347,7 @@ class TestQueueProcessing:
*,
observed: str | None = None, # pyright: ignore[reportUnusedParameter]
observer: str | None = None, # pyright: ignore[reportUnusedParameter]
queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter]
) -> None:
processed_batches.append(
{
@ -909,6 +910,7 @@ class TestQueueProcessing:
*,
observed: str | None = None, # pyright: ignore[reportUnusedParameter]
observer: str | None = None, # pyright: ignore[reportUnusedParameter]
queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter]
) -> None:
processed_batches.append(
{
@ -1027,6 +1029,7 @@ class TestQueueProcessing:
*,
observed: str | None = None, # pyright: ignore[reportUnusedParameter]
observer: str | None = None, # pyright: ignore[reportUnusedParameter]
queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter]
) -> None:
processed_batches.append(
{

1170
tests/integration/test_telemetry.py Normal file
View File

File diff suppressed because it is too large Load Diff

198
tests/integration/test_token_metrics.py
View File

@ -1,20 +1,23 @@
"""Integration tests for prometheus token metrics tracking.
"""Integration tests for OpenTelemetry token metrics tracking.
These tests verify that DERIVER_TOKENS_PROCESSED and DIALECTIC_TOKENS_PROCESSED
metrics are correctly emitted with accurate token counts when processing messages
and dialectic queries.
These tests verify that deriver and dialectic token metrics are correctly
emitted with accurate token counts when processing messages and dialectic queries.
The approach uses delta-based verification:
The approach uses delta-based verification with OTel's InMemoryMetricReader:
1. Capture counter values before test execution
2. Run the code under test (with mocked LLM)
3. Verify deltas match expected values
"""
from collections.abc import Iterator
from typing import Any, cast
from unittest.mock import AsyncMock, patch
import pytest
from nanoid import generate as generate_nanoid
from prometheus_client import REGISTRY
from opentelemetry.metrics import Meter
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from sqlalchemy.ext.asyncio import AsyncSession
from src import crud, models, schemas
@ -26,6 +29,7 @@ from src.schemas import (
ResolvedReasoningConfiguration,
ResolvedSummaryConfiguration,
)
from src.telemetry.otel.metrics import otel_metrics
from src.utils.clients import HonchoLLMCallResponse
from src.utils.representation import ExplicitObservationBase, PromptRepresentation
from src.utils.summarizer import (
@ -39,17 +43,43 @@ from src.utils.summarizer import (
# =============================================================================
class MetricDeltaChecker:
"""Utility class to capture and verify prometheus counter deltas."""
class OTelMetricChecker:
"""Utility class to capture and verify OTel counter deltas."""
_reader: InMemoryMetricReader
def __init__(self, reader: InMemoryMetricReader):
self._reader = reader
def _get_metric_value(self, metric_name: str, labels: dict[str, str]) -> float:
"""Get current value of a counter with specific labels.
Note: The OTel SDK's MetricsData types are not fully typed, so we cast
to Any to avoid type warnings when traversing the metrics data structure.
"""
raw_data = self._reader.get_metrics_data() # pyright: ignore[reportUnknownVariableType]
if raw_data is None:
return 0.0
data = cast(Any, raw_data)
for resource_metrics in data.resource_metrics:
for scope_metrics in resource_metrics.scope_metrics:
for metric in scope_metrics.metrics:
if metric.name == metric_name and hasattr(
metric.data, "data_points"
):
for point in metric.data.data_points:
# Check if labels match
point_attrs: dict[str, str] = (
dict(point.attributes) if point.attributes else {}
)
if all(point_attrs.get(k) == v for k, v in labels.items()):
return float(point.value)
return 0.0
def capture(self, metric_name: str, labels: dict[str, str]) -> float:
"""Capture current value of a counter with specific labels."""
# Counters have _total suffix in prometheus
full_name = (
metric_name if metric_name.endswith("_total") else f"{metric_name}_total"
)
value = REGISTRY.get_sample_value(full_name, labels=labels)
return value or 0.0
return self._get_metric_value(metric_name, labels)
def get_delta(
self, metric_name: str, labels: dict[str, str], before: float
@ -72,18 +102,78 @@ class MetricDeltaChecker:
), f"{message}: expected delta {expected}, got {delta}. Labels: {labels}"
@pytest.fixture
def metric_checker() -> MetricDeltaChecker:
"""Fixture providing a metric delta checker instance."""
return MetricDeltaChecker()
def _reset_otel_metrics_singleton() -> None:
"""Reset the otel_metrics singleton instance so it reinitializes with a new provider.
This clears all instance attributes so _ensure_initialized() will create
new meters tied to the current global MeterProvider.
"""
# Reset initialization flag (instance attribute shadows class attribute)
otel_metrics._is_initialized = False # pyright: ignore[reportPrivateUsage]
# Reset all meters
otel_metrics._api_meter = None # pyright: ignore[reportPrivateUsage]
otel_metrics._deriver_meter = None # pyright: ignore[reportPrivateUsage]
otel_metrics._dialectic_meter = None # pyright: ignore[reportPrivateUsage]
otel_metrics._dreamer_meter = None # pyright: ignore[reportPrivateUsage]
# Reset all counters
otel_metrics._api_requests = None # pyright: ignore[reportPrivateUsage]
otel_metrics._messages_created = None # pyright: ignore[reportPrivateUsage]
otel_metrics._dialectic_calls = None # pyright: ignore[reportPrivateUsage]
otel_metrics._deriver_queue_items = None # pyright: ignore[reportPrivateUsage]
otel_metrics._deriver_tokens = None # pyright: ignore[reportPrivateUsage]
otel_metrics._dialectic_tokens = None # pyright: ignore[reportPrivateUsage]
otel_metrics._dreamer_tokens = None # pyright: ignore[reportPrivateUsage]
@pytest.fixture
def enable_metrics(monkeypatch: pytest.MonkeyPatch):
"""Enable prometheus metrics with a test namespace."""
monkeypatch.setattr("src.prometheus.METRICS_ENABLED", True)
monkeypatch.setattr("src.config.settings.METRICS.NAMESPACE", "test")
yield
def otel_test_setup(
monkeypatch: pytest.MonkeyPatch,
) -> Iterator[tuple[InMemoryMetricReader, OTelMetricChecker]]:
"""Set up OTel metrics with in-memory reader for testing.
The OTel SDK only allows set_meter_provider() to be called once per process.
To work around this for testing, we patch get_meter() to return meters
from our test provider directly.
Yields:
Tuple of (reader, checker) for verifying metrics
"""
# Create in-memory reader
reader = InMemoryMetricReader()
# Create a test meter provider
provider = MeterProvider(metric_readers=[reader])
# Patch get_meter to return meters from our test provider
# This is necessary because set_meter_provider() can only be called once per process
def test_get_meter(name: str, version: str = "") -> Meter:
return provider.get_meter(name, version)
monkeypatch.setattr("src.telemetry.otel.metrics.get_meter", test_get_meter)
# Reset the otel_metrics singleton instance so it reinitializes with our test provider
_reset_otel_metrics_singleton()
# Enable OTEL in settings and set namespace for test assertions
monkeypatch.setattr("src.config.settings.OTEL.ENABLED", True)
monkeypatch.setattr("src.config.settings.OTEL.SERVICE_NAMESPACE", "test")
checker = OTelMetricChecker(reader)
yield reader, checker
# Cleanup: reset singleton so other tests aren't affected
_reset_otel_metrics_singleton()
@pytest.fixture
def metric_checker(
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
) -> OTelMetricChecker:
"""Fixture providing a metric checker instance."""
return otel_test_setup[1]
# =============================================================================
@ -188,7 +278,6 @@ def create_mock_dialectic_response(
@pytest.mark.asyncio
@pytest.mark.usefixtures("enable_metrics")
class TestDeriverIngestionMetrics:
"""Test token metrics for deriver INGESTION task type."""
@ -196,11 +285,12 @@ class TestDeriverIngestionMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify OUTPUT_TOTAL tokens match response.output_tokens from LLM."""
from src.deriver.deriver import process_representation_tasks_batch
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
messages = await create_test_messages(
@ -237,6 +327,7 @@ class TestDeriverIngestionMetrics:
message_level_configuration=create_test_configuration(),
observer=peer.name,
observed=peer.name,
queue_items_count=len(messages),
)
# Verify output tokens metric
@ -252,12 +343,13 @@ class TestDeriverIngestionMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify PROMPT component is tracked for ingestion input."""
from src.deriver.deriver import process_representation_tasks_batch
from src.deriver.prompts import estimate_minimal_deriver_prompt_tokens
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
messages = await create_test_messages(
@ -292,6 +384,7 @@ class TestDeriverIngestionMetrics:
message_level_configuration=create_test_configuration(),
observer=peer.name,
observed=peer.name,
queue_items_count=len(messages),
)
metric_checker.assert_delta(
@ -306,11 +399,12 @@ class TestDeriverIngestionMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify MESSAGES component is tracked for ingestion input."""
from src.deriver.deriver import process_representation_tasks_batch
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
messages = await create_test_messages(
@ -347,6 +441,7 @@ class TestDeriverIngestionMetrics:
message_level_configuration=create_test_configuration(),
observer=peer.name,
observed=peer.name,
queue_items_count=len(messages),
)
# Verify messages tokens were tracked (should be > 0)
@ -360,7 +455,6 @@ class TestDeriverIngestionMetrics:
@pytest.mark.asyncio
@pytest.mark.usefixtures("enable_metrics")
class TestDeriverSummaryMetrics:
"""Test token metrics for deriver SUMMARY task type."""
@ -368,10 +462,11 @@ class TestDeriverSummaryMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify OUTPUT_TOTAL tokens are tracked for summary."""
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
@ -401,7 +496,9 @@ class TestDeriverSummaryMetrics:
with (
patch(
"src.utils.summarizer._create_summary",
new=AsyncMock(return_value=(mock_summary, False)), # is_fallback=False
new=AsyncMock(
return_value=(mock_summary, False, 100, expected_output_tokens)
), # (summary, is_fallback, llm_input_tokens, llm_output_tokens)
),
patch(
"src.utils.summarizer._save_summary",
@ -431,10 +528,11 @@ class TestDeriverSummaryMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify PROMPT component is tracked for summary input."""
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
messages = await create_test_messages(
@ -462,7 +560,9 @@ class TestDeriverSummaryMetrics:
with (
patch(
"src.utils.summarizer._create_summary",
new=AsyncMock(return_value=(mock_summary, False)),
new=AsyncMock(
return_value=(mock_summary, False, 100, 10)
), # (summary, is_fallback, llm_input_tokens, llm_output_tokens)
),
patch(
"src.utils.summarizer._save_summary",
@ -491,10 +591,11 @@ class TestDeriverSummaryMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify MESSAGES component is tracked for summary input."""
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
messages = await create_test_messages(
@ -531,7 +632,9 @@ class TestDeriverSummaryMetrics:
with (
patch(
"src.utils.summarizer._create_summary",
new=AsyncMock(return_value=(mock_summary, False)),
new=AsyncMock(
return_value=(mock_summary, False, 100, 10)
), # (summary, is_fallback, llm_input_tokens, llm_output_tokens)
),
patch(
"src.utils.summarizer._save_summary",
@ -559,10 +662,11 @@ class TestDeriverSummaryMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify metrics are NOT emitted when _create_summary returns is_fallback=True."""
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
messages = await create_test_messages(
@ -599,7 +703,9 @@ class TestDeriverSummaryMetrics:
with patch(
"src.utils.summarizer._create_summary",
new=AsyncMock(return_value=(mock_summary, True)), # is_fallback=True
new=AsyncMock(
return_value=(mock_summary, True, 0, 0)
), # (summary, is_fallback=True, llm_input_tokens, llm_output_tokens)
):
await _create_and_save_summary(
workspace_name=workspace.name,
@ -633,7 +739,6 @@ class TestDeriverSummaryMetrics:
@pytest.mark.asyncio
@pytest.mark.usefixtures("enable_metrics")
class TestDialecticTokenMetrics:
"""Test token metrics for dialectic calls."""
@ -641,11 +746,12 @@ class TestDialecticTokenMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify INPUT tokens are tracked from LLM response."""
from src.dialectic.core import DialecticAgent
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
@ -688,11 +794,12 @@ class TestDialecticTokenMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
):
"""Verify OUTPUT tokens are tracked from LLM response."""
from src.dialectic.core import DialecticAgent
_, metric_checker = otel_test_setup
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
@ -735,15 +842,16 @@ class TestDialecticTokenMetrics:
self,
db_session: AsyncSession,
sample_data: tuple[Workspace, Peer],
metric_checker: MetricDeltaChecker,
otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker],
monkeypatch: pytest.MonkeyPatch,
):
"""Verify metrics are NOT emitted when METRICS_ENABLED=False."""
"""Verify metrics are NOT emitted when OTEL.ENABLED=False."""
from src.dialectic.core import DialecticAgent
# Explicitly disable metrics
monkeypatch.setattr("src.prometheus.METRICS_ENABLED", False)
monkeypatch.setattr("src.config.settings.METRICS.NAMESPACE", "test")
_, metric_checker = otel_test_setup
# Explicitly disable OTEL metrics
monkeypatch.setattr("src.config.settings.OTEL.ENABLED", False)
workspace, peer = sample_data
session = await create_test_session_with_peer(db_session, workspace, peer)
@ -757,11 +865,13 @@ class TestDialecticTokenMetrics:
"namespace": "test",
"token_type": "input",
"component": "total",
"reasoning_level": "low",
}
output_labels = {
"namespace": "test",
"token_type": "output",
"component": "total",
"reasoning_level": "low",
}
before_input = metric_checker.capture(
"dialectic_tokens_processed", input_labels

1
tests/telemetry/__init__.py Normal file
View File

@ -0,0 +1 @@
"""Unit tests for the telemetry module."""

458
tests/telemetry/conftest.py Normal file
View File

@ -0,0 +1,458 @@
# pyright: reportUnknownMemberType=false
"""Fixtures for telemetry unit tests.
This module provides:
- Sample event fixtures for all 12 event types
- Mock settings fixtures for controlling telemetry configuration
- Mock HTTP client fixtures for testing the emitter without network calls
"""
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.telemetry.events.agent import (
AgentIterationEvent,
AgentToolConclusionsCreatedEvent,
AgentToolConclusionsDeletedEvent,
AgentToolPeerCardUpdatedEvent,
AgentToolSummaryCreatedEvent,
)
from src.telemetry.events.base import BaseEvent
from src.telemetry.events.deletion import DeletionCompletedEvent
from src.telemetry.events.dialectic import DialecticCompletedEvent
from src.telemetry.events.dream import DreamRunEvent, DreamSpecialistEvent
from src.telemetry.events.reconciliation import (
CleanupStaleItemsCompletedEvent,
SyncVectorsCompletedEvent,
)
from src.telemetry.events.representation import RepresentationCompletedEvent
# =============================================================================
# Fixed timestamp for deterministic tests
# =============================================================================
@pytest.fixture
def fixed_timestamp() -> datetime:
"""Return a fixed timestamp for deterministic event ID generation."""
return datetime(2024, 1, 15, 12, 0, 0, tzinfo=UTC)
# =============================================================================
# Sample event fixtures for all event types
# =============================================================================
@pytest.fixture
def sample_representation_event(
fixed_timestamp: datetime,
) -> RepresentationCompletedEvent:
"""Create a sample RepresentationCompletedEvent for testing."""
return RepresentationCompletedEvent(
timestamp=fixed_timestamp,
workspace_name="test_workspace",
session_name="test_session",
observed="user_peer",
queue_items_processed=3,
earliest_message_id="msg_001",
latest_message_id="msg_010",
message_count=10,
explicit_conclusion_count=5,
context_preparation_ms=50.0,
llm_call_ms=1200.0,
total_duration_ms=1300.0,
input_tokens=5000,
output_tokens=500,
)
@pytest.fixture
def sample_dream_run_event(fixed_timestamp: datetime) -> DreamRunEvent:
"""Create a sample DreamRunEvent for testing."""
return DreamRunEvent(
timestamp=fixed_timestamp,
run_id="abc12345",
workspace_name="test_workspace",
session_name="test_session",
observer="assistant",
observed="user_peer",
specialists_run=["deduction", "induction"],
deduction_success=True,
induction_success=True,
surprisal_enabled=False,
surprisal_conclusion_count=0,
total_iterations=15,
total_input_tokens=25000,
total_output_tokens=3000,
total_duration_ms=45000.0,
)
@pytest.fixture
def sample_dream_specialist_event(fixed_timestamp: datetime) -> DreamSpecialistEvent:
"""Create a sample DreamSpecialistEvent for testing."""
return DreamSpecialistEvent(
timestamp=fixed_timestamp,
run_id="abc12345",
specialist_type="deduction",
workspace_name="test_workspace",
observer="assistant",
observed="user_peer",
iterations=8,
tool_calls_count=12,
input_tokens=15000,
output_tokens=2000,
duration_ms=25000.0,
success=True,
)
@pytest.fixture
def sample_dialectic_event(fixed_timestamp: datetime) -> DialecticCompletedEvent:
"""Create a sample DialecticCompletedEvent for testing."""
return DialecticCompletedEvent(
timestamp=fixed_timestamp,
run_id="def67890",
workspace_name="test_workspace",
peer_name="user_peer",
session_name="test_session",
reasoning_level="medium",
total_iterations=3,
prefetched_conclusion_count=10,
tool_calls_count=5,
total_duration_ms=3500.0,
input_tokens=8000,
output_tokens=1200,
cache_read_tokens=500,
cache_creation_tokens=200,
)
@pytest.fixture
def sample_agent_iteration_event(fixed_timestamp: datetime) -> AgentIterationEvent:
"""Create a sample AgentIterationEvent for testing."""
return AgentIterationEvent(
timestamp=fixed_timestamp,
run_id="abc12345",
parent_category="dream",
agent_type="deduction",
workspace_name="test_workspace",
observer="assistant",
observed="user_peer",
iteration=3,
tool_calls=["search_memory", "create_observations"],
input_tokens=3000,
output_tokens=500,
cache_read_tokens=100,
cache_creation_tokens=50,
)
@pytest.fixture
def sample_conclusions_created_event(
fixed_timestamp: datetime,
) -> AgentToolConclusionsCreatedEvent:
"""Create a sample AgentToolConclusionsCreatedEvent for testing."""
return AgentToolConclusionsCreatedEvent(
timestamp=fixed_timestamp,
run_id="abc12345",
iteration=3,
parent_category="dream",
agent_type="deduction",
workspace_name="test_workspace",
observer="assistant",
observed="user_peer",
conclusion_count=5,
levels=["explicit", "deductive", "deductive", "explicit", "deductive"],
)
@pytest.fixture
def sample_conclusions_deleted_event(
fixed_timestamp: datetime,
) -> AgentToolConclusionsDeletedEvent:
"""Create a sample AgentToolConclusionsDeletedEvent for testing."""
return AgentToolConclusionsDeletedEvent(
timestamp=fixed_timestamp,
run_id="abc12345",
iteration=5,
parent_category="dream",
agent_type="deduction",
workspace_name="test_workspace",
observer="assistant",
observed="user_peer",
conclusion_count=3,
)
@pytest.fixture
def sample_peer_card_updated_event(
fixed_timestamp: datetime,
) -> AgentToolPeerCardUpdatedEvent:
"""Create a sample AgentToolPeerCardUpdatedEvent for testing."""
return AgentToolPeerCardUpdatedEvent(
timestamp=fixed_timestamp,
run_id="abc12345",
iteration=7,
parent_category="dream",
agent_type="induction",
workspace_name="test_workspace",
observer="assistant",
observed="user_peer",
facts_count=12,
)
@pytest.fixture
def sample_summary_created_event(
fixed_timestamp: datetime,
) -> AgentToolSummaryCreatedEvent:
"""Create a sample AgentToolSummaryCreatedEvent for testing."""
return AgentToolSummaryCreatedEvent(
timestamp=fixed_timestamp,
run_id="ghi11111",
iteration=1,
parent_category="representation",
agent_type="summarizer",
workspace_name="test_workspace",
session_name="test_session",
message_id="msg_020",
message_count=20,
message_seq_in_session=20,
summary_type="short",
input_tokens=4000,
output_tokens=300,
)
@pytest.fixture
def sample_deletion_event(fixed_timestamp: datetime) -> DeletionCompletedEvent:
"""Create a sample DeletionCompletedEvent for testing."""
return DeletionCompletedEvent(
timestamp=fixed_timestamp,
workspace_name="test_workspace",
deletion_type="workspace",
resource_id="ws_123abc",
success=True,
peers_deleted=5,
sessions_deleted=10,
messages_deleted=500,
conclusions_deleted=200,
)
@pytest.fixture
def sample_sync_vectors_event(fixed_timestamp: datetime) -> SyncVectorsCompletedEvent:
"""Create a sample SyncVectorsCompletedEvent for testing."""
return SyncVectorsCompletedEvent(
timestamp=fixed_timestamp,
documents_synced=150,
documents_failed=2,
message_embeddings_synced=500,
message_embeddings_failed=0,
total_duration_ms=12000.0,
)
@pytest.fixture
def sample_cleanup_event(fixed_timestamp: datetime) -> CleanupStaleItemsCompletedEvent:
"""Create a sample CleanupStaleItemsCompletedEvent for testing."""
return CleanupStaleItemsCompletedEvent(
timestamp=fixed_timestamp,
documents_cleaned=25,
queue_items_cleaned=100,
total_duration_ms=5000.0,
)
# =============================================================================
# All sample events as a collection
# =============================================================================
@pytest.fixture
def all_sample_events(
sample_representation_event: RepresentationCompletedEvent,
sample_dream_run_event: DreamRunEvent,
sample_dream_specialist_event: DreamSpecialistEvent,
sample_dialectic_event: DialecticCompletedEvent,
sample_agent_iteration_event: AgentIterationEvent,
sample_conclusions_created_event: AgentToolConclusionsCreatedEvent,
sample_conclusions_deleted_event: AgentToolConclusionsDeletedEvent,
sample_peer_card_updated_event: AgentToolPeerCardUpdatedEvent,
sample_summary_created_event: AgentToolSummaryCreatedEvent,
sample_deletion_event: DeletionCompletedEvent,
sample_sync_vectors_event: SyncVectorsCompletedEvent,
sample_cleanup_event: CleanupStaleItemsCompletedEvent,
) -> list[BaseEvent]:
"""Return all sample events as a list for parametrized tests."""
return [
sample_representation_event,
sample_dream_run_event,
sample_dream_specialist_event,
sample_dialectic_event,
sample_agent_iteration_event,
sample_conclusions_created_event,
sample_conclusions_deleted_event,
sample_peer_card_updated_event,
sample_summary_created_event,
sample_deletion_event,
sample_sync_vectors_event,
sample_cleanup_event,
]
# =============================================================================
# Mock settings fixtures
# =============================================================================
@pytest.fixture
def mock_telemetry_settings():
"""Fixture to mock telemetry settings.
Returns a context manager that patches settings with configurable values.
"""
def _configure(
enabled: bool = True,
endpoint: str = "http://test-endpoint:8001/v1/events",
namespace: str = "test_namespace",
batch_size: int = 100,
flush_interval: float = 1.0,
flush_threshold: int = 50,
max_retries: int = 3,
max_buffer_size: int = 10000,
headers: dict[str, str] | None = None,
):
mock_settings = MagicMock()
mock_settings.TELEMETRY.ENABLED = enabled
mock_settings.TELEMETRY.ENDPOINT = endpoint
mock_settings.TELEMETRY.NAMESPACE = namespace
mock_settings.TELEMETRY.BATCH_SIZE = batch_size
mock_settings.TELEMETRY.FLUSH_INTERVAL_SECONDS = flush_interval
mock_settings.TELEMETRY.FLUSH_THRESHOLD = flush_threshold
mock_settings.TELEMETRY.MAX_RETRIES = max_retries
mock_settings.TELEMETRY.MAX_BUFFER_SIZE = max_buffer_size
mock_settings.TELEMETRY.HEADERS = headers
mock_settings.OTEL.ENABLED = False
return patch("src.telemetry.emitter.settings", mock_settings)
return _configure
# =============================================================================
# Mock HTTP client fixtures
# =============================================================================
@pytest.fixture
def mock_httpx_client():
"""Create a mock httpx.AsyncClient for testing HTTP calls.
Returns a factory function that creates configured mocks.
"""
def _create(
status_code: int = 200,
raise_exception: Exception | None = None,
response_sequence: list[tuple[int, Exception | None]] | None = None,
):
"""Create a mock HTTP client.
Args:
status_code: Default status code to return
raise_exception: Exception to raise on post()
response_sequence: List of (status_code, exception) for sequential calls
"""
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = status_code
mock_response.raise_for_status = MagicMock()
if raise_exception:
mock_client.post = AsyncMock(side_effect=raise_exception)
elif response_sequence:
# Create side effects for sequential responses
side_effects = []
for code, exc in response_sequence:
if exc:
side_effects.append(exc)
else:
resp = MagicMock()
resp.status_code = code
if code >= 400:
import httpx
resp.raise_for_status = MagicMock(
side_effect=httpx.HTTPStatusError(
f"HTTP {code}",
request=MagicMock(),
response=resp,
)
)
else:
resp.raise_for_status = MagicMock()
side_effects.append(resp)
mock_client.post = AsyncMock(side_effect=side_effects)
else:
if status_code >= 400:
import httpx
mock_response.raise_for_status = MagicMock(
side_effect=httpx.HTTPStatusError(
f"HTTP {status_code}",
request=MagicMock(),
response=mock_response,
)
)
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.aclose = AsyncMock()
return mock_client
return _create
# =============================================================================
# Emitter fixtures
# =============================================================================
@pytest.fixture
def disabled_emitter():
"""Create a disabled TelemetryEmitter for testing disabled state."""
from src.telemetry.emitter import TelemetryEmitter
return TelemetryEmitter(
endpoint=None,
enabled=False,
)
@pytest.fixture
def emitter_with_small_buffer():
"""Create an emitter with a small buffer for testing overflow."""
from src.telemetry.emitter import TelemetryEmitter
return TelemetryEmitter(
endpoint="http://test:8001/events",
max_buffer_size=5,
flush_threshold=3,
enabled=True,
)
@pytest.fixture
def emitter_with_low_threshold():
"""Create an emitter with low flush threshold for testing auto-flush."""
from src.telemetry.emitter import TelemetryEmitter
return TelemetryEmitter(
endpoint="http://test:8001/events",
flush_threshold=2,
batch_size=10,
enabled=True,
)

470
tests/telemetry/test_emit_function.py Normal file
View File

@ -0,0 +1,470 @@
# pyright: reportPrivateUsage=false
"""Unit tests for emit() function and initialization lifecycle.
Tests for:
- emit() function behavior with/without initialized emitter
- initialize_telemetry_events() configuration handling
- shutdown_telemetry_events() cleanup
- initialize_telemetry() and initialize_telemetry_async() orchestration
"""
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.telemetry.events.representation import RepresentationCompletedEvent
def create_test_event() -> RepresentationCompletedEvent:
"""Create a minimal test event."""
return RepresentationCompletedEvent(
timestamp=datetime(2024, 1, 15, 12, 0, 0, tzinfo=UTC),
workspace_name="test_workspace",
session_name="test_session",
observed="user_peer",
queue_items_processed=1,
earliest_message_id="msg_001",
latest_message_id="msg_001",
message_count=1,
explicit_conclusion_count=1,
context_preparation_ms=10.0,
llm_call_ms=100.0,
total_duration_ms=110.0,
input_tokens=100,
output_tokens=50,
)
# =============================================================================
# Tests for emit() function
# =============================================================================
class TestEmitFunction:
"""Tests for the emit() function in src.telemetry.events."""
def test_emit_when_emitter_is_none(self):
"""emit() silently returns when emitter is not initialized."""
from src.telemetry.events import emit
# Patch where get_emitter is imported from (inside emit function)
with patch("src.telemetry.emitter.get_emitter", return_value=None):
event = create_test_event()
# Should not raise, just silently return
emit(event)
def test_emit_calls_emitter_emit(self):
"""emit() calls the emitter's emit method when initialized."""
from src.telemetry.events import emit
mock_emitter = MagicMock()
with patch("src.telemetry.emitter.get_emitter", return_value=mock_emitter):
event = create_test_event()
emit(event)
mock_emitter.emit.assert_called_once_with(event)
def test_emit_passes_event_unchanged(self):
"""emit() passes the event object unchanged to emitter."""
from src.telemetry.events import emit
mock_emitter = MagicMock()
with patch("src.telemetry.emitter.get_emitter", return_value=mock_emitter):
event = create_test_event()
emit(event)
# Get the event that was passed to emit
passed_event = mock_emitter.emit.call_args[0][0]
assert passed_event is event
# =============================================================================
# Tests for get_emitter() function
# =============================================================================
class TestGetEmitter:
"""Tests for the get_emitter() function."""
def test_get_emitter_returns_none_when_not_initialized(self):
"""get_emitter() returns None before initialization."""
from src.telemetry import emitter as emitter_module
# Save original value
original = emitter_module._emitter
try:
emitter_module._emitter = None
result = emitter_module.get_emitter()
assert result is None
finally:
# Restore original
emitter_module._emitter = original
def test_get_emitter_returns_emitter_when_initialized(self):
"""get_emitter() returns the emitter after initialization."""
from src.telemetry import emitter as emitter_module
from src.telemetry.emitter import TelemetryEmitter
# Save original value
original = emitter_module._emitter
try:
test_emitter = TelemetryEmitter(endpoint="http://test:8001/events")
emitter_module._emitter = test_emitter
result = emitter_module.get_emitter()
assert result is test_emitter
finally:
# Restore original
emitter_module._emitter = original
# =============================================================================
# Tests for initialize_emitter() function
# =============================================================================
class TestInitializeEmitter:
"""Tests for the initialize_emitter() function."""
@pytest.mark.asyncio
async def test_initialize_emitter_creates_and_starts(self):
"""initialize_emitter() creates emitter and starts it."""
from src.telemetry import emitter as emitter_module
from src.telemetry.emitter import initialize_emitter
# Save original value
original = emitter_module._emitter
# Create a properly async mock client
mock_client = AsyncMock()
mock_client.aclose = AsyncMock()
try:
emitter_module._emitter = None
with patch("httpx.AsyncClient", return_value=mock_client):
result = await initialize_emitter(
endpoint="http://test:8001/events",
enabled=True,
)
assert result is not None
assert result.endpoint == "http://test:8001/events"
assert result.is_running is True
# Should be stored in global
assert emitter_module._emitter is result
# Clean up
await result.shutdown()
finally:
emitter_module._emitter = original
@pytest.mark.asyncio
async def test_initialize_emitter_with_custom_settings(self):
"""initialize_emitter() accepts custom configuration."""
from src.telemetry import emitter as emitter_module
from src.telemetry.emitter import initialize_emitter
original = emitter_module._emitter
# Create a properly async mock client
mock_client = AsyncMock()
mock_client.aclose = AsyncMock()
try:
emitter_module._emitter = None
with patch("httpx.AsyncClient", return_value=mock_client):
result = await initialize_emitter(
endpoint="http://custom:9000/events",
headers={"X-API-Key": "secret"},
batch_size=50,
flush_interval_seconds=2.0,
flush_threshold=25,
max_retries=5,
max_buffer_size=5000,
enabled=True,
)
assert result.endpoint == "http://custom:9000/events"
assert result.headers == {"X-API-Key": "secret"}
assert result.batch_size == 50
assert result.flush_interval == 2.0
assert result.flush_threshold == 25
assert result.max_retries == 5
assert result.max_buffer_size == 5000
await result.shutdown()
finally:
emitter_module._emitter = original
@pytest.mark.asyncio
async def test_initialize_emitter_disabled(self):
"""initialize_emitter() creates disabled emitter when enabled=False."""
from src.telemetry import emitter as emitter_module
from src.telemetry.emitter import initialize_emitter
original = emitter_module._emitter
try:
emitter_module._emitter = None
result = await initialize_emitter(
endpoint="http://test:8001/events",
enabled=False,
)
assert result is not None
assert result.enabled is False
assert result.is_running is False
await result.shutdown()
finally:
emitter_module._emitter = original
# =============================================================================
# Tests for shutdown_emitter() function
# =============================================================================
class TestShutdownEmitter:
"""Tests for the shutdown_emitter() function."""
@pytest.mark.asyncio
async def test_shutdown_emitter_when_none(self):
"""shutdown_emitter() handles None emitter gracefully."""
from src.telemetry import emitter as emitter_module
from src.telemetry.emitter import shutdown_emitter
original = emitter_module._emitter
try:
emitter_module._emitter = None
# Should not raise
await shutdown_emitter()
finally:
emitter_module._emitter = original
@pytest.mark.asyncio
async def test_shutdown_emitter_calls_shutdown(self):
"""shutdown_emitter() calls shutdown on the emitter."""
from src.telemetry import emitter as emitter_module
from src.telemetry.emitter import shutdown_emitter
original = emitter_module._emitter
try:
mock_emitter = AsyncMock()
emitter_module._emitter = mock_emitter
await shutdown_emitter()
mock_emitter.shutdown.assert_called_once()
# Should clear global
assert emitter_module._emitter is None
finally:
emitter_module._emitter = original
# =============================================================================
# Tests for initialize_telemetry_events() function
# =============================================================================
class TestInitializeTelemetryEvents:
"""Tests for initialize_telemetry_events() in src.telemetry.events."""
@pytest.mark.asyncio
async def test_initialize_when_disabled(self):
"""initialize_telemetry_events() is no-op when TELEMETRY.ENABLED=False."""
from src.telemetry.events import initialize_telemetry_events
with (
patch("src.config.settings") as mock_settings,
patch("src.telemetry.emitter.initialize_emitter") as mock_init,
):
mock_settings.TELEMETRY.ENABLED = False
await initialize_telemetry_events()
mock_init.assert_not_called()
@pytest.mark.asyncio
async def test_initialize_when_enabled(self):
"""initialize_telemetry_events() initializes emitter when enabled."""
from src.telemetry.events import initialize_telemetry_events
with (
patch("src.config.settings") as mock_settings,
patch("src.telemetry.emitter.initialize_emitter") as mock_init,
):
mock_settings.TELEMETRY.ENABLED = True
mock_settings.TELEMETRY.ENDPOINT = "http://test:8001/events"
mock_settings.TELEMETRY.HEADERS = {"X-API-Key": "secret"}
mock_settings.TELEMETRY.BATCH_SIZE = 100
mock_settings.TELEMETRY.FLUSH_INTERVAL_SECONDS = 1.0
mock_settings.TELEMETRY.FLUSH_THRESHOLD = 50
mock_settings.TELEMETRY.MAX_RETRIES = 3
mock_settings.TELEMETRY.MAX_BUFFER_SIZE = 10000
mock_init.return_value = AsyncMock()
await initialize_telemetry_events()
mock_init.assert_called_once_with(
endpoint="http://test:8001/events",
headers={"X-API-Key": "secret"},
batch_size=100,
flush_interval_seconds=1.0,
flush_threshold=50,
max_retries=3,
max_buffer_size=10000,
enabled=True,
)
# =============================================================================
# Tests for shutdown_telemetry_events() function
# =============================================================================
class TestShutdownTelemetryEvents:
"""Tests for shutdown_telemetry_events() in src.telemetry.events."""
@pytest.mark.asyncio
async def test_shutdown_calls_shutdown_emitter(self):
"""shutdown_telemetry_events() calls shutdown_emitter."""
from src.telemetry.events import shutdown_telemetry_events
with patch(
"src.telemetry.emitter.shutdown_emitter", new_callable=AsyncMock
) as mock_shutdown:
await shutdown_telemetry_events()
mock_shutdown.assert_called_once()
# =============================================================================
# Tests for initialize_telemetry() function
# =============================================================================
class TestInitializeTelemetry:
"""Tests for initialize_telemetry() in src.telemetry."""
def test_initialize_otel_when_enabled(self):
"""initialize_telemetry() initializes OTel metrics when enabled."""
import src.telemetry as telemetry_module
with (
patch("src.config.settings") as mock_settings,
# Patch where it's used (in src.telemetry), not where it's defined
patch.object(telemetry_module, "initialize_otel_metrics") as mock_otel_init,
):
mock_settings.OTEL.ENABLED = True
mock_settings.OTEL.ENDPOINT = "http://otel:9009/metrics"
mock_settings.OTEL.HEADERS = None
mock_settings.OTEL.EXPORT_INTERVAL_MILLIS = 60000
mock_settings.OTEL.SERVICE_NAME = "honcho"
mock_settings.OTEL.SERVICE_NAMESPACE = "test"
telemetry_module.initialize_telemetry()
mock_otel_init.assert_called_once_with(
endpoint="http://otel:9009/metrics",
headers=None,
export_interval_millis=60000,
service_name="honcho",
service_namespace="test",
enabled=True,
)
def test_skip_otel_when_disabled(self):
"""initialize_telemetry() skips OTel when disabled."""
import src.telemetry as telemetry_module
with (
patch("src.config.settings") as mock_settings,
patch.object(telemetry_module, "initialize_otel_metrics") as mock_otel_init,
):
mock_settings.OTEL.ENABLED = False
telemetry_module.initialize_telemetry()
mock_otel_init.assert_not_called()
# =============================================================================
# Tests for initialize_telemetry_async() function
# =============================================================================
class TestInitializeTelemetryAsync:
"""Tests for initialize_telemetry_async() in src.telemetry."""
@pytest.mark.asyncio
async def test_initialize_cloudevents_when_enabled(self):
"""initialize_telemetry_async() initializes CloudEvents when enabled."""
from src.telemetry import initialize_telemetry_async
with (
patch("src.config.settings") as mock_settings,
patch(
"src.telemetry.events.initialize_telemetry_events",
new_callable=AsyncMock,
) as mock_ce_init,
):
mock_settings.TELEMETRY.ENABLED = True
await initialize_telemetry_async()
mock_ce_init.assert_called_once()
@pytest.mark.asyncio
async def test_skip_cloudevents_when_disabled(self):
"""initialize_telemetry_async() skips CloudEvents when disabled."""
from src.telemetry import initialize_telemetry_async
with (
patch("src.config.settings") as mock_settings,
patch(
"src.telemetry.events.initialize_telemetry_events",
new_callable=AsyncMock,
) as mock_ce_init,
):
mock_settings.TELEMETRY.ENABLED = False
await initialize_telemetry_async()
mock_ce_init.assert_not_called()
# =============================================================================
# Tests for shutdown_telemetry() function
# =============================================================================
class TestShutdownTelemetry:
"""Tests for shutdown_telemetry() in src.telemetry."""
@pytest.mark.asyncio
async def test_shutdown_calls_all_subsystems(self):
"""shutdown_telemetry() shuts down all telemetry subsystems."""
from src.telemetry import shutdown_telemetry
with (
patch(
"src.telemetry.events.shutdown_telemetry_events", new_callable=AsyncMock
) as mock_ce_shutdown,
patch("src.telemetry.shutdown_otel_metrics") as mock_otel_shutdown,
):
await shutdown_telemetry()
mock_ce_shutdown.assert_called_once()
mock_otel_shutdown.assert_called_once()

982
tests/telemetry/test_emitter.py Normal file
View File

@ -0,0 +1,982 @@
# pyright: reportPrivateUsage=false, reportUnknownParameterType=false, reportMissingParameterType=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnusedParameter=false
"""Unit tests for TelemetryEmitter class.
Exhaustive tests covering:
- Initialization and configuration
- Disabled state behavior
- Buffer management (capacity, overflow, threshold)
- Flush mechanics (manual, threshold-triggered, periodic)
- HTTP sending (success, retries, batch formats)
- Error scenarios (network, HTTP status codes, timeouts)
- Lifecycle (start, shutdown, concurrent operations)
"""
import asyncio
import contextlib
import json
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from src.telemetry.emitter import TelemetryEmitter
from src.telemetry.events.representation import RepresentationCompletedEvent
# =============================================================================
# Helper to create minimal test events
# =============================================================================
def create_test_event(message_id: str = "msg_001") -> RepresentationCompletedEvent:
"""Create a minimal test event."""
return RepresentationCompletedEvent(
timestamp=datetime(2024, 1, 15, 12, 0, 0, tzinfo=UTC),
workspace_name="test_workspace",
session_name="test_session",
observed="user_peer",
queue_items_processed=1,
earliest_message_id="msg_001",
latest_message_id=message_id,
message_count=1,
explicit_conclusion_count=1,
context_preparation_ms=10.0,
llm_call_ms=100.0,
total_duration_ms=110.0,
input_tokens=100,
output_tokens=50,
)
# =============================================================================
# Tests for Initialization
# =============================================================================
class TestEmitterInitialization:
"""Tests for TelemetryEmitter initialization."""
def test_default_initialization(self):
"""Emitter initializes with default values."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
assert emitter.endpoint == "http://test:8001/events"
assert emitter.batch_size == 100
assert emitter.flush_interval == 1.0
assert emitter.flush_threshold == 50
assert emitter.max_retries == 3
assert emitter.max_buffer_size == 10000
assert emitter.enabled is True
def test_custom_initialization(self):
"""Emitter initializes with custom values."""
emitter = TelemetryEmitter(
endpoint="http://custom:9000/events",
headers={"X-API-Key": "secret"},
batch_size=50,
flush_interval_seconds=2.5,
flush_threshold=25,
max_retries=5,
max_buffer_size=5000,
enabled=True,
)
assert emitter.endpoint == "http://custom:9000/events"
assert emitter.headers == {"X-API-Key": "secret"}
assert emitter.batch_size == 50
assert emitter.flush_interval == 2.5
assert emitter.flush_threshold == 25
assert emitter.max_retries == 5
assert emitter.max_buffer_size == 5000
def test_disabled_when_no_endpoint(self):
"""Emitter is disabled when endpoint is None."""
emitter = TelemetryEmitter(endpoint=None, enabled=True)
assert emitter.enabled is False
def test_disabled_when_explicitly_disabled(self):
"""Emitter is disabled when enabled=False."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events", enabled=False)
assert emitter.enabled is False
def test_initial_state(self):
"""Emitter starts in non-running state with empty buffer."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
assert emitter.buffer_size == 0
assert emitter.is_running is False
assert emitter._client is None
assert emitter._flush_task is None
# =============================================================================
# Tests for Disabled State
# =============================================================================
class TestDisabledEmitter:
"""Tests for emitter behavior when disabled."""
@pytest.mark.asyncio
async def test_start_when_disabled(self, disabled_emitter: TelemetryEmitter):
"""start() is no-op when disabled."""
await disabled_emitter.start()
assert disabled_emitter.is_running is False
assert disabled_emitter._client is None
@pytest.mark.asyncio
async def test_emit_when_disabled(self, disabled_emitter: TelemetryEmitter):
"""emit() is no-op when disabled."""
event = create_test_event()
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
disabled_emitter.emit(event)
assert disabled_emitter.buffer_size == 0
@pytest.mark.asyncio
async def test_flush_when_disabled(self, disabled_emitter: TelemetryEmitter):
"""flush() is no-op when disabled."""
await disabled_emitter.flush()
# Should complete without error
@pytest.mark.asyncio
async def test_shutdown_when_disabled(self, disabled_emitter: TelemetryEmitter):
"""shutdown() is no-op when disabled."""
await disabled_emitter.shutdown()
# Should complete without error
# =============================================================================
# Tests for Buffer Management
# =============================================================================
class TestBufferManagement:
"""Tests for buffer capacity and overflow handling."""
def test_emit_adds_to_buffer(self):
"""emit() adds events to the buffer."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
emitter._running = True # Simulate started state without full start()
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
assert emitter.buffer_size == 1
def test_buffer_respects_max_size(self):
"""Buffer drops oldest events when max size exceeded."""
# Create an emitter with small buffer but high flush threshold
# to prevent auto-flush from interfering with this synchronous test
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_buffer_size=5,
flush_threshold=100, # High threshold to prevent auto-flush
enabled=True,
)
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
# Add more events than buffer can hold
for i in range(10):
event = create_test_event(f"msg_{i:03d}")
emitter.emit(event)
# Buffer should be at max size (5) - deque drops oldest events
assert emitter.buffer_size == 5
def test_buffer_warns_at_80_percent_capacity(self, caplog):
"""Warning logged when buffer reaches 80% capacity."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_buffer_size=10,
)
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
# Add 8 events (80% of 10)
for i in range(8):
event = create_test_event(f"msg_{i:03d}")
emitter.emit(event)
# Check warning was logged
assert any(
"80%" in record.message or "capacity" in record.message.lower()
for record in caplog.records
if record.levelname == "WARNING"
)
@pytest.mark.asyncio
async def test_flush_threshold_triggers_flush(self):
"""Flush is triggered when buffer reaches threshold."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
flush_threshold=3,
max_buffer_size=100,
)
# Mock the HTTP client
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.raise_for_status = MagicMock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
# Add events up to threshold
for i in range(3):
event = create_test_event(f"msg_{i:03d}")
emitter.emit(event)
# Give async task time to run
await asyncio.sleep(0.1)
# Flush should have been triggered
assert mock_client.post.called
# =============================================================================
# Tests for HTTP Sending
# =============================================================================
class TestHttpSending:
"""Tests for HTTP request formatting and sending."""
@pytest.mark.asyncio
async def test_single_event_uses_structured_format(self):
"""Single event is sent as structured CloudEvent JSON."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.raise_for_status = MagicMock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
# Check that post was called
assert mock_client.post.called
call_args = mock_client.post.call_args
# Single event should NOT have batch header override
headers = call_args.kwargs.get("headers")
assert headers is None # No override for single event
@pytest.mark.asyncio
async def test_multiple_events_use_batch_format(self):
"""Multiple events are sent as CloudEvents batch (JSON array)."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
batch_size=10,
)
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.raise_for_status = MagicMock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
# Add multiple events
for i in range(3):
event = create_test_event(f"msg_{i:03d}")
emitter.emit(event)
await emitter.flush()
# Check that post was called with batch header
call_args = mock_client.post.call_args
headers = call_args.kwargs.get("headers")
assert headers == {"Content-Type": "application/cloudevents-batch+json"}
# Content should be a JSON array
content = call_args.kwargs.get("content")
parsed = json.loads(content)
assert isinstance(parsed, list)
assert len(parsed) == 3
@pytest.mark.asyncio
async def test_batch_respects_batch_size(self):
"""Large buffers are sent in batch_size chunks."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
batch_size=2, # Small batch size
)
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.raise_for_status = MagicMock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
# Add 5 events
for i in range(5):
event = create_test_event(f"msg_{i:03d}")
emitter.emit(event)
await emitter.flush()
# Should have made 3 calls (2, 2, 1)
assert mock_client.post.call_count == 3
# =============================================================================
# Tests for Retry Logic
# =============================================================================
class TestRetryLogic:
"""Tests for exponential backoff retry behavior."""
@pytest.mark.asyncio
async def test_retry_on_http_error(self):
"""Emitter retries on HTTP errors."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_retries=3,
)
# First two calls fail, third succeeds
mock_client = AsyncMock()
fail_response = MagicMock()
fail_response.status_code = 500
fail_response.raise_for_status = MagicMock(
side_effect=httpx.HTTPStatusError(
"500", request=MagicMock(), response=fail_response
)
)
success_response = MagicMock()
success_response.status_code = 200
success_response.raise_for_status = MagicMock()
mock_client.post = AsyncMock(
side_effect=[fail_response, fail_response, success_response]
)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", new_callable=AsyncMock), # Skip actual delays
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
# Should have tried 3 times
assert mock_client.post.call_count == 3
# Buffer should be empty (success on 3rd try)
assert emitter.buffer_size == 0
@pytest.mark.asyncio
async def test_retry_on_request_error(self):
"""Emitter retries on network/request errors."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_retries=3,
)
mock_client = AsyncMock()
success_response = MagicMock()
success_response.status_code = 200
success_response.raise_for_status = MagicMock()
# First call raises connection error, second succeeds
mock_client.post = AsyncMock(
side_effect=[
httpx.ConnectError("Connection refused"),
success_response,
]
)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", new_callable=AsyncMock),
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
assert mock_client.post.call_count == 2
@pytest.mark.asyncio
async def test_all_retries_exhausted(self):
"""Events returned to buffer when all retries fail."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_retries=3,
)
mock_client = AsyncMock()
fail_response = MagicMock()
fail_response.status_code = 500
fail_response.raise_for_status = MagicMock(
side_effect=httpx.HTTPStatusError(
"500", request=MagicMock(), response=fail_response
)
)
mock_client.post = AsyncMock(return_value=fail_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", new_callable=AsyncMock),
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
# All 3 retries should have been attempted
assert mock_client.post.call_count == 3
# Event should be back in buffer
assert emitter.buffer_size == 1
@pytest.mark.asyncio
async def test_exponential_backoff_timing(self):
"""Retry uses exponential backoff delays."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_retries=4,
)
mock_client = AsyncMock()
fail_response = MagicMock()
fail_response.status_code = 503
fail_response.raise_for_status = MagicMock(
side_effect=httpx.HTTPStatusError(
"503", request=MagicMock(), response=fail_response
)
)
mock_client.post = AsyncMock(return_value=fail_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
sleep_calls = []
async def track_sleep(delay):
sleep_calls.append(delay)
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", side_effect=track_sleep),
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
# Should have delays of 1, 2, 4 (2^0, 2^1, 2^2) before attempts 2, 3, 4
assert sleep_calls == [1, 2, 4]
# =============================================================================
# Tests for HTTP Error Scenarios
# =============================================================================
class TestHttpErrorScenarios:
"""Tests for various HTTP error status codes."""
@pytest.mark.asyncio
@pytest.mark.parametrize(
"status_code,should_retry",
[
(400, True), # Bad Request - still retries
(401, True), # Unauthorized
(403, True), # Forbidden
(404, True), # Not Found
(429, True), # Too Many Requests
(500, True), # Internal Server Error
(502, True), # Bad Gateway
(503, True), # Service Unavailable
(504, True), # Gateway Timeout
],
)
async def test_http_status_codes(self, status_code: int, should_retry: bool):
"""Various HTTP status codes trigger retry behavior."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_retries=2,
)
mock_client = AsyncMock()
fail_response = MagicMock()
fail_response.status_code = status_code
fail_response.raise_for_status = MagicMock(
side_effect=httpx.HTTPStatusError(
f"HTTP {status_code}", request=MagicMock(), response=fail_response
)
)
mock_client.post = AsyncMock(return_value=fail_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", new_callable=AsyncMock),
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
if should_retry:
assert mock_client.post.call_count == 2 # max_retries
# =============================================================================
# Tests for Network Error Scenarios
# =============================================================================
class TestNetworkErrorScenarios:
"""Tests for various network error conditions."""
@pytest.mark.asyncio
async def test_connection_refused(self):
"""Connection refused error triggers retry."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events", max_retries=2)
mock_client = AsyncMock()
mock_client.post = AsyncMock(
side_effect=httpx.ConnectError("Connection refused")
)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", new_callable=AsyncMock),
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
assert mock_client.post.call_count == 2
@pytest.mark.asyncio
async def test_connection_timeout(self):
"""Connection timeout error triggers retry."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events", max_retries=2)
mock_client = AsyncMock()
mock_client.post = AsyncMock(
side_effect=httpx.ConnectTimeout("Connection timed out")
)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", new_callable=AsyncMock),
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
assert mock_client.post.call_count == 2
@pytest.mark.asyncio
async def test_read_timeout(self):
"""Read timeout error triggers retry."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events", max_retries=2)
mock_client = AsyncMock()
mock_client.post = AsyncMock(side_effect=httpx.ReadTimeout("Read timed out"))
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with (
patch("src.config.settings") as mock_settings,
patch("asyncio.sleep", new_callable=AsyncMock),
):
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
assert mock_client.post.call_count == 2
# =============================================================================
# Tests for Lifecycle Management
# =============================================================================
class TestLifecycle:
"""Tests for emitter lifecycle (start, shutdown)."""
@pytest.mark.asyncio
async def test_start_creates_client_and_task(self):
"""start() creates HTTP client and periodic flush task."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
with patch("httpx.AsyncClient") as mock_client_class:
mock_client = AsyncMock()
mock_client_class.return_value = mock_client
await emitter.start()
assert emitter.is_running is True
assert emitter._client is not None
assert emitter._flush_task is not None
# Clean up
emitter._running = False
emitter._flush_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await emitter._flush_task
@pytest.mark.asyncio
async def test_shutdown_flushes_and_closes(self):
"""shutdown() flushes buffer and closes client."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.raise_for_status = MagicMock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
emitter._flush_task = asyncio.create_task(asyncio.sleep(100))
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
assert emitter.buffer_size == 1
await emitter.shutdown()
# Client should be closed
assert mock_client.aclose.called
assert emitter._client is None
assert emitter.is_running is False
@pytest.mark.asyncio
async def test_shutdown_with_empty_buffer(self):
"""shutdown() handles empty buffer gracefully."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
mock_client = AsyncMock()
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
emitter._flush_task = asyncio.create_task(asyncio.sleep(100))
await emitter.shutdown()
assert mock_client.aclose.called
# post should not have been called (nothing to flush)
assert not mock_client.post.called
@pytest.mark.asyncio
async def test_double_shutdown_is_safe(self, disabled_emitter: TelemetryEmitter):
"""Multiple shutdown calls are idempotent."""
await disabled_emitter.shutdown()
await disabled_emitter.shutdown()
# Should complete without error
@pytest.mark.asyncio
async def test_emit_after_shutdown(self):
"""Emit after shutdown is a no-op (disabled)."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
mock_client = AsyncMock()
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
emitter._flush_task = asyncio.create_task(asyncio.sleep(100))
await emitter.shutdown()
# Emit after shutdown
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
# Should not be added (emitter no longer running)
# Note: the emit checks self.enabled, not self._running
# After shutdown, enabled is still True but _running is False
# =============================================================================
# Tests for Concurrent Operations
# =============================================================================
class TestConcurrentOperations:
"""Tests for concurrent emit and flush operations."""
@pytest.mark.asyncio
async def test_concurrent_emits(self):
"""Multiple concurrent emits are thread-safe."""
emitter = TelemetryEmitter(
endpoint="http://test:8001/events",
max_buffer_size=1000,
flush_threshold=1000, # High threshold to prevent auto-flush
)
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
# Emit many events concurrently
for i in range(100):
event = create_test_event(f"msg_{i:03d}")
emitter.emit(event)
assert emitter.buffer_size == 100
@pytest.mark.asyncio
async def test_flush_uses_lock(self):
"""Concurrent flush calls are serialized via lock."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.raise_for_status = MagicMock()
# Slow post to simulate long-running flush
async def slow_post(*args, **kwargs):
await asyncio.sleep(0.1)
return mock_response
mock_client.post = slow_post
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
for i in range(5):
event = create_test_event(f"msg_{i:03d}")
emitter.emit(event)
# Start two concurrent flushes
await asyncio.gather(emitter.flush(), emitter.flush())
# Both should complete without error
# Buffer should be empty
assert emitter.buffer_size == 0
# =============================================================================
# Tests for Edge Cases
# =============================================================================
class TestEdgeCases:
"""Tests for edge cases and boundary conditions."""
@pytest.mark.asyncio
async def test_flush_empty_buffer(self):
"""Flushing empty buffer is a no-op."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
mock_client = AsyncMock()
emitter._client = mock_client
emitter._running = True
await emitter.flush()
# post should not have been called
assert not mock_client.post.called
@pytest.mark.asyncio
async def test_flush_without_client(self):
"""Flush without client (not started) is a no-op."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
event = create_test_event()
emitter.emit(event)
# Flush before start (no client)
await emitter.flush()
# Should complete without error, buffer unchanged
assert emitter.buffer_size == 1
def test_buffer_size_property(self):
"""buffer_size property returns current buffer length."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
emitter._running = True
assert emitter.buffer_size == 0
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test"
emitter.emit(create_test_event("msg_001"))
assert emitter.buffer_size == 1
emitter.emit(create_test_event("msg_002"))
assert emitter.buffer_size == 2
def test_is_running_property(self):
"""is_running property reflects internal state."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
assert emitter.is_running is False
emitter._running = True
assert emitter.is_running is True
emitter._running = False
assert emitter.is_running is False
# =============================================================================
# Tests for CloudEvent Format
# =============================================================================
class TestCloudEventFormat:
"""Tests for correct CloudEvent formatting."""
@pytest.mark.asyncio
async def test_cloudevent_attributes(self):
"""CloudEvent has correct attributes."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
captured_content = None
async def capture_post(url, content=None, headers=None):
nonlocal captured_content
captured_content = content
response = MagicMock()
response.status_code = 200
response.raise_for_status = MagicMock()
return response
mock_client = AsyncMock()
mock_client.post = capture_post
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = "test_ns"
event = create_test_event()
emitter.emit(event)
await emitter.flush()
# Parse the captured content
assert captured_content is not None
cloud_event = json.loads(captured_content)
# Check CloudEvent required attributes
assert "id" in cloud_event
assert cloud_event["id"].startswith("evt_")
assert "source" in cloud_event
assert cloud_event["source"] == "/honcho/test_ns/representation"
assert "type" in cloud_event
assert cloud_event["type"] == "representation.completed"
assert "time" in cloud_event
assert "dataschema" in cloud_event
assert "data" in cloud_event
@pytest.mark.asyncio
async def test_cloudevent_source_without_namespace(self):
"""CloudEvent source format when namespace is None."""
emitter = TelemetryEmitter(endpoint="http://test:8001/events")
captured_content = None
async def capture_post(url, content=None, headers=None):
nonlocal captured_content
captured_content = content
response = MagicMock()
response.status_code = 200
response.raise_for_status = MagicMock()
return response
mock_client = AsyncMock()
mock_client.post = capture_post
mock_client.aclose = AsyncMock()
emitter._client = mock_client
emitter._running = True
with patch("src.config.settings") as mock_settings:
mock_settings.TELEMETRY.NAMESPACE = None
event = create_test_event()
emitter.emit(event)
await emitter.flush()
assert captured_content is not None
cloud_event = json.loads(captured_content)
# Without namespace, source should be /honcho/{category}
assert cloud_event["source"] == "/honcho/representation"

724
tests/telemetry/test_events.py Normal file
View File

@ -0,0 +1,724 @@
# pyright: reportUnknownParameterType=false, reportMissingParameterType=false, reportUnusedParameter=false
"""Unit tests for telemetry event classes.
Tests all 12 event types for:
- Correct instantiation with required fields
- event_type(), schema_version(), category() class methods
- get_resource_id() returns expected format
- generate_id() is deterministic (same inputs = same ID)
- Timestamp defaults to UTC now
"""
from datetime import UTC, datetime
import pytest
from pydantic import ValidationError
from src.telemetry.events.agent import (
AgentIterationEvent,
AgentToolConclusionsCreatedEvent,
AgentToolConclusionsDeletedEvent,
AgentToolPeerCardUpdatedEvent,
AgentToolSummaryCreatedEvent,
)
from src.telemetry.events.base import BaseEvent, generate_event_id
from src.telemetry.events.deletion import DeletionCompletedEvent
from src.telemetry.events.dialectic import DialecticCompletedEvent
from src.telemetry.events.dream import DreamRunEvent, DreamSpecialistEvent
from src.telemetry.events.reconciliation import (
CleanupStaleItemsCompletedEvent,
SyncVectorsCompletedEvent,
)
from src.telemetry.events.representation import RepresentationCompletedEvent
# =============================================================================
# Tests for generate_event_id function
# =============================================================================
class TestGenerateEventId:
"""Tests for the generate_event_id function."""
def test_deterministic_output(self, fixed_timestamp: datetime):
"""Same inputs always produce the same ID."""
event_type = "test.event"
resource_id = "resource_123"
id1 = generate_event_id(event_type, fixed_timestamp, resource_id)
id2 = generate_event_id(event_type, fixed_timestamp, resource_id)
assert id1 == id2
def test_different_event_types_produce_different_ids(
self, fixed_timestamp: datetime
):
"""Different event types produce different IDs."""
resource_id = "resource_123"
id1 = generate_event_id("type.a", fixed_timestamp, resource_id)
id2 = generate_event_id("type.b", fixed_timestamp, resource_id)
assert id1 != id2
def test_different_timestamps_produce_different_ids(self):
"""Different timestamps produce different IDs."""
event_type = "test.event"
resource_id = "resource_123"
ts1 = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC)
ts2 = datetime(2024, 1, 1, 12, 0, 1, tzinfo=UTC)
id1 = generate_event_id(event_type, ts1, resource_id)
id2 = generate_event_id(event_type, ts2, resource_id)
assert id1 != id2
def test_different_resource_ids_produce_different_ids(
self, fixed_timestamp: datetime
):
"""Different resource IDs produce different IDs."""
event_type = "test.event"
id1 = generate_event_id(event_type, fixed_timestamp, "resource_a")
id2 = generate_event_id(event_type, fixed_timestamp, "resource_b")
assert id1 != id2
def test_id_format(self, fixed_timestamp: datetime):
"""Event ID has correct format: evt_{base64_hash}."""
event_id = generate_event_id("test.event", fixed_timestamp, "resource_123")
assert event_id.startswith("evt_")
# Base64url encoded 16 bytes = 22 chars (without padding)
assert len(event_id) == 4 + 22 # "evt_" + 22 chars
# =============================================================================
# Tests for BaseEvent class
# =============================================================================
class TestBaseEvent:
"""Tests for BaseEvent base class behavior."""
def test_timestamp_defaults_to_utc_now(self):
"""Events without explicit timestamp get current UTC time."""
# Create event without timestamp
event = RepresentationCompletedEvent(
workspace_name="test",
session_name="test_session",
observed="user",
queue_items_processed=1,
earliest_message_id="msg_1",
latest_message_id="msg_2",
message_count=2,
explicit_conclusion_count=1,
context_preparation_ms=10.0,
llm_call_ms=100.0,
total_duration_ms=110.0,
input_tokens=100,
output_tokens=50,
)
# Timestamp should be set and be UTC
assert event.timestamp is not None
assert event.timestamp.tzinfo == UTC
def test_get_resource_id_not_implemented_on_base(self):
"""BaseEvent.get_resource_id raises NotImplementedError."""
# Can't instantiate BaseEvent directly, but we can test the method
with pytest.raises(NotImplementedError, match="Subclasses must implement"):
BaseEvent.get_resource_id(BaseEvent(timestamp=datetime.now(UTC)))
# =============================================================================
# Tests for RepresentationCompletedEvent
# =============================================================================
class TestRepresentationCompletedEvent:
"""Tests for RepresentationCompletedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert RepresentationCompletedEvent.event_type() == "representation.completed"
def test_schema_version(self):
"""schema_version() returns correct value."""
assert RepresentationCompletedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert RepresentationCompletedEvent.category() == "representation"
def test_get_resource_id(
self, sample_representation_event: RepresentationCompletedEvent
):
"""get_resource_id() returns workspace:session:message format."""
resource_id = sample_representation_event.get_resource_id()
assert resource_id == "test_workspace:test_session:msg_010"
def test_generate_id_deterministic(
self, sample_representation_event: RepresentationCompletedEvent
):
"""generate_id() produces same ID for same event."""
id1 = sample_representation_event.generate_id()
id2 = sample_representation_event.generate_id()
assert id1 == id2
assert id1.startswith("evt_")
def test_required_fields(self):
"""Event requires all mandatory fields."""
with pytest.raises(ValidationError):
RepresentationCompletedEvent() # pyright: ignore[reportCallIssue]
def test_model_dump(
self, sample_representation_event: RepresentationCompletedEvent
):
"""Event can be serialized to dict."""
data = sample_representation_event.model_dump(mode="json")
assert data["workspace_name"] == "test_workspace"
assert data["message_count"] == 10
assert data["explicit_conclusion_count"] == 5
# =============================================================================
# Tests for DreamRunEvent
# =============================================================================
class TestDreamRunEvent:
"""Tests for DreamRunEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert DreamRunEvent.event_type() == "dream.run"
def test_schema_version(self):
"""schema_version() returns correct value."""
assert DreamRunEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert DreamRunEvent.category() == "dream"
def test_get_resource_id(self, sample_dream_run_event: DreamRunEvent):
"""get_resource_id() returns run_id."""
assert sample_dream_run_event.get_resource_id() == "abc12345"
def test_generate_id_deterministic(self, sample_dream_run_event: DreamRunEvent):
"""generate_id() produces same ID for same event."""
id1 = sample_dream_run_event.generate_id()
id2 = sample_dream_run_event.generate_id()
assert id1 == id2
def test_specialists_run_list(self, sample_dream_run_event: DreamRunEvent):
"""specialists_run contains expected values."""
assert "deduction" in sample_dream_run_event.specialists_run
assert "induction" in sample_dream_run_event.specialists_run
# =============================================================================
# Tests for DreamSpecialistEvent
# =============================================================================
class TestDreamSpecialistEvent:
"""Tests for DreamSpecialistEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert DreamSpecialistEvent.event_type() == "dream.specialist"
def test_schema_version(self):
"""schema_version() returns correct value."""
assert DreamSpecialistEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert DreamSpecialistEvent.category() == "dream"
def test_get_resource_id(self, sample_dream_specialist_event: DreamSpecialistEvent):
"""get_resource_id() returns run_id:specialist_type format."""
assert sample_dream_specialist_event.get_resource_id() == "abc12345:deduction"
def test_generate_id_deterministic(
self, sample_dream_specialist_event: DreamSpecialistEvent
):
"""generate_id() produces same ID for same event."""
id1 = sample_dream_specialist_event.generate_id()
id2 = sample_dream_specialist_event.generate_id()
assert id1 == id2
# =============================================================================
# Tests for DialecticCompletedEvent
# =============================================================================
class TestDialecticCompletedEvent:
"""Tests for DialecticCompletedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert DialecticCompletedEvent.event_type() == "dialectic.completed"
def test_schema_version(self):
"""schema_version() returns correct value."""
assert DialecticCompletedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert DialecticCompletedEvent.category() == "dialectic"
def test_get_resource_id(self, sample_dialectic_event: DialecticCompletedEvent):
"""get_resource_id() returns run_id."""
assert sample_dialectic_event.get_resource_id() == "def67890"
def test_generate_id_deterministic(
self, sample_dialectic_event: DialecticCompletedEvent
):
"""generate_id() produces same ID for same event."""
id1 = sample_dialectic_event.generate_id()
id2 = sample_dialectic_event.generate_id()
assert id1 == id2
def test_optional_session_fields(self, fixed_timestamp: datetime):
"""session_id and session_name are optional."""
event = DialecticCompletedEvent(
timestamp=fixed_timestamp,
run_id="test123",
workspace_name="test",
peer_name="user",
reasoning_level="low",
total_duration_ms=1000.0,
input_tokens=500,
output_tokens=100,
)
assert event.session_name is None
def test_cache_token_defaults(self, fixed_timestamp: datetime):
"""Cache tokens default to 0."""
event = DialecticCompletedEvent(
timestamp=fixed_timestamp,
run_id="test123",
workspace_name="test",
peer_name="user",
reasoning_level="low",
total_duration_ms=1000.0,
input_tokens=500,
output_tokens=100,
)
assert event.cache_read_tokens == 0
assert event.cache_creation_tokens == 0
# =============================================================================
# Tests for AgentIterationEvent
# =============================================================================
class TestAgentIterationEvent:
"""Tests for AgentIterationEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert AgentIterationEvent.event_type() == "agent.iteration"
def test_schema_version(self):
"""schema_version() returns correct value."""
assert AgentIterationEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert AgentIterationEvent.category() == "agent"
def test_get_resource_id(self, sample_agent_iteration_event: AgentIterationEvent):
"""get_resource_id() returns run_id:iteration format."""
assert sample_agent_iteration_event.get_resource_id() == "abc12345:3"
def test_generate_id_deterministic(
self, sample_agent_iteration_event: AgentIterationEvent
):
"""generate_id() produces same ID for same event."""
id1 = sample_agent_iteration_event.generate_id()
id2 = sample_agent_iteration_event.generate_id()
assert id1 == id2
def test_tool_calls_list(self, sample_agent_iteration_event: AgentIterationEvent):
"""tool_calls is a list of strings."""
assert isinstance(sample_agent_iteration_event.tool_calls, list)
assert "search_memory" in sample_agent_iteration_event.tool_calls
def test_optional_peer_fields(self, fixed_timestamp: datetime):
"""observer, observed, peer_id are optional."""
event = AgentIterationEvent(
timestamp=fixed_timestamp,
run_id="test123",
parent_category="dialectic",
agent_type="dialectic",
workspace_name="test",
iteration=1,
input_tokens=100,
output_tokens=50,
)
assert event.observer is None
assert event.observed is None
# =============================================================================
# Tests for AgentToolConclusionsCreatedEvent
# =============================================================================
class TestAgentToolConclusionsCreatedEvent:
"""Tests for AgentToolConclusionsCreatedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert (
AgentToolConclusionsCreatedEvent.event_type()
== "agent.tool.conclusions.created"
)
def test_schema_version(self):
"""schema_version() returns correct value."""
assert AgentToolConclusionsCreatedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert AgentToolConclusionsCreatedEvent.category() == "agent"
def test_get_resource_id(
self, sample_conclusions_created_event: AgentToolConclusionsCreatedEvent
):
"""get_resource_id() returns run_id:iteration:conclusions_created format."""
assert (
sample_conclusions_created_event.get_resource_id()
== "abc12345:3:conclusions_created"
)
def test_levels_list(
self, sample_conclusions_created_event: AgentToolConclusionsCreatedEvent
):
"""levels is a list matching conclusion_count."""
assert len(sample_conclusions_created_event.levels) == 5
assert sample_conclusions_created_event.conclusion_count == 5
# =============================================================================
# Tests for AgentToolConclusionsDeletedEvent
# =============================================================================
class TestAgentToolConclusionsDeletedEvent:
"""Tests for AgentToolConclusionsDeletedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert (
AgentToolConclusionsDeletedEvent.event_type()
== "agent.tool.conclusions.deleted"
)
def test_schema_version(self):
"""schema_version() returns correct value."""
assert AgentToolConclusionsDeletedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert AgentToolConclusionsDeletedEvent.category() == "agent"
def test_get_resource_id(
self, sample_conclusions_deleted_event: AgentToolConclusionsDeletedEvent
):
"""get_resource_id() returns run_id:iteration:conclusions_deleted format."""
assert (
sample_conclusions_deleted_event.get_resource_id()
== "abc12345:5:conclusions_deleted"
)
# =============================================================================
# Tests for AgentToolPeerCardUpdatedEvent
# =============================================================================
class TestAgentToolPeerCardUpdatedEvent:
"""Tests for AgentToolPeerCardUpdatedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert (
AgentToolPeerCardUpdatedEvent.event_type() == "agent.tool.peer_card.updated"
)
def test_schema_version(self):
"""schema_version() returns correct value."""
assert AgentToolPeerCardUpdatedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert AgentToolPeerCardUpdatedEvent.category() == "agent"
def test_get_resource_id(
self, sample_peer_card_updated_event: AgentToolPeerCardUpdatedEvent
):
"""get_resource_id() returns run_id:iteration:peer_card_updated format."""
assert (
sample_peer_card_updated_event.get_resource_id()
== "abc12345:7:peer_card_updated"
)
# =============================================================================
# Tests for AgentToolSummaryCreatedEvent
# =============================================================================
class TestAgentToolSummaryCreatedEvent:
"""Tests for AgentToolSummaryCreatedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert AgentToolSummaryCreatedEvent.event_type() == "agent.tool.summary.created"
def test_schema_version(self):
"""schema_version() returns correct value."""
assert AgentToolSummaryCreatedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert AgentToolSummaryCreatedEvent.category() == "agent"
def test_get_resource_id(
self, sample_summary_created_event: AgentToolSummaryCreatedEvent
):
"""get_resource_id() returns run_id:iteration:summary_created format."""
assert (
sample_summary_created_event.get_resource_id()
== "ghi11111:1:summary_created"
)
def test_summary_type_values(self, fixed_timestamp: datetime):
"""summary_type accepts 'short' and 'long'."""
for summary_type in ["short", "long"]:
event = AgentToolSummaryCreatedEvent(
timestamp=fixed_timestamp,
run_id="test123",
iteration=1,
parent_category="representation",
agent_type="summarizer",
workspace_name="test",
session_name="test_session",
message_id="msg_1",
message_count=10,
message_seq_in_session=10,
summary_type=summary_type,
input_tokens=100,
output_tokens=50,
)
assert event.summary_type == summary_type
# =============================================================================
# Tests for DeletionCompletedEvent
# =============================================================================
class TestDeletionCompletedEvent:
"""Tests for DeletionCompletedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert DeletionCompletedEvent.event_type() == "deletion.completed"
def test_schema_version(self):
"""schema_version() returns correct value."""
assert DeletionCompletedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert DeletionCompletedEvent.category() == "deletion"
def test_get_resource_id(self, sample_deletion_event: DeletionCompletedEvent):
"""get_resource_id() returns workspace:type:resource format."""
assert (
sample_deletion_event.get_resource_id()
== "test_workspace:workspace:ws_123abc"
)
def test_cascade_counts_default_to_zero(self, fixed_timestamp: datetime):
"""Cascade counts default to 0 for non-workspace deletions."""
event = DeletionCompletedEvent(
timestamp=fixed_timestamp,
workspace_name="test",
deletion_type="session",
resource_id="sess_456",
success=True,
)
assert event.peers_deleted == 0
assert event.sessions_deleted == 0
assert event.messages_deleted == 0
assert event.conclusions_deleted == 0
def test_error_message_optional(self, fixed_timestamp: datetime):
"""error_message is optional and defaults to None."""
event = DeletionCompletedEvent(
timestamp=fixed_timestamp,
workspace_name="test",
deletion_type="session",
resource_id="sess_456",
success=True,
)
assert event.error_message is None
def test_failed_deletion_with_error(self, fixed_timestamp: datetime):
"""Failed deletion can include error message."""
event = DeletionCompletedEvent(
timestamp=fixed_timestamp,
workspace_name="test",
deletion_type="session",
resource_id="sess_456",
success=False,
error_message="Foreign key constraint violation",
)
assert event.success is False
assert event.error_message == "Foreign key constraint violation"
# =============================================================================
# Tests for SyncVectorsCompletedEvent
# =============================================================================
class TestSyncVectorsCompletedEvent:
"""Tests for SyncVectorsCompletedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert (
SyncVectorsCompletedEvent.event_type()
== "reconciliation.sync_vectors.completed"
)
def test_schema_version(self):
"""schema_version() returns correct value."""
assert SyncVectorsCompletedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert SyncVectorsCompletedEvent.category() == "reconciliation"
def test_get_resource_id(
self, sample_sync_vectors_event: SyncVectorsCompletedEvent
):
"""get_resource_id() returns fixed string."""
assert sample_sync_vectors_event.get_resource_id() == "sync_vectors"
def test_metrics_default_to_zero(self, fixed_timestamp: datetime):
"""Sync metrics default to 0."""
event = SyncVectorsCompletedEvent(
timestamp=fixed_timestamp,
total_duration_ms=1000.0,
)
assert event.documents_synced == 0
assert event.documents_failed == 0
assert event.message_embeddings_synced == 0
assert event.message_embeddings_failed == 0
# =============================================================================
# Tests for CleanupStaleItemsCompletedEvent
# =============================================================================
class TestCleanupStaleItemsCompletedEvent:
"""Tests for CleanupStaleItemsCompletedEvent."""
def test_event_type(self):
"""event_type() returns correct value."""
assert (
CleanupStaleItemsCompletedEvent.event_type()
== "reconciliation.cleanup_stale_items.completed"
)
def test_schema_version(self):
"""schema_version() returns correct value."""
assert CleanupStaleItemsCompletedEvent.schema_version() == 1
def test_category(self):
"""category() returns correct value."""
assert CleanupStaleItemsCompletedEvent.category() == "reconciliation"
def test_get_resource_id(
self, sample_cleanup_event: CleanupStaleItemsCompletedEvent
):
"""get_resource_id() returns fixed string."""
assert sample_cleanup_event.get_resource_id() == "cleanup_stale_items"
def test_cleanup_metrics_default_to_zero(self, fixed_timestamp: datetime):
"""Cleanup metrics default to 0."""
event = CleanupStaleItemsCompletedEvent(
timestamp=fixed_timestamp,
total_duration_ms=500.0,
)
assert event.documents_cleaned == 0
assert event.queue_items_cleaned == 0
# =============================================================================
# Parametrized tests across all event types
# =============================================================================
class TestAllEventTypes:
"""Parametrized tests that run across all event types."""
def test_all_events_have_timestamp(self, all_sample_events: list[BaseEvent]):
"""All events have a timestamp field."""
for event in all_sample_events:
assert hasattr(event, "timestamp")
assert event.timestamp is not None
def test_all_events_generate_valid_ids(self, all_sample_events: list[BaseEvent]):
"""All events generate valid event IDs."""
for event in all_sample_events:
event_id = event.generate_id()
assert event_id.startswith("evt_")
assert len(event_id) == 26 # "evt_" + 22 chars
def test_all_events_have_event_type(self, all_sample_events: list[BaseEvent]):
"""All events return a non-empty event_type."""
for event in all_sample_events:
event_type = event.event_type()
assert event_type
assert isinstance(event_type, str)
def test_all_events_have_category(self, all_sample_events: list[BaseEvent]):
"""All events return a non-empty category."""
for event in all_sample_events:
category = event.category()
assert category
assert isinstance(category, str)
def test_all_events_have_schema_version(self, all_sample_events: list[BaseEvent]):
"""All events return a positive schema version."""
for event in all_sample_events:
version = event.schema_version()
assert version >= 1
assert isinstance(version, int)
def test_all_events_can_serialize_to_json(self, all_sample_events: list[BaseEvent]):
"""All events can be serialized to JSON-compatible dict."""
for event in all_sample_events:
data = event.model_dump(mode="json")
assert isinstance(data, dict)
# Timestamp should be serialized as ISO string
assert "timestamp" in data
def test_all_events_have_get_resource_id(self, all_sample_events: list[BaseEvent]):
"""All events implement get_resource_id()."""
for event in all_sample_events:
resource_id = event.get_resource_id()
assert resource_id
assert isinstance(resource_id, str)

29
uv.lock
View File

@ -459,6 +459,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" },
]
[[package]]
name = "cloudevents"
version = "1.12.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "deprecation" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/aa/804bdb5f2f021fcc887eeabfa24bad0ffd4b150f60850ae88faa51d393a5/cloudevents-1.12.0.tar.gz", hash = "sha256:ebd5544ceb58c8378a0787b657a2ae895e929b80a82d6675cba63f0e8c5539e0", size = 34494, upload-time = "2025-06-02T18:58:45.104Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/4c/b6/4e29b74bb40daa7580310a5ff0df5f121a08ce98340e01a960b668468aab/cloudevents-1.12.0-py3-none-any.whl", hash = "sha256:49196267f5f963d87ae156f93fc0fa32f4af69485f2c8e62e0db8b0b4b8b8921", size = 55762, upload-time = "2025-06-02T18:58:44.013Z" },
]
[[package]]
name = "colorama"
version = "0.4.6"
@ -1018,6 +1030,7 @@ source = { virtual = "." }
dependencies = [
{ name = "alembic" },
{ name = "cashews", extra = ["redis"] },
{ name = "cloudevents" },
{ name = "fastapi", extra = ["standard"] },
{ name = "fastapi-pagination" },
{ name = "google-genai" },
@ -1029,9 +1042,10 @@ dependencies = [
{ name = "langfuse" },
{ name = "nanoid" },
{ name = "openai" },
{ name = "opentelemetry-exporter-otlp-proto-http" },
{ name = "opentelemetry-sdk" },
{ name = "pdfplumber" },
{ name = "pgvector" },
{ name = "prometheus-client" },
{ name = "psycopg", extra = ["binary"] },
{ name = "pydantic" },
{ name = "pydantic-settings" },
@ -1072,6 +1086,7 @@ dev = [
requires-dist = [
{ name = "alembic", specifier = ">=1.14.0" },
{ name = "cashews", extras = ["redis"], specifier = "==7.4.4" },
{ name = "cloudevents", specifier = ">=1.12.0" },
{ name = "fastapi", extras = ["standard"], specifier = ">=0.111.0" },
{ name = "fastapi-pagination", specifier = ">=0.12.24" },
{ name = "google-genai", specifier = ">=1.32.0" },
@ -1083,9 +1098,10 @@ requires-dist = [
{ name = "langfuse", specifier = ">=3.3.2" },
{ name = "nanoid", specifier = ">=2.0.0" },
{ name = "openai", specifier = ">=1.99.7" },
{ name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.36.0" },
{ name = "opentelemetry-sdk", specifier = ">=1.36.0" },
{ name = "pdfplumber", specifier = ">=0.11.7" },
{ name = "pgvector", specifier = ">=0.2.5" },
{ name = "prometheus-client", specifier = ">=0.20.0" },
{ name = "psycopg", extras = ["binary"], specifier = ">=3.1.19" },
{ name = "pydantic", specifier = ">=2.11.7" },
{ name = "pydantic-settings", specifier = ">=2.10.1" },
@ -2130,15 +2146,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/88/74/a88bf1b1efeae488a0c0b7bdf71429c313722d1fc0f377537fbe554e6180/pre_commit-4.2.0-py2.py3-none-any.whl", hash = "sha256:a009ca7205f1eb497d10b845e52c838a98b6cdd2102a6c8e4540e94ee75c58bd", size = 220707, upload-time = "2025-03-18T21:35:19.343Z" },
]
[[package]]
name = "prometheus-client"
version = "0.23.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" },
]
[[package]]
name = "propcache"
version = "0.4.1"