From 6b3ecef60110c4f22e4e94ad998a4331e9f858a1 Mon Sep 17 00:00:00 2001 From: Vineeth Voruganti <13438633+VVoruganti@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:49:14 -0500 Subject: [PATCH] Telemetry Overhaul (#333) * chore: Refactor a top-level telemetry folder * fix: Add OTEL push based metrics * chore: cleanup otel to match prometheus implementation * fix: Remove prometheus * feat: Scaffold CloudEvent Emitter * chore: remove dead code * feat: PoC Cloud Events * chore: cleanup test * fix: Event naming conventions and OTEL Settings * fix: Revamped Dream Event Structure * fix: Instrument Event Code * fix: Add Tests for CloudEvent Telemetry * fix: Code Rabbit Nits * fix: Dedupe Event IDs --- .env.template | 68 +- .gitignore | 3 + README.md | 15 +- config.toml.example | 62 +- docker-compose.yml.example | 91 +- pyproject.toml | 6 +- src/config.py | 76 +- src/crud/__init__.py | 4 + src/crud/representation.py | 2 +- src/crud/session.py | 23 +- src/crud/workspace.py | 62 +- src/deriver/__main__.py | 27 +- src/deriver/consumer.py | 68 +- src/deriver/deriver.py | 49 +- src/deriver/queue_manager.py | 12 +- src/dialectic/core.py | 71 +- src/dreamer/__init__.py | 2 +- src/dreamer/dreamer.py | 51 - src/dreamer/orchestrator.py | 162 ++- src/dreamer/specialists.py | 90 +- src/main.py | 47 +- src/prometheus.py | 186 --- src/routers/messages.py | 22 +- src/routers/peers.py | 17 +- src/routers/workspaces.py | 17 +- src/sentry.py | 46 - src/telemetry/__init__.py | 85 ++ src/telemetry/emitter.py | 379 ++++++ src/telemetry/events/__init__.py | 189 +++ src/telemetry/events/agent.py | 233 ++++ src/telemetry/events/base.py | 97 ++ src/telemetry/events/deletion.py | 65 + src/telemetry/events/dialectic.py | 75 ++ src/telemetry/events/dream.py | 119 ++ src/telemetry/events/reconciliation.py | 88 ++ src/telemetry/events/representation.py | 68 + src/{utils => telemetry}/logging.py | 35 +- src/{utils => telemetry}/metrics_collector.py | 0 src/telemetry/otel/__init__.py | 26 + src/telemetry/otel/metrics.py | 481 +++++++ src/{utils => telemetry}/reasoning_traces.py | 0 src/telemetry/sentry.py | 90 ++ src/utils/agent_tools.py | 86 +- src/utils/clients.py | 64 +- src/utils/summarizer.py | 69 +- src/utils/tokens.py | 21 +- src/utils/tracing.py | 48 - src/utils/types.py | 15 + tests/bench/beam.py | 2 +- tests/bench/locomo.py | 2 +- tests/bench/longmem.py | 2 +- tests/bench/run_tests.py | 4 +- tests/conftest.py | 2 +- tests/crud/test_workspace.py | 17 +- tests/deriver/test_queue_processing.py | 3 + tests/integration/test_telemetry.py | 1170 +++++++++++++++++ tests/integration/test_token_metrics.py | 198 ++- tests/telemetry/__init__.py | 1 + tests/telemetry/conftest.py | 458 +++++++ tests/telemetry/test_emit_function.py | 470 +++++++ tests/telemetry/test_emitter.py | 982 ++++++++++++++ tests/telemetry/test_events.py | 724 ++++++++++ uv.lock | 29 +- 63 files changed, 7074 insertions(+), 602 deletions(-) delete mode 100644 src/dreamer/dreamer.py delete mode 100644 src/prometheus.py delete mode 100644 src/sentry.py create mode 100644 src/telemetry/__init__.py create mode 100644 src/telemetry/emitter.py create mode 100644 src/telemetry/events/__init__.py create mode 100644 src/telemetry/events/agent.py create mode 100644 src/telemetry/events/base.py create mode 100644 src/telemetry/events/deletion.py create mode 100644 src/telemetry/events/dialectic.py create mode 100644 src/telemetry/events/dream.py create mode 100644 src/telemetry/events/reconciliation.py create mode 100644 src/telemetry/events/representation.py rename src/{utils => telemetry}/logging.py (84%) rename src/{utils => telemetry}/metrics_collector.py (100%) create mode 100644 src/telemetry/otel/__init__.py create mode 100644 src/telemetry/otel/metrics.py rename src/{utils => telemetry}/reasoning_traces.py (100%) create mode 100644 src/telemetry/sentry.py delete mode 100644 src/utils/tracing.py create mode 100644 tests/integration/test_telemetry.py create mode 100644 tests/telemetry/__init__.py create mode 100644 tests/telemetry/conftest.py create mode 100644 tests/telemetry/test_emit_function.py create mode 100644 tests/telemetry/test_emitter.py create mode 100644 tests/telemetry/test_events.py diff --git a/.env.template b/.env.template index d72ef49c..781f71b3 100644 --- a/.env.template +++ b/.env.template @@ -23,6 +23,7 @@ LOG_LEVEL=INFO # COLLECT_METRICS_LOCAL=false # LOCAL_METRICS_FILE=metrics.jsonl +# REASONING_TRACES_FILE=traces.jsonl # Path to JSONL file for reasoning traces # NAMESPACE="honcho" @@ -97,12 +98,14 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000 # 30 days # DERIVER_PROVIDER=google # DERIVER_MODEL=gemini-2.5-flash-lite +# DERIVER_TEMPERATURE= # DERIVER_DEDUPLICATE=true # DERIVER_MAX_OUTPUT_TOKENS=4096 # DERIVER_THINKING_BUDGET_TOKENS=1024 # DERIVER_LOG_OBSERVATIONS=false +# DERIVER_MAX_INPUT_TOKENS=23000 # DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100 -# DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=4096 +# DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=1024 # DERIVER_BACKUP_PROVIDER= # DERIVER_BACKUP_MODEL= @@ -127,19 +130,22 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # DIALECTIC_LEVELS__minimal__PROVIDER=google # DIALECTIC_LEVELS__minimal__MODEL=gemini-2.5-flash-lite # DIALECTIC_LEVELS__minimal__THINKING_BUDGET_TOKENS=0 -# DIALECTIC_LEVELS__minimal__MAX_TOOL_ITERATIONS=2 +# DIALECTIC_LEVELS__minimal__MAX_TOOL_ITERATIONS=5 +# DIALECTIC_LEVELS__minimal__TOOL_CHOICE=any # Low level # DIALECTIC_LEVELS__low__PROVIDER=google -# DIALECTIC_LEVELS__low__MODEL=gemini-3-flash +# DIALECTIC_LEVELS__low__MODEL=gemini-3-flash-preview # DIALECTIC_LEVELS__low__THINKING_BUDGET_TOKENS=0 # DIALECTIC_LEVELS__low__MAX_TOOL_ITERATIONS=5 +# DIALECTIC_LEVELS__low__TOOL_CHOICE=any # Medium level # DIALECTIC_LEVELS__medium__PROVIDER=anthropic # DIALECTIC_LEVELS__medium__MODEL=claude-haiku-4-5 -# DIALECTIC_LEVELS__medium__THINKING_BUDGET_TOKENS=512 +# DIALECTIC_LEVELS__medium__THINKING_BUDGET_TOKENS=1024 # DIALECTIC_LEVELS__medium__MAX_TOOL_ITERATIONS=4 +# DIALECTIC_LEVELS__medium__TOOL_CHOICE= # High level # DIALECTIC_LEVELS__high__PROVIDER=anthropic @@ -150,7 +156,7 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # Max level # DIALECTIC_LEVELS__max__PROVIDER=anthropic # DIALECTIC_LEVELS__max__MODEL=claude-opus-4-5 -# DIALECTIC_LEVELS__max__THINKING_BUDGET_TOKENS=512 +# DIALECTIC_LEVELS__max__THINKING_BUDGET_TOKENS=2048 # DIALECTIC_LEVELS__max__MAX_TOOL_ITERATIONS=10 # Optional backup per level (must set both or neither): # DIALECTIC_LEVELS__max__BACKUP_PROVIDER=google @@ -179,16 +185,20 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # DREAM_MIN_HOURS_BETWEEN_DREAMS=8 # DREAM_ENABLED_TYPES=["omni"] # DREAM_PROVIDER=anthropic -# DREAM_MODEL=claude-haiku-4-5 -# DREAM_MAX_OUTPUT_TOKENS=4000 -# DREAM_THINKING_BUDGET_TOKENS=2048 -# DREAM_MAX_TOOL_ITERATIONS=8 -# DREAM_HISTORY_TOKEN_LIMIT=8192 +# DREAM_MODEL=claude-sonnet-4-20250514 +# DREAM_MAX_OUTPUT_TOKENS=16384 +# DREAM_THINKING_BUDGET_TOKENS=8192 +# DREAM_MAX_TOOL_ITERATIONS=20 +# DREAM_HISTORY_TOKEN_LIMIT=16384 # DREAM_BACKUP_PROVIDER= # DREAM_BACKUP_MODEL= +# Specialist models (use same provider as main model) +# DREAM_DEDUCTION_MODEL=claude-haiku-4-5 +# DREAM_INDUCTION_MODEL=claude-haiku-4-5 + # Dream Surprisal Settings (Tree-based observation sampling for targeted reasoning) -# DREAM_SURPRISAL__ENABLED=true +# DREAM_SURPRISAL__ENABLED=false # DREAM_SURPRISAL__TREE_TYPE=kdtree # Options: kdtree, balltree, rptree, covertree, lsh, graph, prototype # DREAM_SURPRISAL__TREE_K=5 # Number of neighbors for kNN-based trees # DREAM_SURPRISAL__SAMPLING_STRATEGY=recent # Options: recent, random, all @@ -215,16 +225,33 @@ LLM_ANTHROPIC_API_KEY=your-anthropic-api-key-here # SENTRY_PROFILES_SAMPLE_RATE=0.1 # ============================================================================= -# Metrics (Optional) +# OpenTelemetry Settings (Push-based metrics via OTLP) # ============================================================================= -# METRICS_ENABLED=false -# METRICS_NAMESPACE=honcho # Inherits from NAMESPACE if not set +# OTEL_ENABLED=false +# OTEL_ENDPOINT=https://mimir.example.com/otlp/v1/metrics +# OTEL_HEADERS={"X-Scope-OrgID": "honcho"} # JSON string for auth headers +# OTEL_EXPORT_INTERVAL_MILLIS=60000 +# OTEL_SERVICE_NAME=honcho +# OTEL_SERVICE_NAMESPACE=honcho # Inherits from NAMESPACE if not set + +# ============================================================================= +# CloudEvents Telemetry Settings (Analytics events) +# ============================================================================= +# TELEMETRY_ENABLED=false +# TELEMETRY_ENDPOINT=https://telemetry.honcho.dev/v1/events +# TELEMETRY_HEADERS={"Authorization": "Bearer your-token"} # JSON string for auth headers +# TELEMETRY_BATCH_SIZE=100 +# TELEMETRY_FLUSH_INTERVAL_SECONDS=1.0 +# TELEMETRY_FLUSH_THRESHOLD=50 +# TELEMETRY_MAX_RETRIES=3 +# TELEMETRY_MAX_BUFFER_SIZE=10000 +# TELEMETRY_NAMESPACE=honcho # Inherits from NAMESPACE if not set # ============================================================================= # Cache # ============================================================================= # CACHE_ENABLED=false -# CACHE_URL="redis://localhost:6379/0?suppress=false" +# CACHE_URL="redis://localhost:6379/0?suppress=true" # CACHE_NAMESPACE="honcho" # Inherits from NAMESPACE if not set # CACHE_DEFAULT_TTL_SECONDS=300 # CACHE_DEFAULT_LOCK_TTL_SECONDS=5 @@ -239,9 +266,10 @@ VECTOR_STORE_TYPE=pgvector VECTOR_STORE_MIGRATED=false # Global namespace prefix for all vector namespaces -# Namespaces follow the pattern: -# - Documents: {NAMESPACE}.{workspace}.{observer}.{observed} -# - Messages: {NAMESPACE}.{workspace}.messages +# Namespaces follow the pattern: {NAMESPACE}.{type}.{hash} +# where hash is a base64url-encoded SHA-256 of the workspace/peer names +# - Documents: {NAMESPACE}.doc.{hash(workspace, observer, observed)} +# - Messages: {NAMESPACE}.msg.{hash(workspace)} # VECTOR_STORE_NAMESPACE=honcho # Inherits from NAMESPACE if not set # Embedding dimensions (default: 1536 for OpenAI text-embedding-3-small) @@ -253,4 +281,6 @@ VECTOR_STORE_MIGRATED=false # LanceDB-specific settings (local embedded mode) # VECTOR_STORE_LANCEDB_PATH=./lancedb_data -#VECTOR_STORE_RECONCILIATION_INTERVAL_SECONDS + +# Reconciliation interval for background sync (default: 5 minutes) +# VECTOR_STORE_RECONCILIATION_INTERVAL_SECONDS=300 diff --git a/.gitignore b/.gitignore index a6d64b56..e7aae15e 100644 --- a/.gitignore +++ b/.gitignore @@ -190,3 +190,6 @@ CRUSH.md metrics.jsonl AGENTS.md lancedb_data/ + +mimir-data/ +grafana-data/ diff --git a/README.md b/README.md index 99245c87..d9e551e4 100644 --- a/README.md +++ b/README.md @@ -411,18 +411,20 @@ cp config.toml.example config.toml Then modify the values as needed. The TOML file is organized into sections: -- `[app]` - Application-level settings (log level, session limits, embedding settings, metrics) +- `[app]` - Application-level settings (log level, session limits, embedding settings, namespace) - `[db]` - Database connection and pool settings - `[auth]` - Authentication configuration - `[cache]` - Redis cache configuration - `[llm]` - LLM provider API keys and general settings -- `[dialectic]` - Dialectic API configuration (provider, model, search settings) - `[deriver]` - Background worker settings and representation configuration - `[peer_card]` - Peer card generation settings +- `[dialectic]` - Dialectic API configuration with per-level reasoning settings - `[summary]` - Session summarization settings -- `[dream]` - Dream processing configuration +- `[dream]` - Dream processing configuration (including specialist models and surprisal settings) - `[webhook]` - Webhook configuration -- `[metrics]` - Metrics collection settings +- `[otel]` - OpenTelemetry push-based metrics via OTLP +- `[telemetry]` - CloudEvents telemetry for analytics +- `[vector_store]` - Vector store configuration (pgvector, turbopuffer, or lancedb) - `[sentry]` - Error tracking and monitoring settings ### Using Environment Variables @@ -436,9 +438,12 @@ Examples: - `DB_CONNECTION_URI` - Database connection string - `AUTH_JWT_SECRET` - JWT secret key -- `DIALECTIC_MODEL` - Dialectic API model +- `DIALECTIC_LEVELS__low__MODEL` - Model for low reasoning level +- `DERIVER_PROVIDER` - Provider for background deriver - `SUMMARY_PROVIDER` - Summary generation provider - `LOG_LEVEL` - Application log level +- `OTEL_ENABLED` - Enable OpenTelemetry metrics +- `TELEMETRY_ENABLED` - Enable CloudEvents telemetry ### Configuration Priority diff --git a/config.toml.example b/config.toml.example index c8a8b496..87d10f6e 100644 --- a/config.toml.example +++ b/config.toml.example @@ -17,7 +17,8 @@ MAX_EMBEDDING_TOKENS_PER_REQUEST = 300000 # LANGFUSE_PUBLIC_KEY = "your-public-key-here" # COLLECT_METRICS_LOCAL = false # LOCAL_METRICS_FILE = "metrics.jsonl" -NAMESPACE="honcho" +# REASONING_TRACES_FILE = "traces.jsonl" # Path to JSONL file for reasoning traces +NAMESPACE = "honcho" # Database settings [db] @@ -75,14 +76,16 @@ STALE_SESSION_TIMEOUT_MINUTES = 5 # QUEUE_ERROR_RETENTION_SECONDS = 2592000 # 30 days PROVIDER = "google" MODEL = "gemini-2.5-flash-lite" +# TEMPERATURE = 0.0 # BACKUP_PROVIDER = "anthropic" # BACKUP_MODEL = "claude-haiku-4-5" DEDUPLICATE = true MAX_OUTPUT_TOKENS = 4096 THINKING_BUDGET_TOKENS = 1024 LOG_OBSERVATIONS = false +MAX_INPUT_TOKENS = 23000 WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100 -REPRESENTATION_BATCH_MAX_TOKENS = 4096 +REPRESENTATION_BATCH_MAX_TOKENS = 1024 # Peer card settings [peer_card] @@ -100,19 +103,22 @@ SESSION_HISTORY_MAX_TOKENS = 16384 PROVIDER = "google" MODEL = "gemini-2.5-flash-lite" THINKING_BUDGET_TOKENS = 0 -MAX_TOOL_ITERATIONS = 2 +MAX_TOOL_ITERATIONS = 5 +TOOL_CHOICE = "any" [dialectic.levels.low] PROVIDER = "google" -MODEL = "gemini-3-flash" +MODEL = "gemini-3-flash-preview" THINKING_BUDGET_TOKENS = 0 MAX_TOOL_ITERATIONS = 5 +TOOL_CHOICE = "any" [dialectic.levels.medium] PROVIDER = "anthropic" MODEL = "claude-haiku-4-5" -THINKING_BUDGET_TOKENS = 512 +THINKING_BUDGET_TOKENS = 1024 MAX_TOOL_ITERATIONS = 4 +# TOOL_CHOICE = "any" # Optional: None/auto lets model decide [dialectic.levels.high] PROVIDER = "anthropic" @@ -123,7 +129,7 @@ MAX_TOOL_ITERATIONS = 4 [dialectic.levels.max] PROVIDER = "anthropic" MODEL = "claude-opus-4-5" -THINKING_BUDGET_TOKENS = 512 +THINKING_BUDGET_TOKENS = 2048 MAX_TOOL_ITERATIONS = 10 # Backup provider example (optional, must set both or neither): # BACKUP_PROVIDER = "google" @@ -150,25 +156,59 @@ IDLE_TIMEOUT_MINUTES = 60 MIN_HOURS_BETWEEN_DREAMS = 8 ENABLED_TYPES = ["omni"] PROVIDER = "anthropic" -MODEL = "claude-haiku-4-5" -MAX_OUTPUT_TOKENS = 4000 +MODEL = "claude-sonnet-4-20250514" +MAX_OUTPUT_TOKENS = 16384 +THINKING_BUDGET_TOKENS = 8192 +MAX_TOOL_ITERATIONS = 20 +HISTORY_TOKEN_LIMIT = 16384 # BACKUP_PROVIDER = "google" # BACKUP_MODEL = "gemini-2.5-flash" +# Specialist models (use same provider as main model) +DEDUCTION_MODEL = "claude-haiku-4-5" +INDUCTION_MODEL = "claude-haiku-4-5" + +# Surprisal-based sampling subsystem +[dream.surprisal] +ENABLED = false +TREE_TYPE = "kdtree" # Options: kdtree, balltree, rptree, covertree, lsh, graph, prototype +TREE_K = 5 # k for kNN-based trees +SAMPLING_STRATEGY = "recent" # Options: recent, random, all +SAMPLE_SIZE = 200 +TOP_PERCENT_SURPRISAL = 0.10 # Top 10% of observations +MIN_HIGH_SURPRISAL_FOR_REPLACE = 10 +INCLUDE_LEVELS = ["explicit", "deductive"] + # Webhook settings [webhook] SECRET = "" MAX_WORKSPACE_LIMIT = 10 -# Metrics settings -[metrics] +# OpenTelemetry settings (push-based metrics via OTLP) +[otel] ENABLED = false +# ENDPOINT = "https://mimir.example.com/otlp/v1/metrics" +# HEADERS = '{"X-Scope-OrgID": "honcho"}' # JSON string for auth headers +EXPORT_INTERVAL_MILLIS = 60000 +SERVICE_NAME = "honcho" +# SERVICE_NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set + +# CloudEvents telemetry settings (analytics events) +[telemetry] +ENABLED = false +# ENDPOINT = "https://telemetry.honcho.dev/v1/events" +# HEADERS = '{"Authorization": "Bearer your-token"}' # JSON string for auth headers +BATCH_SIZE = 100 +FLUSH_INTERVAL_SECONDS = 1.0 +FLUSH_THRESHOLD = 50 +MAX_RETRIES = 3 +MAX_BUFFER_SIZE = 10000 # NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set # Cache settings [cache] ENABLED = false -URL = "redis://localhost:6379/0?suppress=false" +URL = "redis://localhost:6379/0?suppress=true" # NAMESPACE = "honcho" # Inherits from app.NAMESPACE if not set DEFAULT_TTL_SECONDS = 300 DEFAULT_LOCK_TTL_SECONDS = 5 diff --git a/docker-compose.yml.example b/docker-compose.yml.example index 123ec5c5..c72840f0 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -11,6 +11,7 @@ services: - 8000:8000 volumes: - .:/app + - venv:/app/.venv env_file: - .env deriver: @@ -23,6 +24,7 @@ services: condition: service_healthy volumes: - .:/app + - venv:/app/.venv env_file: - .env database: @@ -38,8 +40,8 @@ services: - POSTGRES_HOST_AUTH_METHOD=trust - PGDATA=/var/lib/postgresql/data/pgdata volumes: - - ./init.sql:/docker-entrypoint-initdb.d/init.sql - - ./data:/var/lib/postgresql/data/ + - ./database/init.sql:/docker-entrypoint-initdb.d/init.sql + - pgdata:/var/lib/postgresql/data/ healthcheck: test: ["CMD-SHELL", "pg_isready -U testuser -d honcho"] interval: 5s @@ -57,3 +59,88 @@ services: interval: 5s timeout: 5s retries: 5 + mimir: + image: grafana/mimir:2.14.0 + command: + - -config.file=/etc/mimir/mimir.yaml + ports: + - 9009:9009 + volumes: + - ./mimir-data:/data + configs: + - source: mimir_config + target: /etc/mimir/mimir.yaml + grafana: + image: grafana/grafana:11.4.0 + ports: + - 3000:3000 + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + volumes: + - ./grafana-data:/var/lib/grafana + depends_on: + - mimir + +volumes: + pgdata: + venv: + +configs: + mimir_config: + content: | + multitenancy_enabled: false + + server: + http_listen_port: 9009 + log_level: warn + + common: + storage: + backend: filesystem + filesystem: + dir: /data + + blocks_storage: + backend: filesystem + filesystem: + dir: /data/blocks + bucket_store: + sync_dir: /data/tsdb-sync + tsdb: + dir: /data/tsdb + + compactor: + data_dir: /data/compactor + sharding_ring: + kvstore: + store: memberlist + + distributor: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: memberlist + + ingester: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: memberlist + replication_factor: 1 + + ruler_storage: + backend: filesystem + filesystem: + dir: /data/rules + + alertmanager_storage: + backend: filesystem + filesystem: + dir: /data/alertmanager + + store_gateway: + sharding_ring: + replication_factor: 1 diff --git a/pyproject.toml b/pyproject.toml index 9e4e6360..440958f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,13 +31,15 @@ dependencies = [ "google-genai>=1.32.0", "pdfplumber>=0.11.7", "typing-extensions>=4.11.0", - "prometheus-client>=0.20.0", "json-repair>=0.49.0", "turbopuffer>=1.8.1", "lancedb>=0.25.3", "redis>=7.0.0,<8.0.0", "cashews[redis]==7.4.4", "scikit-learn>=1.6.0", + "opentelemetry-sdk>=1.36.0", + "opentelemetry-exporter-otlp-proto-http>=1.36.0", + "cloudevents>=1.12.0", ] [dependency-groups] dev = [ @@ -114,7 +116,7 @@ reportUnusedCallResult = false reportCallInDefaultInitializer = false reportAny = false reportExplicitAny = false -allowedUntypedLibraries = ["langfuse", "lancedb", "pyarrow"] +allowedUntypedLibraries = ["langfuse", "lancedb", "pyarrow", "opentelemetry"] reportImplicitOverride = false reportImportCycles = false diff --git a/src/config.py b/src/config.py index 6dcdf199..68cb2bf3 100644 --- a/src/config.py +++ b/src/config.py @@ -68,6 +68,8 @@ class TomlConfigSettingsSource(PydanticBaseSettingsSource): "WEBHOOK": "webhook", "DREAM": "dream", "VECTOR_STORE": "vector_store", + "OTEL": "otel", + "TELEMETRY": "telemetry", "": "app", # For AppSettings with no prefix } @@ -439,10 +441,65 @@ class WebhookSettings(HonchoSettings): MAX_WORKSPACE_LIMIT: int = 10 -class MetricsSettings(HonchoSettings): - model_config = SettingsConfigDict(env_prefix="METRICS_", extra="ignore") # pyright: ignore +class OpenTelemetrySettings(HonchoSettings): + """OpenTelemetry settings for push-based metrics via OTLP. + These settings configure the OTel SDK to push metrics via OTLP HTTP + to any compatible backend (Mimir, Grafana Cloud, etc.). + """ + + model_config = SettingsConfigDict(env_prefix="OTEL_", extra="ignore") # pyright: ignore + + # Master toggle for OTel metrics ENABLED: bool = False + + # OTLP HTTP endpoint for metrics (e.g., "https://mimir.example.com/otlp/v1/metrics") + # For Mimir, the endpoint is typically: /otlp/v1/metrics + # For Grafana Cloud: https://otlp-gateway-.grafana.net/otlp/v1/metrics + ENDPOINT: str | None = None + + HEADERS: dict[str, str] | None = None + + # Export interval in milliseconds (default: 60 seconds) + EXPORT_INTERVAL_MILLIS: int = 60000 + + # Service name for resource attributes (identifies what this service is) + SERVICE_NAME: str = "honcho" + + # Service namespace for resource attributes (defaults to top-level NAMESPACE if not set) + SERVICE_NAMESPACE: str | None = None + + +class TelemetrySettings(HonchoSettings): + """CloudEvents telemetry settings for analytics. + + These settings configure the CloudEvents emitter for pushing + structured events to an analytics backend. + """ + + model_config = SettingsConfigDict(env_prefix="TELEMETRY_", extra="ignore") # pyright: ignore + + # Master toggle for CloudEvents emission + ENABLED: bool = False + + # CloudEvents HTTP endpoint (e.g., "https://telemetry.honcho.dev/v1/events") + ENDPOINT: str | None = None + + # Optional headers for authentication + HEADERS: dict[str, str] | None = None + + # Batching configuration + BATCH_SIZE: Annotated[int, Field(default=100, gt=0, le=1000)] = 100 + FLUSH_INTERVAL_SECONDS: Annotated[float, Field(default=1.0, gt=0.0, le=60.0)] = 1.0 + FLUSH_THRESHOLD: Annotated[int, Field(default=50, gt=0, le=1000)] = 50 + + # Retry configuration + MAX_RETRIES: Annotated[int, Field(default=3, gt=0, le=10)] = 3 + + # Buffer configuration + MAX_BUFFER_SIZE: Annotated[int, Field(default=10000, gt=0, le=100000)] = 10000 + + # Namespace for instance identification (propagated from top-level NAMESPACE if not set) NAMESPACE: str | None = None @@ -615,7 +672,8 @@ class AppSettings(HonchoSettings): PEER_CARD: PeerCardSettings = Field(default_factory=PeerCardSettings) SUMMARY: SummarySettings = Field(default_factory=SummarySettings) WEBHOOK: WebhookSettings = Field(default_factory=WebhookSettings) - METRICS: MetricsSettings = Field(default_factory=MetricsSettings) + OTEL: OpenTelemetrySettings = Field(default_factory=OpenTelemetrySettings) + TELEMETRY: TelemetrySettings = Field(default_factory=TelemetrySettings) CACHE: CacheSettings = Field(default_factory=CacheSettings) DREAM: DreamSettings = Field(default_factory=DreamSettings) VECTOR_STORE: VectorStoreSettings = Field(default_factory=VectorStoreSettings) @@ -631,16 +689,18 @@ class AppSettings(HonchoSettings): def propagate_namespace(self) -> "AppSettings": """Propagate top-level NAMESPACE to nested settings if not explicitly set. - After this validator runs, CACHE.NAMESPACE, METRICS.NAMESPACE, and - VECTOR_STORE.NAMESPACE are guaranteed to exist. Explicitly provided - nested namespaces are preserved. + After this validator runs, CACHE.NAMESPACE, + VECTOR_STORE.NAMESPACE, TELEMETRY.NAMESPACE, and OTEL.SERVICE_NAMESPACE + are guaranteed to exist. Explicitly provided nested namespaces are preserved. """ if "NAMESPACE" not in self.CACHE.model_fields_set: self.CACHE.NAMESPACE = self.NAMESPACE - if "NAMESPACE" not in self.METRICS.model_fields_set: - self.METRICS.NAMESPACE = self.NAMESPACE if "NAMESPACE" not in self.VECTOR_STORE.model_fields_set: self.VECTOR_STORE.NAMESPACE = self.NAMESPACE + if "NAMESPACE" not in self.TELEMETRY.model_fields_set: + self.TELEMETRY.NAMESPACE = self.NAMESPACE + if "SERVICE_NAMESPACE" not in self.OTEL.model_fields_set: + self.OTEL.SERVICE_NAMESPACE = self.NAMESPACE return self diff --git a/src/crud/__init__.py b/src/crud/__init__.py index 8e5df729..ef54f599 100644 --- a/src/crud/__init__.py +++ b/src/crud/__init__.py @@ -38,6 +38,7 @@ from .representation import ( get_working_representation, ) from .session import ( + SessionDeletionResult, clone_session, delete_session, get_or_create_session, @@ -57,6 +58,7 @@ from .webhook import ( list_webhook_endpoints, ) from .workspace import ( + WorkspaceDeletionResult, delete_workspace, get_all_workspaces, get_or_create_workspace, @@ -107,6 +109,7 @@ __all__ = [ # Representation "get_working_representation", # Session + "SessionDeletionResult", "get_sessions", "get_or_create_session", "get_session", @@ -124,6 +127,7 @@ __all__ = [ "delete_webhook_endpoint", "list_webhook_endpoints", # Workspace + "WorkspaceDeletionResult", "delete_workspace", "get_or_create_workspace", "get_workspace", diff --git a/src/crud/representation.py b/src/crud/representation.py index 1d8d7875..91407cd8 100644 --- a/src/crud/representation.py +++ b/src/crud/representation.py @@ -14,8 +14,8 @@ from src.dependencies import tracked_db from src.dreamer.dream_scheduler import check_and_schedule_dream from src.embedding_client import embedding_client from src.schemas import ResolvedConfiguration +from src.telemetry.logging import accumulate_metric from src.utils.formatting import format_datetime_utc -from src.utils.logging import accumulate_metric from src.utils.representation import ( DeductiveObservation, ExplicitObservation, diff --git a/src/crud/session.py b/src/crud/session.py index 893731ec..7008725c 100644 --- a/src/crud/session.py +++ b/src/crud/session.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from logging import getLogger from typing import Any @@ -26,6 +27,15 @@ from .workspace import get_or_create_workspace logger = getLogger(__name__) + +@dataclass +class SessionDeletionResult: + """Result of a session deletion including cascade counts.""" + + messages_deleted: int + conclusions_deleted: int + + SESSION_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}:session:{session_name}" SESSION_LOCK_PREFIX = f"{get_cache_namespace()}:lock" @@ -375,7 +385,7 @@ async def _batch_delete_matching( async def delete_session( db: AsyncSession, workspace_name: str, session_name: str -) -> bool: +) -> SessionDeletionResult: """ Delete a session and all associated data (hard delete). @@ -394,7 +404,7 @@ async def delete_session( session_name: Name of the session Returns: - True if the session was deleted successfully + SessionDeletionResult containing cascade counts Raises: ResourceNotFoundException: If the session does not exist @@ -518,7 +528,7 @@ async def delete_session( ) # Delete Document entries associated with this session in batches - await _batch_delete_matching( + conclusions_deleted = await _batch_delete_matching( db, models.Document, [ @@ -529,7 +539,7 @@ async def delete_session( ) # Delete Message entries in batches - await _batch_delete_matching( + messages_deleted = await _batch_delete_matching( db, models.Message, [ @@ -556,7 +566,10 @@ async def delete_session( await db.rollback() raise e - return True + return SessionDeletionResult( + messages_deleted=messages_deleted, + conclusions_deleted=conclusions_deleted, + ) async def clone_session( diff --git a/src/crud/workspace.py b/src/crud/workspace.py index 17364105..1dcd3635 100644 --- a/src/crud/workspace.py +++ b/src/crud/workspace.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from logging import getLogger from typing import Any @@ -16,6 +17,18 @@ from src.vector_store import get_external_vector_store logger = getLogger(__name__) + +@dataclass +class WorkspaceDeletionResult: + """Result of a workspace deletion including cascade counts.""" + + workspace: schemas.Workspace + peers_deleted: int + sessions_deleted: int + messages_deleted: int + conclusions_deleted: int + + WORKSPACE_CACHE_KEY_TEMPLATE = "workspace:{workspace_name}" WORKSPACE_LOCK_PREFIX = f"{get_cache_namespace()}:lock" @@ -214,7 +227,9 @@ async def update_workspace( return honcho_workspace -async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Workspace: +async def delete_workspace( + db: AsyncSession, workspace_name: str +) -> WorkspaceDeletionResult: """ Delete a workspace. @@ -223,7 +238,8 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor workspace_name: Name of the workspace Returns: - A snapshot of the deleted workspace as a Pydantic schema + WorkspaceDeletionResult containing a snapshot of the deleted workspace + and cascade counts for deleted resources """ logger.warning("Deleting workspace %s", workspace_name) stmt = select(models.Workspace).where(models.Workspace.name == workspace_name) @@ -242,6 +258,40 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor created_at=honcho_workspace.created_at, ) + # Count resources before deletion for telemetry + peers_count = int( + await db.scalar( + select(func.count(models.Peer.id)).where( + models.Peer.workspace_name == workspace_name + ) + ) + or 0 + ) + sessions_count = int( + await db.scalar( + select(func.count(models.Session.id)).where( + models.Session.workspace_name == workspace_name + ) + ) + or 0 + ) + messages_count = int( + await db.scalar( + select(func.count(models.Message.id)).where( + models.Message.workspace_name == workspace_name + ) + ) + or 0 + ) + conclusions_count = int( + await db.scalar( + select(func.count(models.Document.id)).where( + models.Document.workspace_name == workspace_name + ) + ) + or 0 + ) + # order is important here. # delete all active queue sessions referencing this workspace first (using work_unit_key parsing) # then queue items referencing this workspace @@ -383,4 +433,10 @@ async def delete_workspace(db: AsyncSession, workspace_name: str) -> schemas.Wor await db.rollback() raise - return workspace_snapshot + return WorkspaceDeletionResult( + workspace=workspace_snapshot, + peers_deleted=peers_count, + sessions_deleted=sessions_count, + messages_deleted=messages_count, + conclusions_deleted=conclusions_count, + ) diff --git a/src/deriver/__main__.py b/src/deriver/__main__.py index 6b54d5a4..c57ac0aa 100644 --- a/src/deriver/__main__.py +++ b/src/deriver/__main__.py @@ -3,9 +3,13 @@ import logging import os import uvloop -from prometheus_client import start_http_server from src.config import settings +from src.telemetry import ( + initialize_telemetry, + initialize_telemetry_async, + shutdown_telemetry, +) from .queue_manager import main @@ -44,14 +48,15 @@ def setup_logging(): logging.getLogger("groq._base_client").setLevel(logging.WARNING) -def start_metrics_server() -> None: - """Start Prometheus metrics HTTP server on port 9090.""" +async def run_deriver(): + """Run the deriver with proper telemetry lifecycle management.""" + # Initialize async telemetry (CloudEvents emitter) + await initialize_telemetry_async() try: - # Uses default REGISTRY from prometheus_client - start_http_server(9090, addr="0.0.0.0") # nosec B104 - print("[DERIVER] Starting Prometheus metrics server on port 9090") - except Exception as e: - print(f"[DERIVER] Failed to start Prometheus metrics server: {str(e)}") + await main() + finally: + # Shutdown telemetry (flush CloudEvents buffer, shutdown OTel metrics) + await shutdown_telemetry() if __name__ == "__main__": @@ -62,10 +67,10 @@ if __name__ == "__main__": asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) try: + # Initialize sync telemetry (OTel metrics) + initialize_telemetry() print("[DERIVER] Running main loop") - if settings.METRICS.ENABLED: - start_metrics_server() - asyncio.run(main()) + asyncio.run(run_deriver()) except KeyboardInterrupt: print("[DERIVER] Shutdown initiated via KeyboardInterrupt") except Exception as e: diff --git a/src/deriver/consumer.py b/src/deriver/consumer.py index 212be5a7..806fff9a 100644 --- a/src/deriver/consumer.py +++ b/src/deriver/consumer.py @@ -1,4 +1,5 @@ import logging +import time import sentry_sdk from pydantic import ValidationError @@ -7,14 +8,20 @@ from sqlalchemy import select from src import crud, models from src.dependencies import tracked_db from src.deriver.deriver import process_representation_tasks_batch -from src.dreamer.dreamer import process_dream +from src.dreamer import process_dream from src.exceptions import ResourceNotFoundException from src.models import Message from src.reconciler.queue_cleanup import cleanup_queue_items from src.reconciler.sync_vectors import run_vector_reconciliation_cycle from src.schemas import ReconcilerType, ResolvedConfiguration +from src.telemetry.events import ( + CleanupStaleItemsCompletedEvent, + DeletionCompletedEvent, + SyncVectorsCompletedEvent, + emit, +) +from src.telemetry.logging import log_performance_metrics from src.utils import summarizer -from src.utils.logging import log_performance_metrics from src.utils.queue_payload import ( DeletionPayload, DreamPayload, @@ -151,6 +158,7 @@ async def process_representation_batch( *, observer: str | None, observed: str | None, + queue_items_count: int, ) -> None: """ Prepares and processes a batch of messages for representation tasks. @@ -173,6 +181,7 @@ async def process_representation_batch( message_level_configuration, observer=observer, observed=observed, + queue_items_count=queue_items_count, ) @@ -195,6 +204,10 @@ async def process_deletion( """ deletion_type = payload.deletion_type resource_id = payload.resource_id + success = True + error_message: str | None = None + messages_deleted = 0 + conclusions_deleted = 0 logger.info( "Processing deletion task: type=%s, resource_id=%s, workspace=%s", @@ -206,13 +219,18 @@ async def process_deletion( async with tracked_db("process_deletion") as db: if deletion_type == "session": try: - await crud.delete_session( + result = await crud.delete_session( db, workspace_name=workspace_name, session_name=resource_id ) + messages_deleted = result.messages_deleted + conclusions_deleted = result.conclusions_deleted logger.info( - "Successfully deleted session %s in workspace %s", + "Successfully deleted session %s in workspace %s " + + "(messages=%d, conclusions=%d)", resource_id, workspace_name, + messages_deleted, + conclusions_deleted, ) except ResourceNotFoundException as e: # Session not found - may have already been deleted, treat as success @@ -227,6 +245,7 @@ async def process_deletion( await crud.delete_document_by_id( db, workspace_name=workspace_name, document_id=resource_id ) + conclusions_deleted = 1 # Single observation deleted logger.info( "Successfully deleted observation %s in workspace %s", resource_id, @@ -241,7 +260,22 @@ async def process_deletion( ) else: - raise ValueError(f"Unsupported deletion type: {deletion_type}") + success = False + error_message = f"Unsupported deletion type: {deletion_type}" + raise ValueError(error_message) + + # Emit telemetry event + emit( + DeletionCompletedEvent( + workspace_name=workspace_name, + deletion_type=deletion_type, + resource_id=resource_id, + success=success, + messages_deleted=messages_deleted, + conclusions_deleted=conclusions_deleted, + error_message=error_message, + ) + ) async def process_reconciler(payload: ReconcilerPayload) -> None: @@ -257,11 +291,14 @@ async def process_reconciler(payload: ReconcilerPayload) -> None: payload: The reconciler payload containing the reconciler type """ reconciler_type = payload.reconciler_type + start_time = time.perf_counter() if reconciler_type == ReconcilerType.SYNC_VECTORS: logger.debug("Processing sync_vectors task") metrics = await run_vector_reconciliation_cycle() + duration_ms = (time.perf_counter() - start_time) * 1000 + if ( metrics.total_synced > 0 or metrics.total_failed > 0 @@ -276,9 +313,30 @@ async def process_reconciler(payload: ReconcilerPayload) -> None: metrics.documents_cleaned, ) + # Emit telemetry event + emit( + SyncVectorsCompletedEvent( + documents_synced=metrics.documents_synced, + documents_failed=metrics.documents_failed, + documents_cleaned=metrics.documents_cleaned, + message_embeddings_synced=metrics.message_embeddings_synced, + message_embeddings_failed=metrics.message_embeddings_failed, + total_duration_ms=duration_ms, + ) + ) + elif reconciler_type == ReconcilerType.CLEANUP_QUEUE: logger.debug("Processing cleanup_queue task") await cleanup_queue_items() + duration_ms = (time.perf_counter() - start_time) * 1000 + + # Emit telemetry event for cleanup stale items + emit( + CleanupStaleItemsCompletedEvent( + total_duration_ms=duration_ms, + ) + ) + else: raise ValueError(f"Unsupported reconciler type: {reconciler_type}") diff --git a/src/deriver/deriver.py b/src/deriver/deriver.py index b765e987..77a3bcb4 100644 --- a/src/deriver/deriver.py +++ b/src/deriver/deriver.py @@ -1,19 +1,22 @@ import logging import time -from src import crud, prometheus +from src import crud from src.config import settings from src.crud.representation import RepresentationManager from src.dependencies import tracked_db from src.models import Message from src.schemas import ResolvedConfiguration +from src.telemetry import otel_metrics +from src.telemetry.events import RepresentationCompletedEvent, emit +from src.telemetry.logging import accumulate_metric, log_performance_metrics +from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes +from src.telemetry.sentry import with_sentry_transaction from src.utils.clients import honcho_llm_call from src.utils.config_helpers import get_configuration from src.utils.formatting import format_new_turn_with_timestamp -from src.utils.logging import accumulate_metric, log_performance_metrics from src.utils.representation import PromptRepresentation, Representation from src.utils.tokens import estimate_tokens, track_deriver_input_tokens -from src.utils.tracing import with_sentry_transaction from .prompts import estimate_minimal_deriver_prompt_tokens, minimal_deriver_prompt @@ -27,6 +30,7 @@ async def process_representation_tasks_batch( *, observer: str, observed: str, + queue_items_count: int, ) -> None: """ Process messages with minimal overhead - single LLM call, no peer card. @@ -36,6 +40,7 @@ async def process_representation_tasks_batch( message_level_configuration: Optional configuration override. observer: The observer peer ID. observed: The observed peer ID. + queue_items_count: Number of QueueItem records being processed in this batch. """ if not messages: return @@ -88,10 +93,10 @@ async def process_representation_tasks_batch( prompt_tokens = estimate_minimal_deriver_prompt_tokens() messages_tokens = estimate_tokens(formatted_messages) track_deriver_input_tokens( - task_type=prometheus.DeriverTaskTypes.INGESTION, + task_type=DeriverTaskTypes.INGESTION, components={ - prometheus.DeriverComponents.PROMPT: prompt_tokens, - prometheus.DeriverComponents.MESSAGES: messages_tokens, + DeriverComponents.PROMPT: prompt_tokens, + DeriverComponents.MESSAGES: messages_tokens, }, ) @@ -136,11 +141,14 @@ async def process_representation_tasks_batch( "ms", ) - prometheus.DERIVER_TOKENS_PROCESSED.labels( - task_type=prometheus.DeriverTaskTypes.INGESTION.value, - token_type=prometheus.TokenTypes.OUTPUT.value, - component=prometheus.DeriverComponents.OUTPUT_TOTAL.value, - ).inc(response.output_tokens) + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_deriver_tokens( + count=response.output_tokens, + task_type=DeriverTaskTypes.INGESTION.value, + token_type=TokenTypes.OUTPUT.value, + component=DeriverComponents.OUTPUT_TOTAL.value, + ) message_ids = [m.id for m in messages if m.peer_name == observed] @@ -209,3 +217,22 @@ async def process_representation_tasks_batch( ) log_performance_metrics("minimal_deriver", f"{latest_message.id}_{observer}") + + # Emit telemetry event + emit( + RepresentationCompletedEvent( + workspace_name=latest_message.workspace_name, + session_name=latest_message.session_name, + observed=observed, + queue_items_processed=queue_items_count, + earliest_message_id=earliest_message.public_id, + latest_message_id=latest_message.public_id, + message_count=len(messages), + explicit_conclusion_count=len(observations.explicit), + context_preparation_ms=context_prep_duration, + llm_call_ms=llm_duration, + total_duration_ms=overall_duration, + input_tokens=response.input_tokens, + output_tokens=response.output_tokens, + ) + ) diff --git a/src/deriver/queue_manager.py b/src/deriver/queue_manager.py index 6132b83d..8da310e7 100644 --- a/src/deriver/queue_manager.py +++ b/src/deriver/queue_manager.py @@ -15,7 +15,7 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql import func -from src import models, prometheus +from src import models from src.cache.client import close_cache, init_cache from src.config import settings from src.dependencies import tracked_db @@ -35,7 +35,8 @@ from src.reconciler import ( set_reconciler_scheduler, ) from src.schemas import ResolvedConfiguration -from src.sentry import initialize_sentry +from src.telemetry import otel_metrics +from src.telemetry.sentry import initialize_sentry from src.utils.work_unit import parse_work_unit_key from src.webhooks.events import ( QueueEmptyEvent, @@ -439,6 +440,7 @@ class QueueManager: message_level_configuration, observer=work_unit.observer, observed=work_unit.observed, + queue_items_count=len(items_to_process), ) await self.mark_queue_items_as_processed( items_to_process, work_unit_key @@ -774,11 +776,13 @@ class QueueManager: if ( work_unit.task_type in ["representation", "summary"] and work_unit.workspace_name is not None + and settings.OTEL.ENABLED ): - prometheus.DERIVER_QUEUE_ITEMS_PROCESSED.labels( + otel_metrics.record_deriver_queue_item( + count=len(items), workspace_name=work_unit.workspace_name, task_type=work_unit.task_type, - ).inc(len(items)) + ) async def mark_queue_item_as_errored( self, item: QueueItem, work_unit_key: str, error: str diff --git a/src/dialectic/core.py b/src/dialectic/core.py index 6e8826a2..b6b26e41 100644 --- a/src/dialectic/core.py +++ b/src/dialectic/core.py @@ -13,9 +13,17 @@ from typing import Any, cast from sqlalchemy.ext.asyncio import AsyncSession -from src import crud, prometheus +from src import crud from src.config import ReasoningLevel, settings from src.dialectic import prompts +from src.telemetry import otel_metrics +from src.telemetry.events import DialecticCompletedEvent, emit +from src.telemetry.logging import ( + accumulate_metric, + log_performance_metrics, + log_token_usage_metrics, +) +from src.telemetry.otel.metrics import DialecticComponents, TokenTypes from src.utils.agent_tools import DIALECTIC_TOOLS, create_tool_executor, search_memory from src.utils.clients import ( HonchoLLMCallResponse, @@ -23,11 +31,6 @@ from src.utils.clients import ( honcho_llm_call, ) from src.utils.formatting import format_new_turn_with_timestamp -from src.utils.logging import ( - accumulate_metric, - log_performance_metrics, - log_token_usage_metrics, -) logger = logging.getLogger(__name__) @@ -87,6 +90,10 @@ class DialecticAgent: } ] self._session_history_initialized: bool = False + self._prefetched_conclusion_count: int = 0 + self._run_id: str = str(uuid.uuid4())[ + :8 + ] # Always generate for event correlation async def _initialize_session_history(self) -> None: """Fetch and inject session history into the system prompt if configured.""" @@ -174,6 +181,11 @@ class DialecticAgent: if explicit_repr.is_empty() and derived_repr.is_empty(): return None + # Count prefetched conclusions for telemetry + explicit_count = len(explicit_repr.explicit) + len(explicit_repr.deductive) + derived_count = len(derived_repr.explicit) + len(derived_repr.deductive) + self._prefetched_conclusion_count = explicit_count + derived_count + # Format as two separate sections parts: list[str] = [] @@ -256,6 +268,9 @@ class DialecticAgent: observer=self.observer, observed=self.observed, history_token_limit=settings.DIALECTIC.HISTORY_TOKEN_LIMIT, + run_id=self._run_id, + agent_type="dialectic", + parent_category="dialectic", ) return tool_executor, task_name, run_id, start_time @@ -272,6 +287,7 @@ class DialecticAgent: cache_creation_input_tokens: int | None, tool_calls_count: int, thinking_content: str | None, + iterations: int, ) -> None: """ Log metrics common to both streaming and non-streaming responses. @@ -287,6 +303,7 @@ class DialecticAgent: cache_creation_input_tokens: Cache creation tokens (if any) tool_calls_count: Number of tool calls made thinking_content: Thinking trace content (if any) + iterations: Number of iterations in the tool execution loop """ accumulate_metric(task_name, "tool_calls", tool_calls_count, "count") @@ -308,19 +325,39 @@ class DialecticAgent: if not self.metric_key and run_id is not None: log_performance_metrics("dialectic_chat", run_id) - # Track prometheus metrics - actual token counts from API - if prometheus.METRICS_ENABLED: - prometheus.DIALECTIC_TOKENS_PROCESSED.labels( - token_type=prometheus.TokenTypes.INPUT.value, - component=prometheus.DialecticComponents.TOTAL.value, + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_dialectic_tokens( + count=input_tokens, + token_type=TokenTypes.INPUT.value, + component=DialecticComponents.TOTAL.value, reasoning_level=self.reasoning_level, - ).inc(input_tokens) + ) + otel_metrics.record_dialectic_tokens( + count=output_tokens, + token_type=TokenTypes.OUTPUT.value, + component=DialecticComponents.TOTAL.value, + reasoning_level=self.reasoning_level, + ) - prometheus.DIALECTIC_TOKENS_PROCESSED.labels( - token_type=prometheus.TokenTypes.OUTPUT.value, - component=prometheus.DialecticComponents.TOTAL.value, + # Emit telemetry event + emit( + DialecticCompletedEvent( + run_id=self._run_id, + workspace_name=self.workspace_name, + peer_name=self.observed, + session_name=self.session_name, reasoning_level=self.reasoning_level, - ).inc(output_tokens) + total_iterations=iterations, + prefetched_conclusion_count=self._prefetched_conclusion_count, + tool_calls_count=tool_calls_count, + total_duration_ms=elapsed_ms, + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_tokens=cache_read_input_tokens or 0, + cache_creation_tokens=cache_creation_input_tokens or 0, + ) + ) async def answer(self, query: str) -> str: """ @@ -368,6 +405,7 @@ class DialecticAgent: cache_creation_input_tokens=response.cache_creation_input_tokens, tool_calls_count=len(response.tool_calls_made), thinking_content=response.thinking_content, + iterations=response.iterations, ) return response.content @@ -429,4 +467,5 @@ class DialecticAgent: cache_creation_input_tokens=response.cache_creation_input_tokens, tool_calls_count=len(response.tool_calls_made), thinking_content=response.thinking_content, + iterations=response.iterations, ) diff --git a/src/dreamer/__init__.py b/src/dreamer/__init__.py index bee16664..bce5c95b 100644 --- a/src/dreamer/__init__.py +++ b/src/dreamer/__init__.py @@ -2,7 +2,7 @@ from .dream_scheduler import ( check_and_schedule_dream, get_dream_scheduler, ) -from .dreamer import process_dream +from .orchestrator import process_dream __all__ = [ "get_dream_scheduler", diff --git a/src/dreamer/dreamer.py b/src/dreamer/dreamer.py deleted file mode 100644 index af3e9689..00000000 --- a/src/dreamer/dreamer.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging - -import sentry_sdk - -from src.config import settings -from src.dependencies import tracked_db -from src.dreamer.orchestrator import run_dream -from src.schemas import DreamType -from src.utils.queue_payload import DreamPayload - -logger = logging.getLogger(__name__) - - -@sentry_sdk.trace -async def process_dream( - payload: DreamPayload, - workspace_name: str, -) -> None: - """ - Process a dream task by performing collection maintenance operations. - - Args: - payload: The dream task payload containing workspace, peer, and dream type information - """ - logger.info( - f""" -(っ- ‸ - ς)ᶻ z 𐰁 ᶻ z 𐰁 ᶻ z 𐰁\n -DREAM: {payload.dream_type} documents for {workspace_name}/{payload.observer}/{payload.observed}\n -𐰁 z ᶻ 𐰁 z ᶻ 𐰁 z ᶻ(っ- ‸ - ς)""" - ) - - try: - match payload.dream_type: - case DreamType.OMNI: - async with tracked_db("dream_orchestrator") as db: - await run_dream( - db=db, - workspace_name=workspace_name, - observer=payload.observer, - observed=payload.observed, - session_name=payload.session_name, - ) - - except Exception as e: - logger.error( - f"Error processing dream task {payload.dream_type} for {payload.observer}/{payload.observed}: {str(e)}", - exc_info=True, - ) - if settings.SENTRY.ENABLED: - sentry_sdk.capture_exception(e) - # Don't re-raise - we want to mark the dream task as processed even if it fails diff --git a/src/dreamer/orchestrator.py b/src/dreamer/orchestrator.py index d85d346d..05c9474e 100644 --- a/src/dreamer/orchestrator.py +++ b/src/dreamer/orchestrator.py @@ -13,23 +13,53 @@ from __future__ import annotations import logging import time import uuid +from dataclasses import dataclass from typing import Any +import sentry_sdk from sqlalchemy.ext.asyncio import AsyncSession from src import crud from src.config import settings -from src.dreamer.specialists import SPECIALISTS +from src.dependencies import tracked_db +from src.dreamer.specialists import SPECIALISTS, SpecialistResult from src.dreamer.surprisal import SurprisalScore # type: ignore from src.exceptions import SpecialistExecutionError, SurprisalError -from src.utils.config_helpers import get_configuration -from src.utils.logging import ( +from src.schemas import DreamType +from src.telemetry.events import DreamRunEvent, emit +from src.telemetry.logging import ( accumulate_metric, log_performance_metrics, ) +from src.utils.config_helpers import get_configuration +from src.utils.queue_payload import DreamPayload logger = logging.getLogger(__name__) + +@dataclass +class DreamResult: + """Result of a dream cycle for telemetry reporting.""" + + # Run identification + run_id: str + specialists_run: list[str] + + # Specialist outcomes + deduction_success: bool + induction_success: bool + + # Surprisal sampling + surprisal_enabled: bool + surprisal_conclusion_count: int + + # Aggregate metrics + total_iterations: int + total_duration_ms: float + input_tokens: int + output_tokens: int + + # Predefined probing questions to guide the specialists # These serve as semantic entry points for searching observations PROBING_QUESTIONS: list[str] = [ @@ -57,7 +87,7 @@ async def run_dream( observer: str, observed: str, session_name: str, -) -> None: +) -> DreamResult | None: """ Run a full dream cycle with optional surprisal-based sampling. @@ -74,7 +104,7 @@ async def run_dream( session_name: Session identifier """ if not settings.DREAM.ENABLED: - return + return None run_id = str(uuid.uuid4())[:8] task_name = f"dream_orchestrator_{run_id}" @@ -94,7 +124,14 @@ async def run_dream( logger.info( f"[{run_id}] Dreams disabled for {workspace_name}/{session_name}, skipping dream" ) - return + return None + + # Track specialist outcomes + deduction_success = False + induction_success = False + surprisal_observation_count = 0 + deduction_result: SpecialistResult | None = None + induction_result: SpecialistResult | None = None # Phase 0: Surprisal-based sampling (if enabled) probing_questions = PROBING_QUESTIONS # Default @@ -114,6 +151,7 @@ async def run_dream( logger.info( f"[{run_id}] Surprisal: Found {len(high_surprisal_obs)} high-surprisal observations" ) + surprisal_observation_count = len(high_surprisal_obs) accumulate_metric( task_name, "surprisal_observations", len(high_surprisal_obs), "count" ) @@ -166,9 +204,15 @@ async def run_dream( session_name=session_name, probing_questions=probing_questions, configuration=configuration, + parent_run_id=run_id, ) - logger.info(f"[{run_id}] Deduction completed: {deduction_result[:200]}...") - accumulate_metric(task_name, "deduction_result", deduction_result, "blob") + logger.info( + f"[{run_id}] Deduction completed: {deduction_result.content[:200]}..." + ) + accumulate_metric( + task_name, "deduction_result", deduction_result.content, "blob" + ) + deduction_success = deduction_result.success except SpecialistExecutionError as e: logger.error(f"[{run_id}] Deduction specialist failed: {e}", exc_info=True) accumulate_metric(task_name, "deduction_error", str(e), "blob") @@ -185,9 +229,15 @@ async def run_dream( session_name=session_name, probing_questions=probing_questions, configuration=configuration, + parent_run_id=run_id, ) - logger.info(f"[{run_id}] Induction completed: {induction_result[:200]}...") - accumulate_metric(task_name, "induction_result", induction_result, "blob") + logger.info( + f"[{run_id}] Induction completed: {induction_result.content[:200]}..." + ) + accumulate_metric( + task_name, "induction_result", induction_result.content, "blob" + ) + induction_success = induction_result.success except SpecialistExecutionError as e: logger.error(f"[{run_id}] Induction specialist failed: {e}", exc_info=True) accumulate_metric(task_name, "induction_error", str(e), "blob") @@ -199,6 +249,50 @@ async def run_dream( logger.info(f"[{run_id}] Dream cycle completed in {duration_ms:.0f}ms") log_performance_metrics("dream_orchestrator", run_id) + # Aggregate metrics from specialist results + total_iterations = (deduction_result.iterations if deduction_result else 0) + ( + induction_result.iterations if induction_result else 0 + ) + total_input_tokens = (deduction_result.input_tokens if deduction_result else 0) + ( + induction_result.input_tokens if induction_result else 0 + ) + total_output_tokens = ( + deduction_result.output_tokens if deduction_result else 0 + ) + (induction_result.output_tokens if induction_result else 0) + + # Emit DreamRunEvent with aggregated metrics + emit( + DreamRunEvent( + run_id=run_id, + workspace_name=workspace_name, + session_name=session_name, + observer=observer, + observed=observed, + specialists_run=["deduction", "induction"], + deduction_success=deduction_success, + induction_success=induction_success, + surprisal_enabled=settings.DREAM.SURPRISAL.ENABLED, + surprisal_conclusion_count=surprisal_observation_count, + total_iterations=total_iterations, + total_input_tokens=total_input_tokens, + total_output_tokens=total_output_tokens, + total_duration_ms=duration_ms, + ) + ) + + return DreamResult( + run_id=run_id, + specialists_run=["deduction", "induction"], + deduction_success=deduction_success, + induction_success=induction_success, + surprisal_enabled=settings.DREAM.SURPRISAL.ENABLED, + surprisal_conclusion_count=surprisal_observation_count, + total_iterations=total_iterations, + total_duration_ms=duration_ms, + input_tokens=total_input_tokens, + output_tokens=total_output_tokens, + ) + def _create_queries_from_surprisal( high_surprisal_obs: list[SurprisalScore], @@ -222,3 +316,51 @@ def _create_queries_from_surprisal( content = content[:200] + "..." queries.append(content) return queries[:10] # Limit to 10 queries + + +@sentry_sdk.trace +async def process_dream( + payload: DreamPayload, + workspace_name: str, +) -> None: + """ + Process a dream task by performing collection maintenance operations. + + Args: + payload: The dream task payload containing workspace, peer, and dream type information + """ + logger.info( + f""" +(っ- ‸ - ς)ᶻ z 𐰁 ᶻ z 𐰁 ᶻ z 𐰁\n +DREAM: {payload.dream_type} documents for {workspace_name}/{payload.observer}/{payload.observed}\n +𐰁 z ᶻ 𐰁 z ᶻ 𐰁 z ᶻ(っ- ‸ - ς)""" + ) + + try: + match payload.dream_type: + case DreamType.OMNI: + async with tracked_db("dream_orchestrator") as db: + result = await run_dream( + db=db, + workspace_name=workspace_name, + observer=payload.observer, + observed=payload.observed, + session_name=payload.session_name, + ) + + # Log completion (telemetry event already emitted in run_dream) + if result is not None: + logger.info( + f"Dream completed: run_id={result.run_id}, " + + f"iterations={result.total_iterations}, " + + f"duration={result.total_duration_ms:.0f}ms" + ) + + except Exception as e: + logger.error( + f"Error processing dream task {payload.dream_type} for {payload.observer}/{payload.observed}: {str(e)}", + exc_info=True, + ) + if settings.SENTRY.ENABLED: + sentry_sdk.capture_exception(e) + # Don't re-raise - we want to mark the dream task as processed even if it fails diff --git a/src/dreamer/specialists.py b/src/dreamer/specialists.py index 946dae89..369ee810 100644 --- a/src/dreamer/specialists.py +++ b/src/dreamer/specialists.py @@ -15,24 +15,42 @@ import time import uuid from abc import ABC, abstractmethod from collections.abc import Callable +from dataclasses import dataclass from typing import Any from sqlalchemy.ext.asyncio import AsyncSession -from src import prometheus from src.config import settings from src.schemas import ResolvedConfiguration +from src.telemetry import otel_metrics +from src.telemetry.events import DreamSpecialistEvent, emit +from src.telemetry.logging import accumulate_metric, log_performance_metrics +from src.telemetry.otel.metrics import TokenTypes from src.utils.agent_tools import ( DEDUCTION_SPECIALIST_TOOLS, INDUCTION_SPECIALIST_TOOLS, create_tool_executor, ) from src.utils.clients import HonchoLLMCallResponse, honcho_llm_call -from src.utils.logging import accumulate_metric, log_performance_metrics logger = logging.getLogger(__name__) +@dataclass +class SpecialistResult: + """Result of a specialist run for telemetry and aggregation.""" + + run_id: str + specialist_type: str + iterations: int + tool_calls_count: int + input_tokens: int + output_tokens: int + duration_ms: float + success: bool + content: str + + # Tool names to exclude when peer card creation is disabled PEER_CARD_TOOL_NAMES = {"update_peer_card", "get_peer_card"} @@ -81,7 +99,8 @@ class BaseSpecialist(ABC): session_name: str, probing_questions: list[str], configuration: ResolvedConfiguration | None = None, - ) -> str: + parent_run_id: str | None = None, + ) -> SpecialistResult: """ Run the specialist agent. @@ -93,11 +112,12 @@ class BaseSpecialist(ABC): session_name: Session identifier probing_questions: Entry point questions to guide exploration configuration: Resolved configuration for checking feature flags (optional) + parent_run_id: Optional run_id from orchestrator for correlation Returns: - Summary of work done + SpecialistResult with metrics and content """ - run_id = str(uuid.uuid4())[:8] + run_id = parent_run_id or str(uuid.uuid4())[:8] task_name = f"dreamer_{self.name}_{run_id}" start_time = time.perf_counter() @@ -115,7 +135,7 @@ class BaseSpecialist(ABC): {"role": "user", "content": self.build_user_prompt(probing_questions)}, ] - # Create tool executor + # Create tool executor with telemetry context tool_executor: Callable[ [str, dict[str, Any]], Any ] = await create_tool_executor( @@ -127,12 +147,22 @@ class BaseSpecialist(ABC): include_observation_ids=True, history_token_limit=settings.DREAM.HISTORY_TOKEN_LIMIT, configuration=configuration, + run_id=run_id, + agent_type=self.name, + parent_category="dream", ) # Get model with potential override model = self.get_model() llm_settings = settings.DREAM.model_copy(update={"MODEL": model}) + # Track iterations via callback + iteration_count = 0 + + def iteration_callback(data: Any) -> None: + nonlocal iteration_count + iteration_count = data.iteration + # Run the agent loop response: HonchoLLMCallResponse[str] = await honcho_llm_call( llm_settings=llm_settings, @@ -144,6 +174,7 @@ class BaseSpecialist(ABC): max_tool_iterations=self.get_max_iterations(), messages=messages, track_name=f"Dreamer/{self.name}", + iteration_callback=iteration_callback, ) # Log metrics @@ -155,16 +186,18 @@ class BaseSpecialist(ABC): accumulate_metric(task_name, "input_tokens", response.input_tokens, "count") accumulate_metric(task_name, "output_tokens", response.output_tokens, "count") - if prometheus.METRICS_ENABLED: - prometheus.DREAMER_TOKENS_PROCESSED.labels( + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_dreamer_tokens( + count=response.input_tokens, specialist_name=self.name, - token_type=prometheus.TokenTypes.INPUT.value, - ).inc(response.input_tokens) - - prometheus.DREAMER_TOKENS_PROCESSED.labels( + token_type=TokenTypes.INPUT.value, + ) + otel_metrics.record_dreamer_tokens( + count=response.output_tokens, specialist_name=self.name, - token_type=prometheus.TokenTypes.OUTPUT.value, - ).inc(response.output_tokens) + token_type=TokenTypes.OUTPUT.value, + ) logger.info( f"{self.name}: Completed in {duration_ms:.0f}ms, " @@ -174,7 +207,34 @@ class BaseSpecialist(ABC): log_performance_metrics(f"dreamer_{self.name}", run_id) - return response.content + # Emit telemetry event + emit( + DreamSpecialistEvent( + run_id=run_id, + specialist_type=self.name, + workspace_name=workspace_name, + observer=observer, + observed=observed, + iterations=iteration_count, + tool_calls_count=len(response.tool_calls_made), + input_tokens=response.input_tokens, + output_tokens=response.output_tokens, + duration_ms=duration_ms, + success=True, + ) + ) + + return SpecialistResult( + run_id=run_id, + specialist_type=self.name, + iterations=iteration_count, + tool_calls_count=len(response.tool_calls_made), + input_tokens=response.input_tokens, + output_tokens=response.output_tokens, + duration_ms=duration_ms, + success=True, + content=response.content, + ) class DeductionSpecialist(BaseSpecialist): diff --git a/src/main.py b/src/main.py index 223eb751..92a793a0 100644 --- a/src/main.py +++ b/src/main.py @@ -15,7 +15,6 @@ from pydantic import ValidationError from sentry_sdk.integrations.fastapi import FastApiIntegration from sentry_sdk.integrations.starlette import StarletteIntegration -from src import prometheus from src.cache.client import close_cache, init_cache from src.config import settings from src.db import engine, request_context @@ -30,8 +29,14 @@ from src.routers import ( workspaces, ) from src.security import create_admin_jwt -from src.sentry import initialize_sentry -from src.utils.logging import get_route_template +from src.telemetry import ( + initialize_telemetry, + initialize_telemetry_async, + otel_metrics, + shutdown_telemetry, +) +from src.telemetry.logging import get_route_template +from src.telemetry.sentry import initialize_sentry if TYPE_CHECKING: from sentry_sdk._types import Event, Hint @@ -115,6 +120,10 @@ if SENTRY_ENABLED: @asynccontextmanager async def lifespan(_: FastAPI): + # Initialize telemetry (OTel metrics + CloudEvents emitter) + initialize_telemetry() + await initialize_telemetry_async() + try: await init_cache() except Exception as e: @@ -131,6 +140,8 @@ async def lifespan(_: FastAPI): await close_external_vector_store() await close_cache() await engine.dispose() + # Shutdown telemetry (flush CloudEvents buffer, shutdown OTel metrics) + await shutdown_telemetry() app = FastAPI( @@ -180,23 +191,13 @@ app.include_router(conclusions.router, prefix="/v2") app.include_router(keys.router, prefix="/v2") app.include_router(webhooks.router, prefix="/v2") -app.add_api_route("/metrics", prometheus.metrics, methods=["GET"]) - # Global exception handlers @app.exception_handler(HonchoException) -async def honcho_exception_handler(request: Request, exc: HonchoException): +async def honcho_exception_handler(_request: Request, exc: HonchoException): """Handle all Honcho-specific exceptions.""" logger.error(f"{exc.__class__.__name__}: {exc.detail}", exc_info=exc) - if prometheus.METRICS_ENABLED and request.url.path != "/metrics": - template = get_route_template(request) - prometheus.API_REQUESTS.labels( - method=request.method, - endpoint=template, - status_code=str(exc.status_code), - ).inc() - return JSONResponse( status_code=exc.status_code, content={"detail": exc.detail}, @@ -204,18 +205,10 @@ async def honcho_exception_handler(request: Request, exc: HonchoException): @app.exception_handler(Exception) -async def global_exception_handler(request: Request, exc: Exception): +async def global_exception_handler(_request: Request, exc: Exception): """Handle all unhandled exceptions.""" logger.error(f"Unhandled exception: {str(exc)}", exc_info=True) - if prometheus.METRICS_ENABLED and request.url.path != "/metrics": - template = get_route_template(request) - prometheus.API_REQUESTS.labels( - method=request.method, - endpoint=template, - status_code="500", - ).inc() - if SENTRY_ENABLED: sentry_sdk.capture_exception(exc) return JSONResponse( @@ -239,14 +232,14 @@ async def track_request( try: response = await call_next(request) - # Track Prometheus metrics if enabled - if prometheus.METRICS_ENABLED and request.url.path != "/metrics": + # Track metrics if enabled + if settings.OTEL.ENABLED: template = get_route_template(request) - prometheus.API_REQUESTS.labels( + otel_metrics.record_api_request( method=request.method, endpoint=template, status_code=str(response.status_code), - ).inc() + ) return response finally: diff --git a/src/prometheus.py b/src/prometheus.py deleted file mode 100644 index 9e45927d..00000000 --- a/src/prometheus.py +++ /dev/null @@ -1,186 +0,0 @@ -""" -Prometheus metrics for Honcho. - -This module defines all Prometheus metrics for all Honcho processes and exposes them via the /metrics endpoint. -""" - -import logging -from enum import Enum -from typing import cast - -from prometheus_client import ( - CONTENT_TYPE_LATEST, - REGISTRY, - Counter, - disable_created_metrics, - generate_latest, -) -from starlette.responses import Response as StarletteResponse - -from src.config import settings - -METRICS_ENABLED = settings.METRICS.ENABLED -disable_created_metrics() # Disables _created metrics on counters, histograms, and summaries - -logger = logging.getLogger(__name__) - - -class NamespacedCounter(Counter): - """Counter that automatically includes namespace label""" - - def labels(self, **kwargs: str) -> "NamespacedCounter": - """Override labels to automatically appends namespace label""" - # METRICS.NAMESPACE is guaranteed to be non-None by AppSettings.propagate_namespace validator - kwargs["namespace"] = cast(str, settings.METRICS.NAMESPACE) - return super().labels(**kwargs) - - -# Tracks all requests to the Honcho API. -# -# Incremented in: src/main.py middleware for every request -# Labels: -# - method: HTTP method (GET, POST, PUT, DELETE, etc.) -# - endpoint: FastAPI route template (e.g., "/v2/workspaces/{workspace_id}/sessions/{session_id}/messages") -# - status_code: HTTP response status code (200, 404, 500, etc.) -API_REQUESTS = NamespacedCounter( - "api_requests_total", - "Total API requests", - [ - "namespace", - "method", - "endpoint", - "status_code", - ], -) - -# Tracks the total number of honcho messages created. -# -# Incremented in: src/routers/messages.py when messages are successfully created -# Labels: -# - workspace_name: The workspace where messages were created -MESSAGES_CREATED = NamespacedCounter( - "messages_created_total", - "Total messages created", - [ - "namespace", - "workspace_name", - ], -) - - -# Tracks the total number of dialectic calls made. -# -# Incremented in: src/routers/peers.py when successful dialectic calls are made -# Labels: -# - workspace_name: The workspace where the dialectic call was made -# - reasoning_level: The reasoning level used for the call -DIALECTIC_CALLS = NamespacedCounter( - "dialectic_calls_total", - "Total dialectic calls", - [ - "namespace", - "workspace_name", - "reasoning_level", - ], -) - -# Tracks the total number of queue items successfully processed by the deriver. -# -# Incremented in: src/deriver/queue_manager.py when queue items are processed -# Labels: -# - workspace_name: The workspace where items were processed -# - task_type: The type of task processed (e.g., "representation", "summary", "peer_card") -DERIVER_QUEUE_ITEMS_PROCESSED = NamespacedCounter( - "deriver_queue_items_processed_total", - "Total deriver queue items processed", - ["namespace", "workspace_name", "task_type"], -) - - -class TokenTypes(Enum): - INPUT = "input" - OUTPUT = "output" - - -# Tracks the total number of input and output tokens processed by the deriver. -# -# Incremented in: src/deriver/deriver.py after the critical analysis call is made -# Labels: -# - task_type: The type of task that processed the tokens -# - token_type: The type of tokens ("input" or "output") -# - component: The component of the input -DERIVER_TOKENS_PROCESSED = NamespacedCounter( - "deriver_tokens_processed_total", - "Total tokens processed by the deriver", - [ - "namespace", - "task_type", - "token_type", - "component", - ], -) - - -class DeriverTaskTypes(Enum): - INGESTION = "ingestion" - SUMMARY = "summary" - - -class DeriverComponents(Enum): - PROMPT = "prompt" # used in ingestion and summary - MESSAGES = "messages" # used in ingestion and summary - PREVIOUS_SUMMARY = "previous_summary" # only used for summary - OUTPUT_TOTAL = "output_total" - - -# Tracks the total number of input and output tokens processed by the dialectic. -# -# Incremented in: src/dialectic/core.py after the dialectic call is made -# Labels: -# - token_type: The type of tokens -# - component: The component of the input -# - reasoning_level: The reasoning level used for the call -DIALECTIC_TOKENS_PROCESSED = NamespacedCounter( - "dialectic_tokens_processed_total", - "Total tokens processed by the dialectic", - [ - "namespace", - "token_type", - "component", - "reasoning_level", - ], -) - - -class DialecticComponents(Enum): - TOTAL = "total" - - -# Tracks the total number of input and output tokens processed by the dreamer. -# -# Incremented in: src/dreamer/specialists.py after the specialist LLM call is made -# Labels: -# - specialist_name: The name of the specialist ("deduction" or "induction") -# - token_type: The type of tokens ("input" or "output") -DREAMER_TOKENS_PROCESSED = NamespacedCounter( - "dreamer_tokens_processed_total", - "Total tokens processed by the dreamer", - [ - "namespace", - "specialist_name", - "token_type", - ], -) - - -async def metrics() -> StarletteResponse: - """Prometheus metrics endpoint""" - if not settings.METRICS.ENABLED: - return StarletteResponse("Metrics are disabled", status_code=404) - try: - return StarletteResponse( - generate_latest(REGISTRY), media_type=CONTENT_TYPE_LATEST - ) - except Exception as e: - logger.error(f"Failed to generate metrics: {e}", exc_info=True) - return StarletteResponse("Failed to generate metrics", status_code=500) diff --git a/src/routers/messages.py b/src/routers/messages.py index cfebf424..d5746d12 100644 --- a/src/routers/messages.py +++ b/src/routers/messages.py @@ -16,12 +16,13 @@ from fastapi_pagination.ext.sqlalchemy import apaginate from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm.attributes import flag_modified -from src import crud, prometheus, schemas +from src import crud, schemas from src.config import settings from src.dependencies import db from src.deriver import enqueue from src.exceptions import FileTooLargeError, ResourceNotFoundException from src.security import require_auth +from src.telemetry import otel_metrics from src.utils.files import process_file_uploads_for_messages logger = logging.getLogger(__name__) @@ -99,9 +100,12 @@ async def create_messages_for_session( session_name=session_id, ) - prometheus.MESSAGES_CREATED.labels( - workspace_name=workspace_id, - ).inc(len(created_messages)) + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_messages_created( + count=len(created_messages), + workspace_name=workspace_id, + ) # Enqueue for processing (existing logic) payloads = [ @@ -194,9 +198,13 @@ async def create_messages_with_file( "Batch of %s messages created from file uploads and queued for processing", len(created_messages), ) - prometheus.MESSAGES_CREATED.labels( - workspace_name=workspace_id, - ).inc(len(created_messages)) + + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_messages_created( + count=len(created_messages), + workspace_name=workspace_id, + ) return created_messages diff --git a/src/routers/peers.py b/src/routers/peers.py index 4355a087..5c759ff2 100644 --- a/src/routers/peers.py +++ b/src/routers/peers.py @@ -8,12 +8,13 @@ from fastapi_pagination import Page from fastapi_pagination.ext.sqlalchemy import apaginate from sqlalchemy.ext.asyncio import AsyncSession -from src import crud, prometheus, schemas +from src import crud, schemas from src.config import settings from src.dependencies import db, tracked_db from src.dialectic.chat import agentic_chat, agentic_chat_stream from src.exceptions import AuthenticationException, ResourceNotFoundException from src.security import JWTParams, require_auth +from src.telemetry import otel_metrics from src.utils.search import search logger = logging.getLogger(__name__) @@ -183,11 +184,12 @@ async def chat( yield f"data: {json.dumps({'delta': {'content': chunk}, 'done': False})}\n\n" yield f"data: {json.dumps({'done': True})}\n\n" - if prometheus.METRICS_ENABLED: - prometheus.DIALECTIC_CALLS.labels( + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_dialectic_call( workspace_name=workspace_id, reasoning_level=options.reasoning_level, - ).inc() + ) return StreamingResponse( format_sse_stream( @@ -214,11 +216,12 @@ async def chat( reasoning_level=options.reasoning_level, ) - if prometheus.METRICS_ENABLED: - prometheus.DIALECTIC_CALLS.labels( + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_dialectic_call( workspace_name=workspace_id, reasoning_level=options.reasoning_level, - ).inc() + ) return schemas.DialecticResponse(content=str(response)) diff --git a/src/routers/workspaces.py b/src/routers/workspaces.py index 535ab886..9831092c 100644 --- a/src/routers/workspaces.py +++ b/src/routers/workspaces.py @@ -12,6 +12,7 @@ from src.dependencies import db from src.deriver.enqueue import enqueue_dream from src.exceptions import AuthenticationException from src.security import JWTParams, require_auth +from src.telemetry.events import DeletionCompletedEvent, emit from src.utils.search import search logger = logging.getLogger(__name__) @@ -114,7 +115,21 @@ async def delete_workspace( This action cannot be undone. """ - await crud.delete_workspace(db, workspace_name=workspace_id) + result = await crud.delete_workspace(db, workspace_name=workspace_id) + + # Emit telemetry event with cascade counts + emit( + DeletionCompletedEvent( + workspace_name=workspace_id, + deletion_type="workspace", + resource_id=workspace_id, + success=True, + peers_deleted=result.peers_deleted, + sessions_deleted=result.sessions_deleted, + messages_deleted=result.messages_deleted, + conclusions_deleted=result.conclusions_deleted, + ) + ) @router.post( diff --git a/src/sentry.py b/src/sentry.py deleted file mode 100644 index b11fe57c..00000000 --- a/src/sentry.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Sentry initialization and configuration.""" - -from __future__ import annotations - -import logging -from collections.abc import Sequence -from typing import TYPE_CHECKING - -import sentry_sdk - -from src.config import settings - -if TYPE_CHECKING: - from sentry_sdk._types import EventProcessor - from sentry_sdk.integrations import Integration - -logger = logging.getLogger(__name__) - - -# Sentry SDK's default behavior: -# - Captures INFO+ level logs as breadcrumbs -# - Captures ERROR+ level logs as Sentry events -# -# For custom log levels, use the LoggingIntegration class: -# sentry_sdk.init(..., integrations=[LoggingIntegration(level=logging.INFO, event_level=logging.ERROR)]) -def initialize_sentry( - *, - integrations: Sequence[Integration], - before_send: EventProcessor | None = None, -) -> None: - """Initialize Sentry SDK with project settings. - - Args: - integrations: Sentry SDK integrations to enable (e.g., Starlette, FastAPI). - before_send: Optional event filter callback to suppress specific exceptions. - """ - sentry_sdk.init( - dsn=settings.SENTRY.DSN, - enable_tracing=True, - release=settings.SENTRY.RELEASE, - environment=settings.SENTRY.ENVIRONMENT, - traces_sample_rate=settings.SENTRY.TRACES_SAMPLE_RATE, - profiles_sample_rate=settings.SENTRY.PROFILES_SAMPLE_RATE, - before_send=before_send, - integrations=integrations, - ) diff --git a/src/telemetry/__init__.py b/src/telemetry/__init__.py new file mode 100644 index 00000000..87c86c8c --- /dev/null +++ b/src/telemetry/__init__.py @@ -0,0 +1,85 @@ +""" +Telemetry module for Honcho. + +This module consolidates all telemetry, metrics, and observability functionality: +- Sentry: Error tracking and performance tracing +- OTel: Push-based metrics via OTLP to any compatible backend (e.g., Mimir) +- CloudEvents: Structured events for analytics (push-based) +- Logging: Langfuse integration, Rich console output, metric accumulation +- Tracing: Sentry transaction decorators +- Metrics Collector: JSON file-based benchmark aggregation +- Reasoning Traces: JSONL logging of LLM inputs/outputs +""" + +from src.telemetry.events import emit +from src.telemetry.otel import get_meter, initialize_otel_metrics, shutdown_otel_metrics +from src.telemetry.otel.metrics import otel_metrics + +__all__ = [ + "emit", + "get_meter", + "initialize_otel_metrics", + "initialize_telemetry", + "initialize_telemetry_async", + "otel_metrics", + "shutdown_otel_metrics", + "shutdown_telemetry", +] + + +def initialize_telemetry() -> None: + """ + Initialize all telemetry systems based on configuration. + + This should be called once at application startup (in main.py lifespan). + It reads configuration from settings and initializes: + - OTel metrics (if OTEL_ENABLED=true) + + Note: CloudEvents telemetry requires async initialization and should be + initialized separately using initialize_telemetry_async(). + + Sentry is initialized separately in sentry.py as it has its own lifecycle. + """ + from src.config import settings + + if settings.OTEL.ENABLED: + initialize_otel_metrics( + endpoint=settings.OTEL.ENDPOINT, + headers=settings.OTEL.HEADERS, + export_interval_millis=settings.OTEL.EXPORT_INTERVAL_MILLIS, + service_name=settings.OTEL.SERVICE_NAME, + service_namespace=settings.OTEL.SERVICE_NAMESPACE, + enabled=True, + ) + + +async def initialize_telemetry_async() -> None: + """ + Initialize async telemetry systems based on configuration. + + This should be called once at application startup (in main.py lifespan), + after initialize_telemetry(). It initializes: + - CloudEvents emitter (if TELEMETRY_ENABLED=true) + """ + from src.config import settings + from src.telemetry.events import initialize_telemetry_events + + if settings.TELEMETRY.ENABLED: + await initialize_telemetry_events() + + +async def shutdown_telemetry() -> None: + """ + Shutdown all telemetry systems gracefully. + + This should be called during application shutdown to ensure: + - OTel metrics are flushed + - CloudEvents buffer is flushed + """ + from src.telemetry.events import shutdown_telemetry_events + + # Shutdown CloudEvents emitter (flushes buffer) + await shutdown_telemetry_events() + + # Shutdown OTel metrics + shutdown_otel_metrics() diff --git a/src/telemetry/emitter.py b/src/telemetry/emitter.py new file mode 100644 index 00000000..56a70e7c --- /dev/null +++ b/src/telemetry/emitter.py @@ -0,0 +1,379 @@ +""" +Buffered CloudEvents HTTP emitter with retry logic. + +This module provides a TelemetryEmitter class that: +- Buffers events in memory for efficient batching +- Sends events as CloudEvents to a configurable HTTP endpoint +- Implements exponential backoff retry logic +- Supports graceful shutdown with flush +""" + +import asyncio +import contextlib +import json +import logging +from collections import deque +from typing import TYPE_CHECKING + +import httpx +from cloudevents.conversion import to_json # pyright: ignore[reportUnknownVariableType] +from cloudevents.http import CloudEvent + +if TYPE_CHECKING: + from src.telemetry.events.base import BaseEvent + +logger = logging.getLogger(__name__) + + +class TelemetryEmitter: + """Buffered, async CloudEvents emitter with retry logic. + + This emitter queues events and sends them in batches to reduce + network overhead. It supports automatic periodic flushing and + exponential backoff retry on failure. + + Usage: + emitter = TelemetryEmitter( + endpoint="https://telemetry.example.com/v1/events", + headers={"X-API-Key": "..."}, + ) + await emitter.start() + + # Queue events (non-blocking) + emitter.emit(my_event) + + # Graceful shutdown + await emitter.shutdown() + """ + + endpoint: str | None + headers: dict[str, str] + batch_size: int + flush_interval: float + flush_threshold: int + max_retries: int + max_buffer_size: int + enabled: bool + _buffer: deque[CloudEvent] + _flush_task: asyncio.Task[None] | None + _client: httpx.AsyncClient | None + _running: bool + _lock: asyncio.Lock + + def __init__( + self, + endpoint: str | None = None, + headers: dict[str, str] | None = None, + batch_size: int = 100, + flush_interval_seconds: float = 1.0, + flush_threshold: int = 50, + max_retries: int = 3, + max_buffer_size: int = 10000, + enabled: bool = True, + ): + """Initialize the telemetry emitter. + + Args: + endpoint: CloudEvents HTTP endpoint URL + headers: Optional HTTP headers for authentication + batch_size: Maximum events per batch + flush_interval_seconds: How often to flush the buffer + flush_threshold: Trigger flush when buffer reaches this size + max_retries: Maximum retry attempts on failure + max_buffer_size: Maximum events to buffer (oldest dropped if exceeded) + enabled: Whether emission is enabled + """ + self.endpoint = endpoint + self.headers = headers or {} + self.batch_size = batch_size + self.flush_interval = flush_interval_seconds + self.flush_threshold = flush_threshold + self.max_retries = max_retries + self.max_buffer_size = max_buffer_size + self.enabled = enabled and endpoint is not None + + self._buffer = deque(maxlen=max_buffer_size) + self._flush_task = None + self._client = None + self._running = False + self._lock = asyncio.Lock() + + async def start(self) -> None: + """Start the emitter background tasks. + + Creates the HTTP client and starts the periodic flush task. + """ + if not self.enabled: + logger.info("Telemetry emitter disabled (no endpoint or disabled)") + return + + self._client = httpx.AsyncClient( + timeout=30.0, + headers={ + "Content-Type": "application/cloudevents+json", + **self.headers, + }, + ) + self._running = True + self._flush_task = asyncio.create_task(self._periodic_flush()) + logger.info("Telemetry emitter started, endpoint: %s", self.endpoint) + + async def shutdown(self) -> None: + """Gracefully shutdown the emitter. + + Stops the periodic flush task, flushes remaining events, + and closes the HTTP client. + """ + if not self.enabled: + return + + self._running = False + + # Cancel the periodic flush task + if self._flush_task is not None: + self._flush_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._flush_task + + # Final flush of remaining events + await self.flush() + + # Close HTTP client + if self._client is not None: + await self._client.aclose() + self._client = None + + logger.info("Telemetry emitter shutdown complete") + + def emit(self, event: "BaseEvent") -> None: + """Queue an event for emission. Non-blocking. + + The event is converted to a CloudEvent and added to the buffer. + If the buffer is full, the oldest events are dropped. + Triggers an immediate flush when buffer reaches flush_threshold. + + Args: + event: The Honcho event to emit + """ + if not self.enabled: + return + + from src.config import settings + + # Generate deterministic event ID + event_id = event.generate_id() + + # Build source with namespace for tenant routing + # Format: /honcho/{namespace}/{category} or /honcho/{category} + namespace = settings.TELEMETRY.NAMESPACE + source = f"/honcho/{event.category()}" + if namespace: + source = f"/honcho/{namespace}/{event.category()}" + + # Build CloudEvent attributes + attributes: dict[str, str] = { + "id": event_id, + "source": source, + "type": event.event_type(), + "time": event.timestamp.isoformat(), + "dataschema": f"https://honcho.dev/schemas/{event.event_type()}/v{event.schema_version()}", + } + + # Create CloudEvent + cloud_event = CloudEvent(attributes, event.model_dump(mode="json")) + + self._buffer.append(cloud_event) + buffer_size = len(self._buffer) + logger.debug("Queued event %s (buffer size: %d)", event_id, buffer_size) + + # Warning logs as buffer approaches max capacity + capacity_ratio = buffer_size / self.max_buffer_size + if capacity_ratio >= 0.8: + logger.warning( + "Telemetry buffer at %.0f%% capacity (%d/%d events)", + capacity_ratio * 100, + buffer_size, + self.max_buffer_size, + ) + + logger.debug("Event added to emitter (buffer size: %d)", buffer_size) + # Threshold-based flush trigger + if buffer_size >= self.flush_threshold and self._running: + logger.debug("Triggering flush (buffer size: %d)", buffer_size) + asyncio.create_task(self.flush()) + + async def flush(self) -> None: + """Flush buffered events to the endpoint. + + Sends events in batches up to batch_size. Uses exponential + backoff retry on failure. Events are returned to the buffer + on permanent failure. + """ + if not self.enabled or not self._buffer or self._client is None: + return + + async with self._lock: + while self._buffer: + # Extract a batch + batch: list[CloudEvent] = [] + while self._buffer and len(batch) < self.batch_size: + batch.append(self._buffer.popleft()) + + if not batch: + break + + # Try to send the batch + success = await self._send_batch(batch) + if not success: + # Put events back at the front of the buffer + for event in reversed(batch): + self._buffer.appendleft(event) + logger.warning( + "Failed to send batch of %d events, returned to buffer", + len(batch), + ) + break + + async def _send_batch(self, batch: list[CloudEvent]) -> bool: + """Send a batch of events to the endpoint with retry logic. + + Args: + batch: List of CloudEvents to send + + Returns: + True if successful, False if all retries failed + """ + if self._client is None or self.endpoint is None: + return False + + for attempt in range(self.max_retries): + try: + # Convert batch to JSON + # For a single event, use structured format + # For multiple events, use batch format (array) + if len(batch) == 1: + body = to_json(batch[0]) + else: + # CloudEvents batch format is a JSON array + events_json = [json.loads(to_json(e)) for e in batch] + body = json.dumps(events_json).encode() + + response = await self._client.post( + self.endpoint, + content=body, + headers=( + {"Content-Type": "application/cloudevents-batch+json"} + if len(batch) > 1 + else None + ), + ) + response.raise_for_status() + + logger.debug( + "Sent batch of %d events (status: %d)", + len(batch), + response.status_code, + ) + return True + + except httpx.HTTPStatusError as e: + logger.warning( + "HTTP error sending events (attempt %d/%d): %s", + attempt + 1, + self.max_retries, + e, + ) + except httpx.RequestError as e: + logger.warning( + "Request error sending events (attempt %d/%d): %s", + attempt + 1, + self.max_retries, + e, + ) + + # Exponential backoff + if attempt < self.max_retries - 1: + delay = 2**attempt + await asyncio.sleep(delay) + + return False + + async def _periodic_flush(self) -> None: + """Background task that periodically flushes the buffer.""" + while self._running: + try: + await asyncio.sleep(self.flush_interval) + if self._buffer: + await self.flush() + except asyncio.CancelledError: + break + except Exception as e: + logger.exception("Error in periodic flush: %s", e) + + @property + def buffer_size(self) -> int: + """Return the current number of buffered events.""" + return len(self._buffer) + + @property + def is_running(self) -> bool: + """Return whether the emitter is running.""" + return self._running + + +# Global emitter instance (initialized by initialize_telemetry_events) +_emitter: TelemetryEmitter | None = None + + +def get_emitter() -> TelemetryEmitter | None: + """Get the global telemetry emitter instance.""" + return _emitter + + +async def initialize_emitter( + endpoint: str | None = None, + headers: dict[str, str] | None = None, + batch_size: int = 100, + flush_interval_seconds: float = 1.0, + flush_threshold: int = 50, + max_retries: int = 3, + max_buffer_size: int = 10000, + enabled: bool = True, +) -> TelemetryEmitter: + """Initialize and start the global telemetry emitter. + + Args: + endpoint: CloudEvents HTTP endpoint URL + headers: Optional HTTP headers for authentication + batch_size: Maximum events per batch + flush_interval_seconds: How often to flush the buffer + flush_threshold: Trigger flush when buffer reaches this size + max_retries: Maximum retry attempts on failure + max_buffer_size: Maximum events to buffer + enabled: Whether emission is enabled + + Returns: + The initialized emitter instance + """ + global _emitter + + _emitter = TelemetryEmitter( + endpoint=endpoint, + headers=headers, + batch_size=batch_size, + flush_interval_seconds=flush_interval_seconds, + flush_threshold=flush_threshold, + max_retries=max_retries, + max_buffer_size=max_buffer_size, + enabled=enabled, + ) + await _emitter.start() + return _emitter + + +async def shutdown_emitter() -> None: + """Shutdown the global telemetry emitter.""" + global _emitter + if _emitter is not None: + await _emitter.shutdown() + _emitter = None diff --git a/src/telemetry/events/__init__.py b/src/telemetry/events/__init__.py new file mode 100644 index 00000000..697ecec8 --- /dev/null +++ b/src/telemetry/events/__init__.py @@ -0,0 +1,189 @@ +""" +CloudEvents telemetry event system for Honcho. + +This module provides: +- emit(): Queue an event for emission to the telemetry backend +- BaseEvent: Base class for all event types +- Concrete event types for all Honcho operations + +Event Categories: + representation: Message processing and conclusion extraction + - RepresentationCompletedEvent: Message batch processed, conclusions extracted + + dream: Memory consolidation operations + - DreamRunEvent: Full dream orchestration completed + - DreamSpecialistEvent: Individual specialist (deduction/induction) completed + + dialectic: User-initiated queries + - DialecticCompletedEvent: Chat query completed + + agent: Agentic loop tracking (correlates via run_id) + - AgentIterationEvent: Per-LLM-call metrics + - AgentToolConclusionsCreatedEvent: Conclusions created by agent + - AgentToolConclusionsDeletedEvent: Conclusions deleted by agent + - AgentToolPeerCardUpdatedEvent: Peer card updated by agent + - AgentToolSummaryCreatedEvent: Summary created + + deletion: Resource removal + - DeletionCompletedEvent: Resource deletion completed (with cascade counts) + + reconciliation: Maintenance operations + - SyncVectorsCompletedEvent: Vector store sync completed + - CleanupStaleItemsCompletedEvent: Stale items cleanup completed + +Usage: + from src.telemetry.events import emit, RepresentationCompletedEvent + + emit(RepresentationCompletedEvent( + workspace_name="my_workspace", + session_name="my_session", + observed="assistant_peer", + queue_items_processed=3, + earliest_message_id="msg_001", + latest_message_id="msg_010", + message_count=10, + explicit_conclusion_count=5, + context_preparation_ms=50.0, + llm_call_ms=1200.0, + total_duration_ms=1300.0, + input_tokens=5000, + output_tokens=500, + )) +""" + +import logging + +from src.telemetry.events.agent import ( + AgentIterationEvent, + AgentToolConclusionsCreatedEvent, + AgentToolConclusionsDeletedEvent, + AgentToolPeerCardUpdatedEvent, + AgentToolSummaryCreatedEvent, +) +from src.telemetry.events.base import BaseEvent, generate_event_id +from src.telemetry.events.deletion import DeletionCompletedEvent +from src.telemetry.events.dialectic import DialecticCompletedEvent +from src.telemetry.events.dream import ( + DreamRunEvent, + DreamSpecialistEvent, +) +from src.telemetry.events.reconciliation import ( + CleanupStaleItemsCompletedEvent, + SyncVectorsCompletedEvent, +) +from src.telemetry.events.representation import RepresentationCompletedEvent + +logger = logging.getLogger(__name__) + +__all__ = [ + # Core + "emit", + "BaseEvent", + "generate_event_id", + # Representation events + "RepresentationCompletedEvent", + # Dream events + "DreamRunEvent", + "DreamSpecialistEvent", + # Dialectic events + "DialecticCompletedEvent", + # Agent events + "AgentIterationEvent", + "AgentToolConclusionsCreatedEvent", + "AgentToolConclusionsDeletedEvent", + "AgentToolPeerCardUpdatedEvent", + "AgentToolSummaryCreatedEvent", + # Reconciliation events + "SyncVectorsCompletedEvent", + "CleanupStaleItemsCompletedEvent", + # Deletion events + "DeletionCompletedEvent", + # Lifecycle + "initialize_telemetry_events", + "shutdown_telemetry_events", +] + + +def emit(event: BaseEvent) -> None: + """Queue an event for emission to the telemetry backend. + + This is the main entry point for emitting telemetry events. + Events are buffered and sent asynchronously to the configured endpoint. + + If telemetry is disabled or not initialized, this is a no-op. + Telemetry failures are caught and logged to Sentry to avoid blocking operations. + + Args: + event: The event to emit (must be a BaseEvent subclass instance) + + Example: + from src.telemetry.events import emit + + emit(MyEvent( + my_field="some_value", + ... + )) + """ + try: + from src.telemetry.emitter import get_emitter + + emitter = get_emitter() + if emitter is None: + logger.debug("Telemetry emitter not initialized, dropping event") + return + + emitter.emit(event) + except Exception as e: + # Log to Sentry but don't block the main operation + import sentry_sdk + + sentry_sdk.capture_exception(e) + logger.warning( + "Failed to emit telemetry event %s: %s", + type(event).__name__, + str(e), + ) + + +async def initialize_telemetry_events() -> None: + """Initialize the telemetry events system based on configuration. + + This should be called once at application startup. It reads + configuration from settings and initializes the CloudEvents emitter. + + This is typically called from initialize_telemetry() in the main + telemetry module. + """ + from src.config import settings + from src.telemetry.emitter import initialize_emitter + + if not settings.TELEMETRY.ENABLED: + logger.info("CloudEvents telemetry disabled") + return + + await initialize_emitter( + endpoint=settings.TELEMETRY.ENDPOINT, + headers=settings.TELEMETRY.HEADERS, + batch_size=settings.TELEMETRY.BATCH_SIZE, + flush_interval_seconds=settings.TELEMETRY.FLUSH_INTERVAL_SECONDS, + flush_threshold=settings.TELEMETRY.FLUSH_THRESHOLD, + max_retries=settings.TELEMETRY.MAX_RETRIES, + max_buffer_size=settings.TELEMETRY.MAX_BUFFER_SIZE, + enabled=True, + ) + + logger.info( + "CloudEvents telemetry initialized, endpoint: %s", settings.TELEMETRY.ENDPOINT + ) + + +async def shutdown_telemetry_events() -> None: + """Shutdown the telemetry events system. + + This should be called during application shutdown to ensure + all buffered events are flushed before exit. + """ + from src.telemetry.emitter import shutdown_emitter + + await shutdown_emitter() + logger.info("CloudEvents telemetry shutdown complete") diff --git a/src/telemetry/events/agent.py b/src/telemetry/events/agent.py new file mode 100644 index 00000000..77abe2c5 --- /dev/null +++ b/src/telemetry/events/agent.py @@ -0,0 +1,233 @@ +""" +Agent events for Honcho telemetry. + +Agent events track per-iteration metrics and state-changing tool executions +within agentic loops (dream specialists, dialectic agents). These events +correlate with parent events via run_id for detailed analytics. + +Events in this module: +- AgentIterationEvent: Per-LLM-call metrics within an agent run +- AgentToolConclusionsCreatedEvent: Conclusions created by agent tool +- AgentToolConclusionsDeletedEvent: Conclusions deleted by agent tool +- AgentToolPeerCardUpdatedEvent: Peer card updated by agent tool +- AgentToolSummaryCreatedEvent: Summary created by agent tool (future) +""" + +from typing import ClassVar + +from pydantic import Field + +from src.telemetry.events.base import BaseEvent + + +class AgentIterationEvent(BaseEvent): + """Emitted after each LLM call within an agentic loop. + + Tracks per-iteration token usage and tool calls for detailed cost analysis. + Correlates with parent events (dream.run, dream.specialist, dialectic.completed) + via run_id. + """ + + _event_type: ClassVar[str] = "agent.iteration" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "agent" + + # Run identification + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + + # Context + parent_category: str = Field( + ..., description="Parent category: 'dream' or 'dialectic'" + ) + agent_type: str = Field( + ..., description="Agent type: 'deduction', 'induction', or 'dialectic'" + ) + workspace_name: str = Field(..., description="Workspace name") + + # Peer context (for dream agents) + observer: str | None = Field(default=None, description="Observer peer name") + observed: str | None = Field(default=None, description="Observed peer name") + + # Peer context (for dialectic agent) + peer_name: str | None = Field(default=None, description="Peer name being queried") + + # Iteration info + iteration: int = Field(..., description="Iteration number (1-indexed)") + + # What happened in this iteration + tool_calls: list[str] = Field( + default_factory=list, + description="Tool names called in this iteration (can be empty or multiple)", + ) + + # Token usage for this single LLM call + input_tokens: int = Field(..., description="Input tokens for this iteration") + output_tokens: int = Field(..., description="Output tokens for this iteration") + cache_read_tokens: int = Field( + default=0, description="Tokens read from prompt cache" + ) + cache_creation_tokens: int = Field( + default=0, description="Tokens written to prompt cache" + ) + + def get_resource_id(self) -> str: + """Resource ID includes run_id and iteration for uniqueness.""" + return f"{self.run_id}:{self.iteration}" + + +class AgentToolConclusionsCreatedEvent(BaseEvent): + """Emitted when the create_conclusions tool is executed. + + Tracks conclusion creation with level breakdown for analytics. + """ + + _event_type: ClassVar[str] = "agent.tool.conclusions.created" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "agent" + + # Run identification + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + iteration: int = Field(..., description="Iteration number when this occurred") + + # Context + parent_category: str = Field( + ..., description="Parent category: 'dream' or 'dialectic'" + ) + agent_type: str = Field( + ..., description="Agent type: 'deduction', 'induction', or 'dialectic'" + ) + workspace_name: str = Field(..., description="Workspace name") + + # Peer context + observer: str = Field(..., description="Observer peer name") + observed: str = Field(..., description="Observed peer name") + + # What was created + conclusion_count: int = Field(..., description="Number of conclusions created") + levels: list[str] = Field( + default_factory=list, + description="Level of each conclusion (e.g., ['explicit', 'deductive', 'deductive'])", + ) + + def get_resource_id(self) -> str: + """Resource ID includes run_id and iteration for uniqueness.""" + return f"{self.run_id}:{self.iteration}:conclusions_created" + + +class AgentToolConclusionsDeletedEvent(BaseEvent): + """Emitted when the delete_conclusions tool is executed. + + Tracks conclusion deletion for memory consolidation analytics. + """ + + _event_type: ClassVar[str] = "agent.tool.conclusions.deleted" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "agent" + + # Run identification + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + iteration: int = Field(..., description="Iteration number when this occurred") + + # Context + parent_category: str = Field(..., description="Parent category (typically 'dream')") + agent_type: str = Field( + ..., description="Agent type (typically 'deduction' for deletions)" + ) + workspace_name: str = Field(..., description="Workspace name") + + # Peer context + observer: str = Field(..., description="Observer peer name") + observed: str = Field(..., description="Observed peer name") + + # What was deleted + conclusion_count: int = Field(..., description="Number of conclusions deleted") + + def get_resource_id(self) -> str: + """Resource ID includes run_id and iteration for uniqueness.""" + return f"{self.run_id}:{self.iteration}:conclusions_deleted" + + +class AgentToolPeerCardUpdatedEvent(BaseEvent): + """Emitted when the update_peer_card tool is executed. + + Tracks peer card updates during dream consolidation. + """ + + _event_type: ClassVar[str] = "agent.tool.peer_card.updated" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "agent" + + # Run identification + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + iteration: int = Field(..., description="Iteration number when this occurred") + + # Context + parent_category: str = Field(..., description="Parent category (typically 'dream')") + agent_type: str = Field(..., description="Agent type: 'deduction' or 'induction'") + workspace_name: str = Field(..., description="Workspace name") + + # Peer context + observer: str = Field(..., description="Observer peer name") + observed: str = Field(..., description="Observed peer name") + + # What was updated + facts_count: int = Field(..., description="Number of facts in the peer card") + + def get_resource_id(self) -> str: + """Resource ID includes run_id and iteration for uniqueness.""" + return f"{self.run_id}:{self.iteration}:peer_card_updated" + + +class AgentToolSummaryCreatedEvent(BaseEvent): + """Emitted when a summary is created. + + Tracks summary creation with full context about what was summarized + and the resources consumed. + """ + + _event_type: ClassVar[str] = "agent.tool.summary.created" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "agent" + + # Run identification (may be placeholder if not from an agentic loop) + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + iteration: int = Field(..., description="Iteration number when this occurred") + + # Context + parent_category: str = Field(..., description="Parent category") + agent_type: str = Field(..., description="Agent type") + workspace_name: str = Field(..., description="Workspace name") + + # Session context + session_name: str = Field(..., description="Session name") + + # Message context - what was summarized + message_id: str = Field(..., description="Message ID the summary covers up to") + message_count: int = Field( + ..., description="Number of messages included in summary" + ) + message_seq_in_session: int = Field( + ..., description="Sequence number of the base message in session" + ) + + # Summary details + summary_type: str = Field(..., description="Summary type: 'short' or 'long'") + + # Token usage + input_tokens: int = Field( + ..., description="Input tokens used for summary generation" + ) + output_tokens: int = Field(..., description="Output tokens (summary token count)") + + def get_resource_id(self) -> str: + """Resource ID includes run_id and iteration for uniqueness.""" + return f"{self.run_id}:{self.iteration}:summary_created" + + +__all__ = [ + "AgentIterationEvent", + "AgentToolConclusionsCreatedEvent", + "AgentToolConclusionsDeletedEvent", + "AgentToolPeerCardUpdatedEvent", + "AgentToolSummaryCreatedEvent", +] diff --git a/src/telemetry/events/base.py b/src/telemetry/events/base.py new file mode 100644 index 00000000..ea2371d9 --- /dev/null +++ b/src/telemetry/events/base.py @@ -0,0 +1,97 @@ +""" +Base event classes and idempotency key generation for CloudEvents telemetry. + +This module provides: +- BaseEvent: Abstract base class for all Honcho telemetry events +- generate_event_id(): Deterministic event ID generation for idempotency +""" + +import base64 +import hashlib +from datetime import UTC, datetime +from typing import ClassVar + +from pydantic import BaseModel, Field + + +def generate_event_id( + event_type: str, + timestamp: datetime, + resource_id: str, +) -> str: + """Generate a deterministic event ID for idempotency. + + Same inputs always produce the same ID, so retries are automatically + deduplicated on the receiving end. + + Args: + event_type: The CloudEvents type (e.g., "honcho.work.representation.completed") + timestamp: When the event occurred + resource_id: A unique identifier for the resource (can include workspace_id if relevant) + + Returns: + A deterministic event ID in the format "evt_{base64_hash}" + """ + payload = f"{event_type}:{resource_id}:{timestamp.isoformat()}" + hash_bytes = hashlib.sha256(payload.encode()).digest()[:16] + # Use URL-safe base64 encoding, strip padding + encoded = base64.urlsafe_b64encode(hash_bytes).decode().rstrip("=") + return f"evt_{encoded}" + + +class BaseEvent(BaseModel): + """Base class for all Honcho telemetry events. + + All events inherit from this class and define: + - _event_type: The CloudEvents type string + - _schema_version: Integer version for schema evolution + - _category: Event category (work, activity, resource) + + Subclasses must implement the abstract class methods by setting class variables + and providing get_resource_id(). Subclasses define their own context fields + (e.g., workspace_id, workspace_name) as needed - not all events have workspace context. + """ + + # Class variables for event metadata (set by subclasses) + _event_type: ClassVar[str] + _schema_version: ClassVar[int] + _category: ClassVar[str] # "work", "activity", or "resource" + + # Common timestamp field present in all events + timestamp: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="When the event occurred (UTC)", + ) + + @classmethod + def event_type(cls) -> str: + """Return the CloudEvents type string for this event.""" + return cls._event_type + + @classmethod + def schema_version(cls) -> int: + """Return the schema version for this event.""" + return cls._schema_version + + @classmethod + def category(cls) -> str: + """Return the event category (work, activity, or resource).""" + return cls._category + + def get_resource_id(self) -> str: + """Return the resource ID for idempotency key generation. + + This should be a unique identifier for the specific operation, + such as a task_id or a hash of message IDs. + + Must be implemented by subclasses. + """ + raise NotImplementedError("Subclasses must implement get_resource_id()") + + def generate_id(self) -> str: + """Generate a deterministic event ID for this event instance.""" + return generate_event_id( + event_type=self.event_type(), + timestamp=self.timestamp, + resource_id=self.get_resource_id(), + ) diff --git a/src/telemetry/events/deletion.py b/src/telemetry/events/deletion.py new file mode 100644 index 00000000..eed336a1 --- /dev/null +++ b/src/telemetry/events/deletion.py @@ -0,0 +1,65 @@ +""" +Deletion events for Honcho telemetry. + +Deletion tasks handle async removal of resources like workspaces, sessions, +and conclusions, ensuring proper cleanup across all storage layers. +""" + +from typing import ClassVar + +from pydantic import Field + +from src.telemetry.events.base import BaseEvent + + +class DeletionCompletedEvent(BaseEvent): + """Emitted when a deletion task completes. + + Deletion tasks handle async removal of workspaces, sessions, and conclusions, + ensuring proper cleanup across all storage layers. For workspace deletions, + cascade counts track how many child resources were also deleted. + """ + + _event_type: ClassVar[str] = "deletion.completed" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "deletion" + + # Workspace context + workspace_name: str = Field(..., description="Workspace name") + + # Deletion details + deletion_type: str = Field( + ..., description="Type of deletion: 'workspace', 'session', or 'conclusions'" + ) + resource_id: str = Field(..., description="ID of the deleted resource") + + # Outcome + success: bool = Field(..., description="Whether deletion succeeded") + + # Cascade counts (populated for workspace and session deletions) + peers_deleted: int = Field( + default=0, description="Number of peers deleted (workspace deletion only)" + ) + sessions_deleted: int = Field( + default=0, description="Number of sessions deleted (workspace deletion only)" + ) + messages_deleted: int = Field( + default=0, + description="Number of messages deleted (workspace or session deletion)", + ) + conclusions_deleted: int = Field( + default=0, + description="Number of conclusions/documents deleted (workspace or session deletion)", + ) + + # Optional error info + error_message: str | None = Field( + default=None, description="Error message if deletion failed" + ) + + def get_resource_id(self) -> str: + """Resource ID includes workspace, type, and resource for uniqueness.""" + return f"{self.workspace_name}:{self.deletion_type}:{self.resource_id}" + + +__all__ = ["DeletionCompletedEvent"] diff --git a/src/telemetry/events/dialectic.py b/src/telemetry/events/dialectic.py new file mode 100644 index 00000000..89ab9044 --- /dev/null +++ b/src/telemetry/events/dialectic.py @@ -0,0 +1,75 @@ +""" +Dialectic events for Honcho telemetry. + +Dialectic tasks answer questions about peers by gathering context from memory. +These are user-initiated operations that query the representation system. +The run_id field enables correlation with agent.iteration events. +""" + +from typing import ClassVar + +from pydantic import Field + +from src.telemetry.events.base import BaseEvent + + +class DialecticCompletedEvent(BaseEvent): + """Emitted when a dialectic (chat) query completes. + + Dialectic queries answer questions about peers by gathering context + from memory. This event captures the full context of the query and + its execution metrics. + + The run_id correlates with AgentIterationEvent and AgentTool* events + for detailed analytics. + """ + + _event_type: ClassVar[str] = "dialectic.completed" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "dialectic" + + # Run identification (for correlating with iteration/tool events) + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + + # Workspace context + workspace_name: str = Field(..., description="Workspace name") + + # Peer context + peer_name: str = Field(..., description="Peer name being queried about") + + # Session context (optional - dialectic can run without session) + session_name: str | None = Field( + default=None, description="Session name if provided" + ) + + # Query configuration + reasoning_level: str = Field( + ..., description="Reasoning level: minimal, low, medium, high, max" + ) + + # Execution metrics + total_iterations: int = Field(default=1, description="Number of LLM iterations") + prefetched_conclusion_count: int = Field( + default=0, description="Number of conclusions prefetched" + ) + tool_calls_count: int = Field(default=0, description="Number of tool calls made") + + # Timing metrics (milliseconds) + total_duration_ms: float = Field(..., description="Total processing time") + + # Token usage with cache breakdown + input_tokens: int = Field(..., description="Total input tokens") + output_tokens: int = Field(..., description="Output tokens generated") + cache_read_tokens: int = Field( + default=0, description="Tokens read from prompt cache" + ) + cache_creation_tokens: int = Field( + default=0, description="Tokens written to prompt cache" + ) + + def get_resource_id(self) -> str: + """Resource ID is the run_id for uniqueness.""" + return self.run_id + + +__all__ = ["DialecticCompletedEvent"] diff --git a/src/telemetry/events/dream.py b/src/telemetry/events/dream.py new file mode 100644 index 00000000..2b65a2ea --- /dev/null +++ b/src/telemetry/events/dream.py @@ -0,0 +1,119 @@ +""" +Dream events for Honcho telemetry. + +Dream tasks handle memory consolidation through specialist agents (deduction, induction). +Events track both the overall dream run and individual specialist executions, +with run_id correlation for analytics queries. +""" + +from typing import ClassVar + +from pydantic import Field + +from src.telemetry.events.base import BaseEvent + + +class DreamRunEvent(BaseEvent): + """Emitted when a full dream orchestration completes. + + This is the top-level event for a dream run, providing aggregate metrics + across all specialists. Individual specialist details are captured in + DreamSpecialistEvent with the same run_id for correlation. + """ + + _event_type: ClassVar[str] = "dream.run" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "dream" + + # Run identification (for correlating with specialist/iteration/tool events) + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + + # Workspace context + workspace_name: str = Field(..., description="Workspace name") + + # Session context + session_name: str = Field(..., description="Most recent session name") + + # Peer context + observer: str = Field(..., description="Observer peer name") + observed: str = Field(..., description="Observed peer name") + + # What ran + specialists_run: list[str] = Field( + ..., description="Specialists executed (e.g., ['deduction', 'induction'])" + ) + deduction_success: bool = Field( + ..., description="Whether deduction specialist succeeded" + ) + induction_success: bool = Field( + ..., description="Whether induction specialist succeeded" + ) + + # Surprisal sampling (optional phase) + surprisal_enabled: bool = Field( + default=False, description="Whether surprisal sampling was enabled" + ) + surprisal_conclusion_count: int = Field( + default=0, description="High-surprisal conclusions found" + ) + + # Aggregated metrics across all specialists + total_iterations: int = Field( + ..., description="Total LLM iterations across all specialists" + ) + total_input_tokens: int = Field( + ..., description="Total input tokens across all specialists" + ) + total_output_tokens: int = Field( + ..., description="Total output tokens across all specialists" + ) + total_duration_ms: float = Field(..., description="Total processing time") + + def get_resource_id(self) -> str: + """Resource ID is the run_id for uniqueness.""" + return self.run_id + + +class DreamSpecialistEvent(BaseEvent): + """Emitted when an individual dream specialist completes. + + Each specialist (deduction, induction) emits its own event with + the same run_id as the parent DreamRunEvent for correlation. + """ + + _event_type: ClassVar[str] = "dream.specialist" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "dream" + + # Run identification (correlates with parent dream.run) + run_id: str = Field(..., description="8-char UUID prefix for run correlation") + + # Specialist info + specialist_type: str = Field( + ..., description="Specialist type: 'deduction' or 'induction'" + ) + + # Workspace context + workspace_name: str = Field(..., description="Workspace name") + + # Peer context + observer: str = Field(..., description="Observer peer name") + observed: str = Field(..., description="Observed peer name") + + # Execution metrics + iterations: int = Field(..., description="Number of LLM iterations") + tool_calls_count: int = Field(..., description="Total tool calls made") + input_tokens: int = Field(..., description="Input tokens used") + output_tokens: int = Field(..., description="Output tokens generated") + duration_ms: float = Field(..., description="Processing time") + success: bool = Field(..., description="Whether the specialist succeeded") + + def get_resource_id(self) -> str: + """Resource ID includes run_id and specialist type for uniqueness.""" + return f"{self.run_id}:{self.specialist_type}" + + +__all__ = [ + "DreamRunEvent", + "DreamSpecialistEvent", +] diff --git a/src/telemetry/events/reconciliation.py b/src/telemetry/events/reconciliation.py new file mode 100644 index 00000000..2f8e7aca --- /dev/null +++ b/src/telemetry/events/reconciliation.py @@ -0,0 +1,88 @@ +""" +Reconciliation events for Honcho telemetry. + +Reconciliation tasks handle maintenance operations: +- Sync vectors: Synchronize documents and message embeddings to external vector stores +- Cleanup stale items: Clean up soft-deleted records and expired queue items +""" + +from typing import ClassVar + +from pydantic import Field + +from src.telemetry.events.base import BaseEvent + + +class SyncVectorsCompletedEvent(BaseEvent): + """Emitted when a vector sync cycle completes. + + Vector sync tasks synchronize documents and message embeddings to external + vector stores. These run periodically and operate across all workspaces. + + Note: This event has no workspace context as it operates globally. + """ + + _event_type: ClassVar[str] = "reconciliation.sync_vectors.completed" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "reconciliation" + + # Document metrics + documents_synced: int = Field( + default=0, description="Documents successfully synced to vector store" + ) + documents_failed: int = Field( + default=0, description="Documents that failed to sync" + ) + documents_cleaned: int = Field( + default=0, description="Soft-deleted documents cleaned up during sync" + ) + + # Message embedding metrics + message_embeddings_synced: int = Field( + default=0, description="Message embeddings successfully synced" + ) + message_embeddings_failed: int = Field( + default=0, description="Message embeddings that failed to sync" + ) + + # Timing metrics (milliseconds) + total_duration_ms: float = Field(..., description="Total processing time") + + def get_resource_id(self) -> str: + """Resource ID is fixed for this global operation.""" + return "sync_vectors" + + +class CleanupStaleItemsCompletedEvent(BaseEvent): + """Emitted when a stale items cleanup cycle completes. + + Cleanup tasks remove soft-deleted documents and expired queue items. + These run periodically and operate across all workspaces. + + Note: This event has no workspace context as it operates globally. + """ + + _event_type: ClassVar[str] = "reconciliation.cleanup_stale_items.completed" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "reconciliation" + + # Cleanup metrics + documents_cleaned: int = Field( + default=0, description="Soft-deleted documents cleaned up" + ) + queue_items_cleaned: int = Field( + default=0, description="Expired queue items cleaned up" + ) + + # Timing metrics (milliseconds) + total_duration_ms: float = Field(..., description="Total processing time") + + def get_resource_id(self) -> str: + """Resource ID is fixed for this global operation.""" + return "cleanup_stale_items" + + +__all__ = [ + "SyncVectorsCompletedEvent", + "CleanupStaleItemsCompletedEvent", +] diff --git a/src/telemetry/events/representation.py b/src/telemetry/events/representation.py new file mode 100644 index 00000000..5dff20f2 --- /dev/null +++ b/src/telemetry/events/representation.py @@ -0,0 +1,68 @@ +""" +Representation events for Honcho telemetry. + +Representation tasks extract conclusions from messages to build peer +representations. These events track the processing of message batches +and the conclusions extracted from them. +""" + +from typing import ClassVar + +from pydantic import Field + +from src.telemetry.events.base import BaseEvent + + +class RepresentationCompletedEvent(BaseEvent): + """Emitted when a representation task completes processing a message batch. + + Representation tasks extract conclusions from messages to build peer + representations. This event captures the full context of what was processed + and what was extracted. + """ + + _event_type: ClassVar[str] = "representation.completed" + _schema_version: ClassVar[int] = 1 + _category: ClassVar[str] = "representation" + + # Workspace context + workspace_name: str = Field(..., description="Workspace name") + + # Session context + session_name: str = Field(..., description="Session name") + + # Peer context + observed: str = Field(..., description="Observed peer name") + + # Queue processing info + queue_items_processed: int = Field( + ..., description="Number of QueueItem records dequeued and processed" + ) + + # Message batch info + earliest_message_id: str = Field(..., description="First message ID in batch") + latest_message_id: str = Field(..., description="Last message ID in batch") + message_count: int = Field(..., description="Number of messages processed") + + # Conclusion counts + explicit_conclusion_count: int = Field( + ..., description="Number of explicit conclusions extracted" + ) + + # Timing metrics (milliseconds) + context_preparation_ms: float = Field( + ..., description="Time spent preparing context" + ) + llm_call_ms: float = Field(..., description="Time spent in LLM call") + total_duration_ms: float = Field(..., description="Total processing time") + + # Token usage + input_tokens: int = Field(..., description="Input tokens used") + output_tokens: int = Field(..., description="Output tokens generated") + + def get_resource_id(self) -> str: + """Resource ID includes workspace, session, and latest message for uniqueness.""" + return f"{self.workspace_name}:{self.session_name}:{self.latest_message_id}" + + +__all__ = ["RepresentationCompletedEvent"] diff --git a/src/utils/logging.py b/src/telemetry/logging.py similarity index 84% rename from src/utils/logging.py rename to src/telemetry/logging.py index 287faab2..0796d22f 100644 --- a/src/utils/logging.py +++ b/src/telemetry/logging.py @@ -5,6 +5,7 @@ and a conditional observe decorator that only applies when Langfuse is configure """ import datetime +from collections import OrderedDict from collections.abc import Callable from typing import ParamSpec, TypeVar, overload @@ -18,7 +19,7 @@ from rich.text import Text from rich.tree import Tree from src.config import settings -from src.utils.metrics_collector import append_metrics_to_file +from src.telemetry.metrics_collector import append_metrics_to_file from src.utils.representation import ( Representation, ) @@ -80,8 +81,16 @@ def conditional_observe( return decorator -# dict[task_name, list[tuple[metric_name, metric_value, metric_unit]]] -accumulated_metrics: dict[str, list[tuple[str, str | int | float, str]]] = {} +# Bounded OrderedDict for accumulated metrics to prevent memory leaks. +# If an exception occurs between accumulate_metric() and log_performance_metrics(), +# the metrics would stay in memory forever. Using OrderedDict allows us to evict +# the oldest entries (FIFO) when exceeding MAX_ACCUMULATED_TASKS. +MAX_ACCUMULATED_TASKS = 1000 + +# OrderedDict[task_name, list[tuple[metric_name, metric_value, metric_unit]]] +accumulated_metrics: OrderedDict[str, list[tuple[str, str | int | float, str]]] = ( + OrderedDict() +) def format_reasoning_inputs_as_markdown( @@ -157,11 +166,25 @@ def accumulate_metric( ) -> None: """ Accumulate a metric value to be printed the next time log_performance_metrics is called. + + This function uses a bounded OrderedDict to prevent memory leaks. If the number of + tracked tasks exceeds MAX_ACCUMULATED_TASKS, the oldest entries are evicted (FIFO). + Args: + task_name: Task identifier label: Metric label value: Metric value unit: Metric unit """ + # Evict oldest entries if we've exceeded the maximum to prevent memory leaks. + # This handles the case where exceptions occur before log_performance_metrics is called. + while ( + len(accumulated_metrics) >= MAX_ACCUMULATED_TASKS + and task_name not in accumulated_metrics + ): + # popitem(last=False) removes the oldest (first inserted) entry + accumulated_metrics.popitem(last=False) + accumulated_metrics.setdefault(task_name, []).append((label, value, unit)) @@ -225,12 +248,14 @@ def log_performance_metrics( title: Table title """ task_name = f"{task_slug}_{task_name}" + # No-op if metrics were evicted (due to MAX_ACCUMULATED_TASKS limit) and no + # additional metrics are passed. This handles the case where an exception + # caused old metrics to be evicted before log_performance_metrics was called. if not accumulated_metrics.get(task_name) and not metrics: return if metrics is None: metrics = [] - metrics = accumulated_metrics.get(task_name, []) + metrics - accumulated_metrics[task_name].clear() + metrics = accumulated_metrics.pop(task_name, []) + metrics if COLLECT_METRICS_LOCAL: append_metrics_to_file(task_slug, task_name, metrics) diff --git a/src/utils/metrics_collector.py b/src/telemetry/metrics_collector.py similarity index 100% rename from src/utils/metrics_collector.py rename to src/telemetry/metrics_collector.py diff --git a/src/telemetry/otel/__init__.py b/src/telemetry/otel/__init__.py new file mode 100644 index 00000000..08c6ede8 --- /dev/null +++ b/src/telemetry/otel/__init__.py @@ -0,0 +1,26 @@ +""" +OpenTelemetry metrics module for Honcho. + +This module provides push-based metrics using OpenTelemetry SDK +with OTLP HTTP export to any compatible backend (Mimir, Grafana Cloud, etc.). +""" + +from src.telemetry.otel.metrics import ( + DeriverComponents, + DeriverTaskTypes, + DialecticComponents, + TokenTypes, + get_meter, + initialize_otel_metrics, + shutdown_otel_metrics, +) + +__all__ = [ + "DeriverComponents", + "DeriverTaskTypes", + "DialecticComponents", + "TokenTypes", + "get_meter", + "initialize_otel_metrics", + "shutdown_otel_metrics", +] diff --git a/src/telemetry/otel/metrics.py b/src/telemetry/otel/metrics.py new file mode 100644 index 00000000..0c3ec7c7 --- /dev/null +++ b/src/telemetry/otel/metrics.py @@ -0,0 +1,481 @@ +""" +OpenTelemetry metrics implementation with OTLP export. + +This module provides push-based metrics that replace the pull-based Prometheus +/metrics endpoint. Metrics are pushed via OTLP to any compatible backend +(Mimir, Grafana Cloud, etc.) at configurable intervals. + +Usage: + from src.telemetry.otel import get_meter, initialize_otel_metrics + + # Initialize once at startup + initialize_otel_metrics() + + # Get a meter for your component + meter = get_meter("honcho.deriver") + + # Create instruments + counter = meter.create_counter("tokens_processed", unit="tokens") + counter.add(100, {"task_type": "ingestion"}) +""" + +from __future__ import annotations + +import atexit +import logging +from enum import Enum +from typing import TYPE_CHECKING, final + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource + +if TYPE_CHECKING: + from opentelemetry.metrics import Counter, Meter + +logger = logging.getLogger(__name__) + +# ============================================================================= +# Metric label enums +# ============================================================================= + + +class TokenTypes(Enum): + INPUT = "input" + OUTPUT = "output" + + +class DeriverTaskTypes(Enum): + INGESTION = "ingestion" + SUMMARY = "summary" + + +class DeriverComponents(Enum): + PROMPT = "prompt" # used in ingestion and summary + MESSAGES = "messages" # used in ingestion and summary + PREVIOUS_SUMMARY = "previous_summary" # only used for summary + OUTPUT_TOTAL = "output_total" + + +class DialecticComponents(Enum): + TOTAL = "total" + + +# ============================================================================= +# OTel metrics infrastructure +# ============================================================================= + +# Global state +_meter_provider: MeterProvider | None = None +_initialized: bool = False + + +def initialize_otel_metrics( + *, + endpoint: str | None = None, + headers: dict[str, str] | None = None, + export_interval_millis: int = 60000, + service_name: str = "honcho", + service_namespace: str | None = None, + enabled: bool = True, +) -> None: + """ + Initialize OpenTelemetry metrics with OTLP export. + + This should be called once at application startup. If already initialized, + subsequent calls are no-ops. + + Args: + endpoint: OTLP HTTP endpoint URL (e.g., "https://mimir.example.com/otlp/v1/metrics"). + If None, metrics are collected but not exported (useful for testing). + headers: Optional headers to include in requests (e.g., {"X-Scope-OrgID": "tenant"}). + export_interval_millis: How often to export metrics (default: 60 seconds). + service_name: Service name for resource attributes (default: "honcho"). + service_namespace: Optional namespace for the service. + enabled: If False, metrics are no-ops (default: True). + """ + global _meter_provider, _initialized + + if _initialized: + logger.debug("OTel metrics already initialized, skipping") + return + + if not enabled: + logger.info("OTel metrics disabled") + _initialized = True + return + + # Build resource attributes + resource_attributes = { + "service.name": service_name, + } + if service_namespace: + resource_attributes["service.namespace"] = service_namespace + + resource = Resource.create(resource_attributes) + + # Create metric reader + readers: list[PeriodicExportingMetricReader] = [] + + if endpoint: + try: + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + + exporter = OTLPMetricExporter( + endpoint=endpoint, + headers=headers or {}, + ) + reader = PeriodicExportingMetricReader( + exporter, + export_interval_millis=export_interval_millis, + ) + readers.append(reader) + logger.info(f"OTel metrics configured to push via OTLP to {endpoint}") + except Exception as e: + logger.error(f"Failed to configure OTLP metrics exporter: {e}") + # Continue without exporter - metrics still work locally + else: + logger.info( + "OTel metrics initialized without remote export (no endpoint configured)" + ) + + # Create and set the meter provider + # Note: empty list is valid for metric_readers (metrics still work, just not exported) + _meter_provider = MeterProvider( + resource=resource, + metric_readers=readers, + ) + metrics.set_meter_provider(_meter_provider) + + # Register shutdown handler + atexit.register(shutdown_otel_metrics) + + _initialized = True + logger.info("OTel metrics initialized successfully") + + +def shutdown_otel_metrics() -> None: + """ + Shutdown the OTel metrics provider, flushing any pending metrics. + + This is automatically called at process exit via atexit, but can be + called manually for graceful shutdown. + """ + global _meter_provider, _initialized + + if _meter_provider is not None: + try: + _meter_provider.shutdown() + logger.info("OTel metrics shutdown complete") + except Exception as e: + logger.error(f"Error during OTel metrics shutdown: {e}") + finally: + _meter_provider = None + _initialized = False + + +def get_meter(name: str, version: str = "") -> Meter: + """ + Get an OTel Meter for creating instruments. + + Args: + name: The name of the instrumentation scope (e.g., "honcho.deriver"). + version: Optional version of the instrumentation scope. + + Returns: + An OTel Meter instance for creating counters, histograms, etc. + + Example: + meter = get_meter("honcho.deriver") + counter = meter.create_counter("tokens_processed", unit="tokens") + counter.add(100, {"task_type": "ingestion"}) + """ + return metrics.get_meter(name, version) + + +# ============================================================================= +# Pre-defined metrics that mirror existing Prometheus counters +# ============================================================================= + +# These are created lazily on first use to avoid issues with initialization order + + +@final +class OTelMetrics: + """ + Container for OTel metrics that mirror existing Prometheus counters. + + This class provides a bridge during migration - the same metrics are + available via both Prometheus (pull) and OTel (push). + + Namespace is managed at the instance level (from settings), not per-call. + """ + + _instance: OTelMetrics | None = None + _is_initialized: bool = False + _namespace: str = "honcho" + + # Meters (lazily initialized) + _api_meter: Meter | None = None + _deriver_meter: Meter | None = None + _dialectic_meter: Meter | None = None + _dreamer_meter: Meter | None = None + + # Counters (lazily initialized) + _api_requests: Counter | None = None + _messages_created: Counter | None = None + _dialectic_calls: Counter | None = None + _deriver_queue_items: Counter | None = None + _deriver_tokens: Counter | None = None + _dialectic_tokens: Counter | None = None + _dreamer_tokens: Counter | None = None + + def __new__(cls) -> OTelMetrics: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def _ensure_initialized(self) -> None: + """Lazily initialize meters and instruments.""" + if self._is_initialized: + return + + # Get namespace from settings (same as Prometheus) + from src.config import settings + + self._namespace = settings.OTEL.SERVICE_NAMESPACE or "honcho" + + # Get meters for different components + self._api_meter = get_meter("honcho.api") + self._deriver_meter = get_meter("honcho.deriver") + self._dialectic_meter = get_meter("honcho.dialectic") + self._dreamer_meter = get_meter("honcho.dreamer") + + # Create counters that mirror Prometheus metrics + # API requests + self._api_requests = self._api_meter.create_counter( + name="api_requests", + unit="requests", + description="Total API requests", + ) + + # Messages created + self._messages_created = self._api_meter.create_counter( + name="messages_created", + unit="messages", + description="Total messages created", + ) + + # Dialectic calls + self._dialectic_calls = self._dialectic_meter.create_counter( + name="dialectic_calls", + unit="calls", + description="Total dialectic calls", + ) + + # Deriver queue items processed + self._deriver_queue_items = self._deriver_meter.create_counter( + name="deriver_queue_items_processed", + unit="items", + description="Total deriver queue items processed", + ) + + # Token counters + self._deriver_tokens = self._deriver_meter.create_counter( + name="deriver_tokens_processed", + unit="tokens", + description="Total tokens processed by the deriver", + ) + + self._dialectic_tokens = self._dialectic_meter.create_counter( + name="dialectic_tokens_processed", + unit="tokens", + description="Total tokens processed by the dialectic", + ) + + self._dreamer_tokens = self._dreamer_meter.create_counter( + name="dreamer_tokens_processed", + unit="tokens", + description="Total tokens processed by the dreamer", + ) + + self._is_initialized = True + + def _handle_metric_error(self, method_name: str, error: Exception) -> None: + """Handle errors from metric recording by logging to Sentry.""" + import sentry_sdk + + sentry_sdk.capture_exception(error) + logger.warning( + "Failed to record OTel metric in %s: %s", method_name, str(error) + ) + + def record_api_request( + self, + *, + method: str, + endpoint: str, + status_code: str, + ) -> None: + """Record an API request metric.""" + try: + self._ensure_initialized() + if self._api_requests is None: + return # Not initialized, skip silently + self._api_requests.add( + 1, + { + "method": method, + "endpoint": endpoint, + "status_code": status_code, + "namespace": self._namespace, + }, + ) + except Exception as e: + self._handle_metric_error("record_api_request", e) + + def record_messages_created( + self, + *, + count: int, + workspace_name: str, + ) -> None: + """Record messages created metric.""" + try: + self._ensure_initialized() + if self._messages_created is None: + return # Not initialized, skip silently + self._messages_created.add( + count, + { + "workspace_name": workspace_name, + "namespace": self._namespace, + }, + ) + except Exception as e: + self._handle_metric_error("record_messages_created", e) + + def record_dialectic_call( + self, + *, + workspace_name: str, + reasoning_level: str, + ) -> None: + """Record a dialectic call metric.""" + try: + self._ensure_initialized() + if self._dialectic_calls is None: + return # Not initialized, skip silently + self._dialectic_calls.add( + 1, + { + "workspace_name": workspace_name, + "reasoning_level": reasoning_level, + "namespace": self._namespace, + }, + ) + except Exception as e: + self._handle_metric_error("record_dialectic_call", e) + + def record_deriver_queue_item( + self, + *, + count: int, + workspace_name: str, + task_type: str, + ) -> None: + """Record deriver queue items processed metric.""" + try: + self._ensure_initialized() + if self._deriver_queue_items is None: + return # Not initialized, skip silently + self._deriver_queue_items.add( + count, + { + "workspace_name": workspace_name, + "task_type": task_type, + "namespace": self._namespace, + }, + ) + except Exception as e: + self._handle_metric_error("record_deriver_queue_item", e) + + def record_deriver_tokens( + self, + *, + count: int, + task_type: str, + token_type: str, + component: str, + ) -> None: + """Record deriver token usage metric.""" + try: + self._ensure_initialized() + if self._deriver_tokens is None: + return # Not initialized, skip silently + self._deriver_tokens.add( + count, + { + "task_type": task_type, + "token_type": token_type, + "component": component, + "namespace": self._namespace, + }, + ) + except Exception as e: + self._handle_metric_error("record_deriver_tokens", e) + + def record_dialectic_tokens( + self, + *, + count: int, + token_type: str, + component: str, + reasoning_level: str, + ) -> None: + """Record dialectic token usage metric.""" + try: + self._ensure_initialized() + if self._dialectic_tokens is None: + return # Not initialized, skip silently + self._dialectic_tokens.add( + count, + { + "token_type": token_type, + "component": component, + "reasoning_level": reasoning_level, + "namespace": self._namespace, + }, + ) + except Exception as e: + self._handle_metric_error("record_dialectic_tokens", e) + + def record_dreamer_tokens( + self, + *, + count: int, + specialist_name: str, + token_type: str, + ) -> None: + """Record dreamer token usage metric.""" + try: + self._ensure_initialized() + if self._dreamer_tokens is None: + return # Not initialized, skip silently + self._dreamer_tokens.add( + count, + { + "specialist_name": specialist_name, + "token_type": token_type, + "namespace": self._namespace, + }, + ) + except Exception as e: + self._handle_metric_error("record_dreamer_tokens", e) + + +# Singleton instance +otel_metrics = OTelMetrics() diff --git a/src/utils/reasoning_traces.py b/src/telemetry/reasoning_traces.py similarity index 100% rename from src/utils/reasoning_traces.py rename to src/telemetry/reasoning_traces.py diff --git a/src/telemetry/sentry.py b/src/telemetry/sentry.py new file mode 100644 index 00000000..1f6a6bd8 --- /dev/null +++ b/src/telemetry/sentry.py @@ -0,0 +1,90 @@ +"""Sentry initialization and configuration.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Callable, Sequence +from functools import wraps +from typing import TYPE_CHECKING, ParamSpec, TypeVar, cast + +import sentry_sdk + +from src.config import settings + +P = ParamSpec("P") +T = TypeVar("T") + +if TYPE_CHECKING: + from sentry_sdk._types import EventProcessor + from sentry_sdk.integrations import Integration + +logger = logging.getLogger(__name__) + + +# Sentry SDK's default behavior: +# - Captures INFO+ level logs as breadcrumbs +# - Captures ERROR+ level logs as Sentry events +# +# For custom log levels, use the LoggingIntegration class: +# sentry_sdk.init(..., integrations=[LoggingIntegration(level=logging.INFO, event_level=logging.ERROR)]) +def initialize_sentry( + *, + integrations: Sequence[Integration], + before_send: EventProcessor | None = None, +) -> None: + """Initialize Sentry SDK with project settings. + + Args: + integrations: Sentry SDK integrations to enable (e.g., Starlette, FastAPI). + before_send: Optional event filter callback to suppress specific exceptions. + """ + sentry_sdk.init( + dsn=settings.SENTRY.DSN, + enable_tracing=True, + release=settings.SENTRY.RELEASE, + environment=settings.SENTRY.ENVIRONMENT, + traces_sample_rate=settings.SENTRY.TRACES_SAMPLE_RATE, + profiles_sample_rate=settings.SENTRY.PROFILES_SAMPLE_RATE, + before_send=before_send, + integrations=integrations, + ) + + +def with_sentry_transaction( + name: str, op: str +) -> Callable[[Callable[P, T]], Callable[P, T]]: + """Decorator to wrap a function in a Sentry transaction. + + Args: + name: The name of the transaction. + op: The operation type (e.g., "deriver", "http.server"). + + Returns: + A decorator that wraps the target function in a Sentry transaction. + + Example: + @with_sentry_transaction("process_task", op="deriver") + def process_task(data): + ... + """ + + def decorator(func: Callable[P, T]) -> Callable[P, T]: + if asyncio.iscoroutinefunction(func): + + @wraps(func) + async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + with sentry_sdk.start_transaction(name=name, op=op): + return await func(*args, **kwargs) + + return cast(Callable[P, T], async_wrapper) + else: + + @wraps(func) + def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + with sentry_sdk.start_transaction(name=name, op=op): + return func(*args, **kwargs) + + return sync_wrapper + + return decorator diff --git a/src/utils/agent_tools.py b/src/utils/agent_tools.py index 62faea4f..a392532c 100644 --- a/src/utils/agent_tools.py +++ b/src/utils/agent_tools.py @@ -3,7 +3,7 @@ import logging from collections.abc import Callable from dataclasses import dataclass from datetime import datetime -from typing import Any +from typing import Any, cast from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -13,13 +13,20 @@ from src.config import settings from src.embedding_client import embedding_client from src.models import Document from src.schemas import ResolvedConfiguration +from src.telemetry.events import ( + AgentToolConclusionsCreatedEvent, + AgentToolConclusionsDeletedEvent, + AgentToolPeerCardUpdatedEvent, + emit, +) from src.utils import summarizer from src.utils.formatting import format_new_turn_with_timestamp, utc_now_iso from src.utils.representation import Representation -from src.utils.types import DocumentLevel +from src.utils.types import DocumentLevel, get_current_iteration logger = logging.getLogger(__name__) + # Module-level lock registry for thread-safe observation creation. # Keyed by (workspace_name, observer, observed) to ensure all tool executors # operating on the same data share the same lock. @@ -887,6 +894,10 @@ class ToolContext: db_lock: asyncio.Lock # Optional resolved configuration for checking feature flags configuration: ResolvedConfiguration | None = None + # Telemetry context fields + run_id: str | None = None + agent_type: str | None = None # "dialectic", "deriver", "dreamer" + parent_category: str | None = None # Parent category for CloudEvents async def _handle_create_observations( @@ -996,6 +1007,23 @@ async def _handle_create_observations( contradiction_count = sum( 1 for o in observations if o.get("level") == "contradiction" ) + + # Emit telemetry event if context is available + if ctx.run_id and ctx.agent_type and ctx.parent_category: + emit( + AgentToolConclusionsCreatedEvent( + run_id=ctx.run_id, + iteration=get_current_iteration(), + parent_category=ctx.parent_category, + agent_type=ctx.agent_type, + workspace_name=ctx.workspace_name, + observer=ctx.observer, + observed=ctx.observed, + conclusion_count=len(observations), + levels=[o.get("level", "explicit") for o in observations], + ) + ) + return f"Created {len(observations)} observations for {ctx.observed} by {ctx.observer} ({explicit_count} explicit, {deductive_count} deductive, {inductive_count} inductive, {contradiction_count} contradiction)" @@ -1010,17 +1038,44 @@ async def _handle_update_peer_card(ctx: ToolContext, tool_input: dict[str, Any]) "Peer card creation is disabled for this workspace/session configuration." ) + peer_card_content = tool_input["content"] async with ctx.db_lock: await crud.set_peer_card( ctx.db, workspace_name=ctx.workspace_name, - peer_card=tool_input["content"], + peer_card=peer_card_content, observer=ctx.observer, observed=ctx.observed, ) logger.info( f"Updated peer card for {ctx.workspace_name}/{ctx.observer}/{ctx.observed}" ) + + # Emit telemetry event if context is available + if ctx.run_id and ctx.agent_type and ctx.parent_category: + # Count facts in peer card (content is a list of strings per tool schema) + facts_count: int + if isinstance(peer_card_content, list): + content_list = cast(list[str], peer_card_content) + facts_count = len([line for line in content_list if line.strip()]) + else: + # Fallback for string (defensive) + facts_count = len( + [line for line in str(peer_card_content).split("\n") if line.strip()] + ) + emit( + AgentToolPeerCardUpdatedEvent( + run_id=ctx.run_id, + iteration=get_current_iteration(), + parent_category=ctx.parent_category, + agent_type=ctx.agent_type, + workspace_name=ctx.workspace_name, + observer=ctx.observer, + observed=ctx.observed, + facts_count=facts_count, + ) + ) + return f"Updated peer card for {ctx.observed} by {ctx.observer}" @@ -1370,6 +1425,22 @@ async def _handle_delete_observations( deleted_count += 1 except Exception as e: logger.warning(f"Failed to delete observation {obs_id}: {e}") + + # Emit telemetry event if context is available + if deleted_count > 0 and ctx.run_id and ctx.agent_type and ctx.parent_category: + emit( + AgentToolConclusionsDeletedEvent( + run_id=ctx.run_id, + iteration=get_current_iteration(), + parent_category=ctx.parent_category, + agent_type=ctx.agent_type, + workspace_name=ctx.workspace_name, + observer=ctx.observer, + observed=ctx.observed, + conclusion_count=deleted_count, + ) + ) + return f"Deleted {deleted_count} observations" @@ -1557,6 +1628,9 @@ async def create_tool_executor( include_observation_ids: bool = False, history_token_limit: int = 8192, configuration: ResolvedConfiguration | None = None, + run_id: str | None = None, + agent_type: str | None = None, + parent_category: str | None = None, ) -> Callable[[str, dict[str, Any]], Any]: """ Create a unified tool executor function for all agent operations. @@ -1574,6 +1648,9 @@ async def create_tool_executor( include_observation_ids: If True, include observation IDs in output (for dreamer agent) history_token_limit: Maximum tokens for get_recent_history (default: 8192) configuration: Resolved configuration for checking feature flags (optional) + run_id: Optional run ID for telemetry correlation + agent_type: Optional agent type for telemetry (dialectic, deriver, dreamer) + parent_category: Optional parent category for CloudEvents Returns: An async callable that executes tools with the captured context @@ -1593,6 +1670,9 @@ async def create_tool_executor( history_token_limit=history_token_limit, db_lock=shared_lock, configuration=configuration, + run_id=run_id, + agent_type=agent_type, + parent_category=parent_category, ) async def execute_tool(tool_name: str, tool_input: dict[str, Any]) -> str: diff --git a/src/utils/clients.py b/src/utils/clients.py index 2edc1a28..5ce820f7 100644 --- a/src/utils/clients.py +++ b/src/utils/clients.py @@ -2,6 +2,7 @@ import json import logging from collections.abc import AsyncIterator, Callable from contextvars import ContextVar +from dataclasses import dataclass from typing import Any, Generic, Literal, TypeVar, cast, overload from anthropic import AsyncAnthropic @@ -22,15 +23,37 @@ from sentry_sdk.ai.monitoring import ai_track from tenacity import retry, stop_after_attempt, wait_exponential from src.config import LLMComponentSettings, settings +from src.telemetry.logging import conditional_observe +from src.telemetry.reasoning_traces import log_reasoning_trace from src.utils.json_parser import validate_and_repair_json -from src.utils.logging import conditional_observe -from src.utils.reasoning_traces import log_reasoning_trace from src.utils.representation import PromptRepresentation from src.utils.tokens import estimate_tokens -from src.utils.types import SupportedProviders +from src.utils.types import SupportedProviders, set_current_iteration logger = logging.getLogger(__name__) + +@dataclass +class IterationData: + """Data passed to iteration callbacks after each tool execution loop iteration.""" + + iteration: int + """1-indexed iteration number.""" + tool_calls: list[str] + """List of tool names called in this iteration.""" + input_tokens: int + """Input tokens used in this iteration's LLM call.""" + output_tokens: int + """Output tokens generated in this iteration's LLM call.""" + cache_read_tokens: int = 0 + """Tokens read from cache in this iteration.""" + cache_creation_tokens: int = 0 + """Tokens written to cache in this iteration.""" + + +# Type alias for iteration callback +IterationCallback = Callable[[IterationData], None] + T = TypeVar("T") # Type aliases for OpenAI GPT-5 specific parameters @@ -467,6 +490,8 @@ class HonchoLLMCallResponse(BaseModel, Generic[T]): cache_read_input_tokens: int = 0 finish_reasons: list[str] tool_calls_made: list[dict[str, Any]] = Field(default_factory=list) + iterations: int = 0 + """Number of LLM calls made in the tool execution loop (1 = single response, 2+ = tool use iterations plus final synthesis).""" thinking_content: str | None = None # Full thinking blocks with signatures for multi-turn conversation replay (Anthropic only) thinking_blocks: list[dict[str, Any]] = Field(default_factory=list) @@ -506,6 +531,7 @@ class StreamingResponseWithMetadata: cache_creation_input_tokens: int cache_read_input_tokens: int thinking_content: str | None + iterations: int def __init__( self, @@ -516,6 +542,7 @@ class StreamingResponseWithMetadata: cache_creation_input_tokens: int, cache_read_input_tokens: int, thinking_content: str | None = None, + iterations: int = 0, ): self._stream = stream self.tool_calls_made = tool_calls_made @@ -524,6 +551,7 @@ class StreamingResponseWithMetadata: self.cache_creation_input_tokens = cache_creation_input_tokens self.cache_read_input_tokens = cache_read_input_tokens self.thinking_content = thinking_content + self.iterations = iterations def __aiter__(self) -> AsyncIterator[HonchoLLMCallStreamChunk]: return self._stream.__aiter__() @@ -628,6 +656,7 @@ async def _execute_tool_loop( ], before_retry_callback: Callable[[Any], None], stream_final: bool = False, + iteration_callback: IterationCallback | None = None, ) -> HonchoLLMCallResponse[Any] | StreamingResponseWithMetadata: """ Execute the tool calling loop for agentic LLM interactions. @@ -660,6 +689,7 @@ async def _execute_tool_loop( get_provider_and_model: Function to get current provider/model based on attempt before_retry_callback: Callback for retry events stream_final: If True, stream the final response instead of returning it synchronously + iteration_callback: Optional callback invoked after each iteration with IterationData Returns: Final HonchoLLMCallResponse with accumulated token counts and tool call history, @@ -771,6 +801,7 @@ async def _execute_tool_loop( cache_creation_input_tokens=total_cache_creation_tokens, cache_read_input_tokens=total_cache_read_tokens, thinking_content=response.thinking_content, + iterations=iteration + 1, ) response.tool_calls_made = all_tool_calls @@ -778,6 +809,7 @@ async def _execute_tool_loop( response.output_tokens = total_output_tokens response.cache_creation_input_tokens = total_cache_creation_tokens response.cache_read_input_tokens = total_cache_read_tokens + response.iterations = iteration + 1 return response # Determine which provider we're using (reuse the helper) @@ -793,6 +825,9 @@ async def _execute_tool_loop( ) conversation_messages.append(assistant_message) + # Set current iteration for telemetry context (1-indexed) + set_current_iteration(iteration + 1) + # Execute tools and add results tool_results: list[dict[str, Any]] = [] for tool_call in response.tool_calls_made: @@ -837,6 +872,21 @@ async def _execute_tool_loop( # Add tool result message in provider-specific format _append_tool_results(current_provider, tool_results, conversation_messages) + # Call iteration callback if provided + if iteration_callback is not None: + try: + iteration_data = IterationData( + iteration=iteration + 1, # 1-indexed + tool_calls=[tc["name"] for tc in response.tool_calls_made], + input_tokens=response.input_tokens, + output_tokens=response.output_tokens, + cache_read_tokens=response.cache_read_input_tokens or 0, + cache_creation_tokens=response.cache_creation_input_tokens or 0, + ) + iteration_callback(iteration_data) + except Exception: + logger.warning("iteration_callback failed", exc_info=True) + # After first iteration, switch from "required" to "auto" to allow model to stop if iteration == 0 and effective_tool_choice in ("required", "any"): effective_tool_choice = "auto" @@ -884,6 +934,7 @@ async def _execute_tool_loop( cache_creation_input_tokens=total_cache_creation_tokens, cache_read_input_tokens=total_cache_read_tokens, thinking_content=None, # No thinking content at max iterations + iterations=iteration + 1, # +1 for the synthesis call ) # Make one final call to get a text response @@ -927,6 +978,7 @@ async def _execute_tool_loop( final_response = await final_call_func() final_response.tool_calls_made = all_tool_calls + final_response.iterations = iteration + 1 # +1 for the synthesis call # Include accumulated tokens from all iterations plus the final call final_response.input_tokens = total_input_tokens + final_response.input_tokens final_response.output_tokens = total_output_tokens + final_response.output_tokens @@ -1126,6 +1178,7 @@ async def honcho_llm_call( messages: list[dict[str, Any]] | None = None, max_input_tokens: int | None = None, trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, ) -> HonchoLLMCallResponse[M]: ... @@ -1154,6 +1207,7 @@ async def honcho_llm_call( messages: list[dict[str, Any]] | None = None, max_input_tokens: int | None = None, trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, ) -> HonchoLLMCallResponse[str]: ... @@ -1182,6 +1236,7 @@ async def honcho_llm_call( messages: list[dict[str, Any]] | None = None, max_input_tokens: int | None = None, trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, ) -> AsyncIterator[HonchoLLMCallStreamChunk] | StreamingResponseWithMetadata: ... @@ -1210,6 +1265,7 @@ async def honcho_llm_call( messages: list[dict[str, Any]] | None = None, max_input_tokens: int | None = None, trace_name: str | None = None, + iteration_callback: IterationCallback | None = None, ) -> ( HonchoLLMCallResponse[Any] | AsyncIterator[HonchoLLMCallStreamChunk] @@ -1241,6 +1297,7 @@ async def honcho_llm_call( tool_executor: Async callable to execute tools, receives (tool_name, tool_input) max_tool_iterations: Maximum number of tool execution loops messages: Optional message list for multi-turn conversations (overrides prompt) + iteration_callback: Optional callback invoked after each tool iteration with IterationData Returns: HonchoLLMCallResponse or AsyncIterator depending on stream parameter @@ -1455,6 +1512,7 @@ async def honcho_llm_call( get_provider_and_model=_get_provider_and_model, before_retry_callback=before_retry_callback, stream_final=stream_final_only, + iteration_callback=iteration_callback, ) if trace_name and isinstance(result, HonchoLLMCallResponse): log_reasoning_trace( diff --git a/src/utils/summarizer.py b/src/utils/summarizer.py index 29cc02cb..e7fec89d 100644 --- a/src/utils/summarizer.py +++ b/src/utils/summarizer.py @@ -9,16 +9,19 @@ from typing import TypedDict from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession -from src import prometheus, schemas +from src import schemas from src.cache.client import cache as cache_client from src.config import settings from src.crud.session import session_cache_key from src.dependencies import tracked_db from src.exceptions import ResourceNotFoundException from src.models import Message +from src.telemetry import otel_metrics +from src.telemetry.events import AgentToolSummaryCreatedEvent, emit +from src.telemetry.logging import accumulate_metric, conditional_observe +from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes from src.utils.clients import HonchoLLMCallResponse, honcho_llm_call from src.utils.formatting import utc_now_iso -from src.utils.logging import accumulate_metric, conditional_observe from src.utils.tokens import estimate_tokens, track_deriver_input_tokens from .. import crud, models @@ -412,7 +415,12 @@ async def _create_and_save_summary( previous_summary_tokens = latest_summary["token_count"] if latest_summary else 0 input_tokens = messages_tokens + previous_summary_tokens - new_summary, is_fallback = await _create_summary( + ( + new_summary, + is_fallback, + llm_input_tokens, + llm_output_tokens, + ) = await _create_summary( formatted_messages=formatted_messages, previous_summary_text=previous_summary_text, summary_type=summary_type, @@ -432,20 +440,22 @@ async def _create_and_save_summary( prompt_tokens = estimate_long_summary_prompt_tokens() track_deriver_input_tokens( - task_type=prometheus.DeriverTaskTypes.SUMMARY, + task_type=DeriverTaskTypes.SUMMARY, components={ - prometheus.DeriverComponents.PROMPT: prompt_tokens, - prometheus.DeriverComponents.MESSAGES: messages_tokens, - prometheus.DeriverComponents.PREVIOUS_SUMMARY: previous_summary_tokens, + DeriverComponents.PROMPT: prompt_tokens, + DeriverComponents.MESSAGES: messages_tokens, + DeriverComponents.PREVIOUS_SUMMARY: previous_summary_tokens, }, ) # Track output tokens - prometheus.DERIVER_TOKENS_PROCESSED.labels( - task_type=prometheus.DeriverTaskTypes.SUMMARY.value, - token_type=prometheus.TokenTypes.OUTPUT.value, - component=prometheus.DeriverComponents.OUTPUT_TOTAL.value, - ).inc(new_summary["token_count"]) + if settings.OTEL.ENABLED: + otel_metrics.record_deriver_tokens( + count=new_summary["token_count"], + task_type=DeriverTaskTypes.SUMMARY.value, + token_type=TokenTypes.OUTPUT.value, + component=DeriverComponents.OUTPUT_TOTAL.value, + ) # Save summary to database with new transaction async with tracked_db("summary.save") as db: @@ -477,6 +487,27 @@ async def _create_and_save_summary( "ms", ) + # Emit telemetry event (only for non-fallback summaries) + # Note: Using AgentToolSummaryCreatedEvent with dummy run_id/iteration since + # this is called from the deriver, not from an agentic loop + if not is_fallback: + emit( + AgentToolSummaryCreatedEvent( + run_id="deriver", # Placeholder - not from an agentic run + iteration=0, # Placeholder - not from an agentic loop + parent_category="deriver", + agent_type="summarizer", + workspace_name=workspace_name, + session_name=session_name, + message_id=message_public_id, + message_count=len(messages), + message_seq_in_session=message_seq_in_session, + summary_type="short" if summary_type == SummaryType.SHORT else "long", + input_tokens=llm_input_tokens, + output_tokens=llm_output_tokens, + ) + ) + async def _create_summary( formatted_messages: str, @@ -487,7 +518,7 @@ async def _create_summary( last_message_id: int, last_message_content_preview: str, message_count: int, -) -> tuple[Summary, bool]: +) -> tuple[Summary, bool, int, int]: """ Generate a summary of the provided messages using an LLM. @@ -502,12 +533,16 @@ async def _create_summary( message_count: Number of messages for fallback Returns: - A tuple of (Summary, is_fallback) where is_fallback indicates if - the summary was generated using a fallback instead of an LLM call + A tuple of (Summary, is_fallback, llm_input_tokens, llm_output_tokens) + where is_fallback indicates if the summary was generated using a + fallback instead of an LLM call, and the token counts are from the LLM call + (0 if fallback was used) """ response: HonchoLLMCallResponse[str] | None = None is_fallback = False + llm_input_tokens = 0 + llm_output_tokens = 0 try: if summary_type == SummaryType.SHORT: response = await create_short_summary( @@ -520,6 +555,8 @@ async def _create_summary( summary_text = response.content summary_tokens = response.output_tokens + llm_input_tokens = response.input_tokens + llm_output_tokens = response.output_tokens # Detect potential issues with the summary if not summary_text.strip(): @@ -547,6 +584,8 @@ async def _create_summary( message_public_id=message_public_id, ), is_fallback, + llm_input_tokens, + llm_output_tokens, ) diff --git a/src/utils/tokens.py b/src/utils/tokens.py index f4c488d8..4e71e3fa 100644 --- a/src/utils/tokens.py +++ b/src/utils/tokens.py @@ -1,6 +1,8 @@ import tiktoken -from src import prometheus +from src.config import settings +from src.telemetry import otel_metrics +from src.telemetry.otel.metrics import DeriverComponents, DeriverTaskTypes, TokenTypes tokenizer = tiktoken.get_encoding("o200k_base") @@ -18,8 +20,8 @@ def estimate_tokens(text: str | list[str] | None) -> int: def track_deriver_input_tokens( - task_type: prometheus.DeriverTaskTypes, - components: dict[prometheus.DeriverComponents, int], + task_type: DeriverTaskTypes, + components: dict[DeriverComponents, int], ) -> None: """ Helper method to track input token components for a given task type. @@ -29,8 +31,11 @@ def track_deriver_input_tokens( components: Dict mapping component names to token counts """ for component, token_count in components.items(): - prometheus.DERIVER_TOKENS_PROCESSED.labels( - task_type=task_type.value, - token_type=prometheus.TokenTypes.INPUT.value, - component=component.value, - ).inc(token_count) + # OTel metrics (push-based) + if settings.OTEL.ENABLED: + otel_metrics.record_deriver_tokens( + count=token_count, + task_type=task_type.value, + token_type=TokenTypes.INPUT.value, + component=component.value, + ) diff --git a/src/utils/tracing.py b/src/utils/tracing.py deleted file mode 100644 index 9d08dbc7..00000000 --- a/src/utils/tracing.py +++ /dev/null @@ -1,48 +0,0 @@ -import asyncio -from collections.abc import Callable -from functools import wraps -from typing import ParamSpec, TypeVar, cast - -import sentry_sdk - -P = ParamSpec("P") -T = TypeVar("T") - - -def with_sentry_transaction( - name: str, op: str -) -> Callable[[Callable[P, T]], Callable[P, T]]: - """Decorator to wrap a function in a Sentry transaction. - - Args: - name: The name of the transaction. - op: The operation type (e.g., "deriver", "http.server"). - - Returns: - A decorator that wraps the target function in a Sentry transaction. - - Example: - @with_sentry_transaction("process_task", op="deriver") - def process_task(data): - ... - """ - - def decorator(func: Callable[P, T]) -> Callable[P, T]: - if asyncio.iscoroutinefunction(func): - - @wraps(func) - async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: - with sentry_sdk.start_transaction(name=name, op=op): - return await func(*args, **kwargs) - - return cast(Callable[P, T], async_wrapper) - else: - - @wraps(func) - def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: - with sentry_sdk.start_transaction(name=name, op=op): - return func(*args, **kwargs) - - return sync_wrapper - - return decorator diff --git a/src/utils/types.py b/src/utils/types.py index f9efffd1..0824b0cd 100644 --- a/src/utils/types.py +++ b/src/utils/types.py @@ -1,7 +1,22 @@ +from contextvars import ContextVar from typing import Generic, Literal, NamedTuple, TypeVar T = TypeVar("T") +# Context variable for tracking current iteration in tool execution loop +# This is used for telemetry to associate tool calls with their iteration +_current_iteration: ContextVar[int] = ContextVar("current_iteration", default=0) + + +def set_current_iteration(iteration: int) -> None: + """Set the current iteration number for telemetry context.""" + _current_iteration.set(iteration) + + +def get_current_iteration() -> int: + """Get the current iteration number from telemetry context.""" + return _current_iteration.get() + class GetOrCreateResult(NamedTuple, Generic[T]): """Result of a get_or_create operation indicating whether the resource was created.""" diff --git a/tests/bench/beam.py b/tests/bench/beam.py index 773d555a..6d8cc2b5 100644 --- a/tests/bench/beam.py +++ b/tests/bench/beam.py @@ -81,7 +81,7 @@ from honcho_core.types.workspaces.sessions.message_create_param import ( from openai import AsyncOpenAI from src.config import settings -from src.utils.metrics_collector import MetricsCollector +from src.telemetry.metrics_collector import MetricsCollector from .beam_common import ( ConversationResult, diff --git a/tests/bench/locomo.py b/tests/bench/locomo.py index 3f6b8284..866aa4be 100644 --- a/tests/bench/locomo.py +++ b/tests/bench/locomo.py @@ -78,7 +78,7 @@ from honcho_core.types.workspaces.sessions.message_create_param import ( from openai import AsyncOpenAI from src.config import settings -from src.utils.metrics_collector import MetricsCollector +from src.telemetry.metrics_collector import MetricsCollector from .locomo_common import ( CATEGORY_NAMES, diff --git a/tests/bench/longmem.py b/tests/bench/longmem.py index 18eb5970..aff6568b 100644 --- a/tests/bench/longmem.py +++ b/tests/bench/longmem.py @@ -77,7 +77,7 @@ from openai import AsyncOpenAI from typing_extensions import TypedDict from src.config import settings -from src.utils.metrics_collector import MetricsCollector +from src.telemetry.metrics_collector import MetricsCollector from .longmem_common import ( calculate_timing_statistics, diff --git a/tests/bench/run_tests.py b/tests/bench/run_tests.py index 9054aa3f..c0efa234 100644 --- a/tests/bench/run_tests.py +++ b/tests/bench/run_tests.py @@ -452,12 +452,12 @@ Evaluate whether the actual response contains the core correct information from if session_name and observed: response_text = await query_peer.chat( query, - session_id=session_name, + session=session_name, target=peers[observed], ) elif session_name: response_text = await query_peer.chat( - query, session_id=session_name + query, session=session_name ) elif observed: response_text = await query_peer.chat( diff --git a/tests/conftest.py b/tests/conftest.py index 75aade07..513cd48e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -661,7 +661,7 @@ def mock_tracked_db(db_session: AsyncSession): patch("src.routers.sessions.tracked_db", mock_tracked_db_context), patch("src.routers.peers.tracked_db", mock_tracked_db_context), patch("src.crud.representation.tracked_db", mock_tracked_db_context), - patch("src.dreamer.dreamer.tracked_db", mock_tracked_db_context), + patch("src.dreamer.orchestrator.tracked_db", mock_tracked_db_context), patch("src.dreamer.dream_scheduler.tracked_db", mock_tracked_db_context), patch("src.dialectic.chat.tracked_db", mock_tracked_db_context), patch("src.utils.summarizer.tracked_db", mock_tracked_db_context), diff --git a/tests/crud/test_workspace.py b/tests/crud/test_workspace.py index f93545b5..0bd9e463 100644 --- a/tests/crud/test_workspace.py +++ b/tests/crud/test_workspace.py @@ -380,18 +380,25 @@ class TestWorkspaceCRUD: db_session: AsyncSession, sample_data: tuple[models.Workspace, models.Peer], ): - """Test that delete_workspace returns the deleted workspace object""" + """Test that delete_workspace returns WorkspaceDeletionResult with cascade counts""" test_workspace, _test_peer = sample_data # Store workspace details before deletion workspace_name = test_workspace.name # Delete workspace - deleted_workspace = await crud.delete_workspace(db_session, test_workspace.name) + result = await crud.delete_workspace(db_session, test_workspace.name) - # Verify returned workspace matches the deleted workspace - assert deleted_workspace.name == workspace_name - assert isinstance(deleted_workspace, schemas.Workspace) + # Verify returned result is WorkspaceDeletionResult with correct workspace + assert isinstance(result, crud.WorkspaceDeletionResult) + assert result.workspace.name == workspace_name + assert isinstance(result.workspace, schemas.Workspace) + + # Verify cascade counts are present (should have 1 peer from sample_data) + assert result.peers_deleted == 1 + assert result.sessions_deleted >= 0 + assert result.messages_deleted >= 0 + assert result.conclusions_deleted >= 0 @pytest.mark.asyncio async def test_delete_workspace_complex_cascade( diff --git a/tests/deriver/test_queue_processing.py b/tests/deriver/test_queue_processing.py index 6cb3dc19..b36839aa 100644 --- a/tests/deriver/test_queue_processing.py +++ b/tests/deriver/test_queue_processing.py @@ -347,6 +347,7 @@ class TestQueueProcessing: *, observed: str | None = None, # pyright: ignore[reportUnusedParameter] observer: str | None = None, # pyright: ignore[reportUnusedParameter] + queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter] ) -> None: processed_batches.append( { @@ -909,6 +910,7 @@ class TestQueueProcessing: *, observed: str | None = None, # pyright: ignore[reportUnusedParameter] observer: str | None = None, # pyright: ignore[reportUnusedParameter] + queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter] ) -> None: processed_batches.append( { @@ -1027,6 +1029,7 @@ class TestQueueProcessing: *, observed: str | None = None, # pyright: ignore[reportUnusedParameter] observer: str | None = None, # pyright: ignore[reportUnusedParameter] + queue_items_count: int | None = None, # pyright: ignore[reportUnusedParameter] ) -> None: processed_batches.append( { diff --git a/tests/integration/test_telemetry.py b/tests/integration/test_telemetry.py new file mode 100644 index 00000000..96b45c53 --- /dev/null +++ b/tests/integration/test_telemetry.py @@ -0,0 +1,1170 @@ +# pyright: reportPrivateUsage=false +"""Integration tests for telemetry with a mock ingestion endpoint. + +These tests verify the full telemetry pipeline: +- Event creation → CloudEvent conversion → HTTP transport → Server receipt + +Uses httpx's MockTransport to simulate the server without real network calls. +This avoids event loop issues and doesn't need database/cache fixtures. +""" + +import asyncio +import json +from datetime import UTC, datetime +from typing import Any +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +from src.telemetry.emitter import TelemetryEmitter +from src.telemetry.events.agent import ( + AgentIterationEvent, + AgentToolConclusionsCreatedEvent, + AgentToolConclusionsDeletedEvent, + AgentToolPeerCardUpdatedEvent, + AgentToolSummaryCreatedEvent, +) +from src.telemetry.events.deletion import DeletionCompletedEvent +from src.telemetry.events.dialectic import DialecticCompletedEvent +from src.telemetry.events.dream import DreamRunEvent, DreamSpecialistEvent +from src.telemetry.events.reconciliation import ( + CleanupStaleItemsCompletedEvent, + SyncVectorsCompletedEvent, +) +from src.telemetry.events.representation import RepresentationCompletedEvent + +# ============================================================================= +# Mock HTTP Transport for capturing requests +# ============================================================================= + + +class MockTransport: + """A mock transport that captures requests for testing.""" + + def __init__(self): + self.requests: list[dict[str, Any]] = [] + self.received_events: list[dict[str, Any]] = [] + self.response_code: int = 200 + self.fail_next_n: int = 0 + + def reset(self): + """Reset captured data.""" + self.requests.clear() + self.received_events.clear() + self.fail_next_n = 0 + + async def aclose(self) -> None: + """Mock close method to satisfy AsyncClient interface.""" + pass + + def _make_response(self, status_code: int, url: str) -> httpx.Response: + """Create an httpx.Response with a properly attached request.""" + request = httpx.Request("POST", url) + response = httpx.Response(status_code, request=request) + return response + + async def post( + self, url: str, content: bytes = b"", headers: dict[str, str] | None = None + ) -> httpx.Response: + """Handle a POST request and return a response. + + This mimics httpx.AsyncClient.post() interface. + """ + self.requests.append( + { + "method": "POST", + "url": url, + "headers": headers or {}, + "content": content, + } + ) + + # Check if we should fail + if self.fail_next_n > 0: + self.fail_next_n -= 1 + return self._make_response(500, url) + + # Parse the request body + try: + content_str = content.decode("utf-8") + content_type = (headers or {}).get("Content-Type", "") + + if "batch" in content_type: + events = json.loads(content_str) + self.received_events.extend(events) + else: + event = json.loads(content_str) + self.received_events.append(event) + except (json.JSONDecodeError, UnicodeDecodeError): + return self._make_response(400, url) + + return self._make_response(self.response_code, url) + + +# ============================================================================= +# Fixtures +# ============================================================================= + + +@pytest.fixture +def fixed_timestamp() -> datetime: + """Return a fixed timestamp for deterministic event testing.""" + return datetime(2024, 1, 15, 12, 0, 0, tzinfo=UTC) + + +@pytest.fixture +def mock_transport() -> MockTransport: + """Create a mock transport for capturing HTTP requests.""" + return MockTransport() + + +# ============================================================================= +# Sample Event Factories +# ============================================================================= + + +def create_representation_event( + timestamp: datetime, suffix: str = "" +) -> RepresentationCompletedEvent: + """Create a RepresentationCompletedEvent.""" + return RepresentationCompletedEvent( + timestamp=timestamp, + workspace_name="test_workspace", + session_name="test_session", + observed="user_peer", + queue_items_processed=3, + earliest_message_id="msg_001", + latest_message_id=f"msg_010{suffix}", + message_count=10, + explicit_conclusion_count=5, + context_preparation_ms=50.0, + llm_call_ms=1200.0, + total_duration_ms=1300.0, + input_tokens=5000, + output_tokens=500, + ) + + +def create_dream_run_event(timestamp: datetime) -> DreamRunEvent: + """Create a DreamRunEvent.""" + return DreamRunEvent( + timestamp=timestamp, + run_id="abc12345", + workspace_name="test_workspace", + session_name="test_session", + observer="assistant", + observed="user_peer", + specialists_run=["deduction", "induction"], + deduction_success=True, + induction_success=True, + total_iterations=15, + total_input_tokens=25000, + total_output_tokens=3000, + total_duration_ms=45000.0, + ) + + +def create_dream_specialist_event(timestamp: datetime) -> DreamSpecialistEvent: + """Create a DreamSpecialistEvent.""" + return DreamSpecialistEvent( + timestamp=timestamp, + run_id="abc12345", + specialist_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + iterations=8, + tool_calls_count=12, + input_tokens=15000, + output_tokens=2000, + duration_ms=25000.0, + success=True, + ) + + +def create_dialectic_event(timestamp: datetime) -> DialecticCompletedEvent: + """Create a DialecticCompletedEvent.""" + return DialecticCompletedEvent( + timestamp=timestamp, + run_id="def67890", + workspace_name="test_workspace", + peer_name="user_peer", + reasoning_level="medium", + total_duration_ms=3500.0, + input_tokens=8000, + output_tokens=1200, + ) + + +def create_agent_iteration_event(timestamp: datetime) -> AgentIterationEvent: + """Create an AgentIterationEvent.""" + return AgentIterationEvent( + timestamp=timestamp, + run_id="abc12345", + parent_category="dream", + agent_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + iteration=3, + tool_calls=["search_memory", "create_observations"], + input_tokens=3000, + output_tokens=500, + ) + + +def create_conclusions_created_event( + timestamp: datetime, +) -> AgentToolConclusionsCreatedEvent: + """Create an AgentToolConclusionsCreatedEvent.""" + return AgentToolConclusionsCreatedEvent( + timestamp=timestamp, + run_id="abc12345", + iteration=3, + parent_category="dream", + agent_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + conclusion_count=5, + levels=["explicit", "deductive", "deductive", "explicit", "deductive"], + ) + + +def create_conclusions_deleted_event( + timestamp: datetime, +) -> AgentToolConclusionsDeletedEvent: + """Create an AgentToolConclusionsDeletedEvent.""" + return AgentToolConclusionsDeletedEvent( + timestamp=timestamp, + run_id="abc12345", + iteration=5, + parent_category="dream", + agent_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + conclusion_count=3, + ) + + +def create_peer_card_updated_event( + timestamp: datetime, +) -> AgentToolPeerCardUpdatedEvent: + """Create an AgentToolPeerCardUpdatedEvent.""" + return AgentToolPeerCardUpdatedEvent( + timestamp=timestamp, + run_id="abc12345", + iteration=7, + parent_category="dream", + agent_type="induction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + facts_count=12, + ) + + +def create_summary_created_event(timestamp: datetime) -> AgentToolSummaryCreatedEvent: + """Create an AgentToolSummaryCreatedEvent.""" + return AgentToolSummaryCreatedEvent( + timestamp=timestamp, + run_id="ghi11111", + iteration=1, + parent_category="representation", + agent_type="summarizer", + workspace_name="test_workspace", + session_name="test_session", + message_id="msg_020", + message_count=20, + message_seq_in_session=20, + summary_type="short", + input_tokens=4000, + output_tokens=300, + ) + + +def create_deletion_event(timestamp: datetime) -> DeletionCompletedEvent: + """Create a DeletionCompletedEvent.""" + return DeletionCompletedEvent( + timestamp=timestamp, + workspace_name="test_workspace", + deletion_type="workspace", + resource_id="ws_123", + success=True, + peers_deleted=5, + sessions_deleted=10, + messages_deleted=500, + conclusions_deleted=200, + ) + + +def create_sync_vectors_event(timestamp: datetime) -> SyncVectorsCompletedEvent: + """Create a SyncVectorsCompletedEvent.""" + return SyncVectorsCompletedEvent( + timestamp=timestamp, + documents_synced=150, + documents_failed=2, + message_embeddings_synced=500, + message_embeddings_failed=0, + total_duration_ms=12000.0, + ) + + +def create_cleanup_event(timestamp: datetime) -> CleanupStaleItemsCompletedEvent: + """Create a CleanupStaleItemsCompletedEvent.""" + return CleanupStaleItemsCompletedEvent( + timestamp=timestamp, + documents_cleaned=25, + queue_items_cleaned=100, + total_duration_ms=5000.0, + ) + + +# ============================================================================= +# Integration Tests - Basic Event Sending +# ============================================================================= + + +class TestBasicEventSending: + """Tests for basic event sending functionality.""" + + @pytest.mark.asyncio + async def test_single_event_received( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Single event is received by the server.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=10, + flush_interval_seconds=10, # Don't auto-flush + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "integration_test" + event = create_representation_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "representation.completed" + assert "id" in received + assert received["id"].startswith("evt_") + + @pytest.mark.asyncio + async def test_multiple_events_received( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Multiple events are received in batches.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=10, + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "integration_test" + for i in range(5): + event = create_representation_event(fixed_timestamp, f"_{i}") + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 5 + + @pytest.mark.asyncio + async def test_cloudevent_structure( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """CloudEvent has correct structure and attributes.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=10, + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "integration_test" + event = create_representation_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + received = mock_transport.received_events[0] + + # Required CloudEvent attributes + assert "id" in received + assert "source" in received + assert "type" in received + assert "time" in received + assert "data" in received + + # Verify source includes namespace + assert "integration_test" in received["source"] + assert "/honcho/" in received["source"] + + # Verify data contains event fields + data = received["data"] + assert data["workspace_name"] == "test_workspace" + assert data["message_count"] == 10 + + +# ============================================================================= +# Integration Tests - All Event Types +# ============================================================================= + + +class TestAllEventTypes: + """Tests that all 12 event types can be sent and received.""" + + @pytest.mark.asyncio + async def test_representation_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """RepresentationCompletedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_representation_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + assert mock_transport.received_events[0]["type"] == "representation.completed" + + @pytest.mark.asyncio + async def test_dream_run_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """DreamRunEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_dream_run_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "dream.run" + assert received["data"]["run_id"] == "abc12345" + + @pytest.mark.asyncio + async def test_dream_specialist_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """DreamSpecialistEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_dream_specialist_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "dream.specialist" + assert received["data"]["specialist_type"] == "deduction" + + @pytest.mark.asyncio + async def test_dialectic_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """DialecticCompletedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_dialectic_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "dialectic.completed" + assert received["data"]["reasoning_level"] == "medium" + + @pytest.mark.asyncio + async def test_agent_iteration_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """AgentIterationEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_agent_iteration_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "agent.iteration" + assert received["data"]["iteration"] == 3 + + @pytest.mark.asyncio + async def test_conclusions_created_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """AgentToolConclusionsCreatedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_conclusions_created_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "agent.tool.conclusions.created" + assert received["data"]["conclusion_count"] == 5 + + @pytest.mark.asyncio + async def test_conclusions_deleted_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """AgentToolConclusionsDeletedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_conclusions_deleted_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "agent.tool.conclusions.deleted" + assert received["data"]["conclusion_count"] == 3 + + @pytest.mark.asyncio + async def test_peer_card_updated_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """AgentToolPeerCardUpdatedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_peer_card_updated_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "agent.tool.peer_card.updated" + assert received["data"]["facts_count"] == 12 + + @pytest.mark.asyncio + async def test_summary_created_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """AgentToolSummaryCreatedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_summary_created_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "agent.tool.summary.created" + assert received["data"]["summary_type"] == "short" + + @pytest.mark.asyncio + async def test_deletion_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """DeletionCompletedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_deletion_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "deletion.completed" + assert received["data"]["deletion_type"] == "workspace" + + @pytest.mark.asyncio + async def test_sync_vectors_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """SyncVectorsCompletedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_sync_vectors_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "reconciliation.sync_vectors.completed" + assert received["data"]["documents_synced"] == 150 + + @pytest.mark.asyncio + async def test_cleanup_event( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """CleanupStaleItemsCompletedEvent is sent correctly.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_cleanup_event(fixed_timestamp) + emitter.emit(event) + + await emitter.flush() + + assert len(mock_transport.received_events) == 1 + received = mock_transport.received_events[0] + assert received["type"] == "reconciliation.cleanup_stale_items.completed" + assert received["data"]["documents_cleaned"] == 25 + + +# ============================================================================= +# Integration Tests - Error Scenarios +# ============================================================================= + + +class TestErrorScenarios: + """Tests for error handling with HTTP transport.""" + + @pytest.mark.asyncio + async def test_server_error_triggers_retry( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Server errors trigger retry and eventual success.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=3, + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Configure to fail first 2 requests + mock_transport.fail_next_n = 2 + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), # Skip delays + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_representation_event(fixed_timestamp) + emitter.emit(event) + await emitter.flush() + + # Should have made 3 requests (2 failures + 1 success) + assert len(mock_transport.requests) == 3 + # Event should have been received on 3rd attempt + assert len(mock_transport.received_events) == 1 + + @pytest.mark.asyncio + async def test_all_retries_exhausted_keeps_events( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Events returned to buffer when all retries fail.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=2, + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Configure to fail all requests + mock_transport.fail_next_n = 10 + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_representation_event(fixed_timestamp) + emitter.emit(event) + await emitter.flush() + + # All retries should have been attempted + assert len(mock_transport.requests) == 2 + # Event should be back in buffer + assert emitter.buffer_size == 1 + + @pytest.mark.asyncio + async def test_server_returns_4xx_error( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Client errors (4xx) also trigger retries.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=2, + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + mock_transport.response_code = 400 + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_representation_event(fixed_timestamp) + emitter.emit(event) + await emitter.flush() + + # Should have tried 2 times + assert len(mock_transport.requests) == 2 + + +# ============================================================================= +# Integration Tests - Batching Behavior +# ============================================================================= + + +class TestBatchingBehavior: + """Tests for batch sending behavior.""" + + @pytest.mark.asyncio + async def test_events_batched_correctly( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Events are batched according to batch_size.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=3, # Small batch size for testing + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Emit 7 events + for i in range(7): + event = create_representation_event(fixed_timestamp, f"_{i}") + emitter.emit(event) + + await emitter.flush() + + # Should have made 3 requests (batches of 3, 3, 1) + assert len(mock_transport.requests) == 3 + # All events should be received + assert len(mock_transport.received_events) == 7 + + @pytest.mark.asyncio + async def test_flush_threshold_triggers_send( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Reaching flush_threshold triggers automatic flush.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=100, + flush_interval_seconds=10, # Long interval + flush_threshold=3, # Low threshold + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Emit exactly threshold events + for i in range(3): + event = create_representation_event(fixed_timestamp, f"_{i}") + emitter.emit(event) + + # Give time for async flush to trigger + await asyncio.sleep(0.1) + + # Should have auto-flushed + assert len(mock_transport.requests) >= 1 + assert len(mock_transport.received_events) == 3 + + +# ============================================================================= +# Integration Tests - Idempotency +# ============================================================================= + + +class TestIdempotency: + """Tests for event ID idempotency.""" + + @pytest.mark.asyncio + async def test_same_event_has_same_id( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Same event parameters produce same event ID.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Create two identical events + event1 = create_representation_event(fixed_timestamp) + event2 = create_representation_event(fixed_timestamp) + + emitter.emit(event1) + emitter.emit(event2) + + await emitter.flush() + + # Both events should have same ID + assert len(mock_transport.received_events) == 2 + assert ( + mock_transport.received_events[0]["id"] + == mock_transport.received_events[1]["id"] + ) + + @pytest.mark.asyncio + async def test_different_events_have_different_ids( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Different event parameters produce different event IDs.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Create events with different suffixes (different latest_message_id) + event1 = create_representation_event(fixed_timestamp, "_a") + event2 = create_representation_event(fixed_timestamp, "_b") + + emitter.emit(event1) + emitter.emit(event2) + + await emitter.flush() + + # Events should have different IDs + assert len(mock_transport.received_events) == 2 + assert ( + mock_transport.received_events[0]["id"] + != mock_transport.received_events[1]["id"] + ) + + +# ============================================================================= +# Integration Tests - Shutdown Behavior +# ============================================================================= + + +class TestShutdownBehavior: + """Tests for graceful shutdown with pending events.""" + + @pytest.mark.asyncio + async def test_shutdown_flushes_pending_events( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Shutdown flushes all pending events before closing.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=100, + flush_interval_seconds=60, # Very long interval + flush_threshold=100, # High threshold + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + emitter._flush_task = asyncio.create_task(asyncio.sleep(100)) + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Emit events without triggering auto-flush + for i in range(5): + event = create_representation_event(fixed_timestamp, f"_{i}") + emitter.emit(event) + + # Verify events are buffered + assert emitter.buffer_size == 5 + + # Shutdown should flush + await emitter.shutdown() + + # All events should have been sent + assert len(mock_transport.received_events) == 5 + assert emitter.buffer_size == 0 + + +# ============================================================================= +# Integration Tests - Mixed Event Types +# ============================================================================= + + +class TestMixedEventTypes: + """Tests for sending different event types together.""" + + @pytest.mark.asyncio + async def test_mixed_event_types_in_batch( + self, + mock_transport: MockTransport, + fixed_timestamp: datetime, + ): + """Different event types can be batched together.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=10, + flush_interval_seconds=10, + flush_threshold=100, + enabled=True, + ) + + # Inject mock transport as client + emitter._client = mock_transport # pyright: ignore[reportAttributeAccessIssue] + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + + # Emit different event types + emitter.emit(create_representation_event(fixed_timestamp)) + emitter.emit(create_dream_run_event(fixed_timestamp)) + emitter.emit(create_dialectic_event(fixed_timestamp)) + emitter.emit(create_agent_iteration_event(fixed_timestamp)) + emitter.emit(create_deletion_event(fixed_timestamp)) + + await emitter.flush() + + # All events should be received + assert len(mock_transport.received_events) == 5 + + # Check we got different types + types = {e["type"] for e in mock_transport.received_events} + assert "representation.completed" in types + assert "dream.run" in types + assert "dialectic.completed" in types + assert "agent.iteration" in types + assert "deletion.completed" in types diff --git a/tests/integration/test_token_metrics.py b/tests/integration/test_token_metrics.py index d4a55338..d0c38834 100644 --- a/tests/integration/test_token_metrics.py +++ b/tests/integration/test_token_metrics.py @@ -1,20 +1,23 @@ -"""Integration tests for prometheus token metrics tracking. +"""Integration tests for OpenTelemetry token metrics tracking. -These tests verify that DERIVER_TOKENS_PROCESSED and DIALECTIC_TOKENS_PROCESSED -metrics are correctly emitted with accurate token counts when processing messages -and dialectic queries. +These tests verify that deriver and dialectic token metrics are correctly +emitted with accurate token counts when processing messages and dialectic queries. -The approach uses delta-based verification: +The approach uses delta-based verification with OTel's InMemoryMetricReader: 1. Capture counter values before test execution 2. Run the code under test (with mocked LLM) 3. Verify deltas match expected values """ +from collections.abc import Iterator +from typing import Any, cast from unittest.mock import AsyncMock, patch import pytest from nanoid import generate as generate_nanoid -from prometheus_client import REGISTRY +from opentelemetry.metrics import Meter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models, schemas @@ -26,6 +29,7 @@ from src.schemas import ( ResolvedReasoningConfiguration, ResolvedSummaryConfiguration, ) +from src.telemetry.otel.metrics import otel_metrics from src.utils.clients import HonchoLLMCallResponse from src.utils.representation import ExplicitObservationBase, PromptRepresentation from src.utils.summarizer import ( @@ -39,17 +43,43 @@ from src.utils.summarizer import ( # ============================================================================= -class MetricDeltaChecker: - """Utility class to capture and verify prometheus counter deltas.""" +class OTelMetricChecker: + """Utility class to capture and verify OTel counter deltas.""" + + _reader: InMemoryMetricReader + + def __init__(self, reader: InMemoryMetricReader): + self._reader = reader + + def _get_metric_value(self, metric_name: str, labels: dict[str, str]) -> float: + """Get current value of a counter with specific labels. + + Note: The OTel SDK's MetricsData types are not fully typed, so we cast + to Any to avoid type warnings when traversing the metrics data structure. + """ + raw_data = self._reader.get_metrics_data() # pyright: ignore[reportUnknownVariableType] + if raw_data is None: + return 0.0 + data = cast(Any, raw_data) + + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + if metric.name == metric_name and hasattr( + metric.data, "data_points" + ): + for point in metric.data.data_points: + # Check if labels match + point_attrs: dict[str, str] = ( + dict(point.attributes) if point.attributes else {} + ) + if all(point_attrs.get(k) == v for k, v in labels.items()): + return float(point.value) + return 0.0 def capture(self, metric_name: str, labels: dict[str, str]) -> float: """Capture current value of a counter with specific labels.""" - # Counters have _total suffix in prometheus - full_name = ( - metric_name if metric_name.endswith("_total") else f"{metric_name}_total" - ) - value = REGISTRY.get_sample_value(full_name, labels=labels) - return value or 0.0 + return self._get_metric_value(metric_name, labels) def get_delta( self, metric_name: str, labels: dict[str, str], before: float @@ -72,18 +102,78 @@ class MetricDeltaChecker: ), f"{message}: expected delta {expected}, got {delta}. Labels: {labels}" -@pytest.fixture -def metric_checker() -> MetricDeltaChecker: - """Fixture providing a metric delta checker instance.""" - return MetricDeltaChecker() +def _reset_otel_metrics_singleton() -> None: + """Reset the otel_metrics singleton instance so it reinitializes with a new provider. + + This clears all instance attributes so _ensure_initialized() will create + new meters tied to the current global MeterProvider. + """ + # Reset initialization flag (instance attribute shadows class attribute) + otel_metrics._is_initialized = False # pyright: ignore[reportPrivateUsage] + + # Reset all meters + otel_metrics._api_meter = None # pyright: ignore[reportPrivateUsage] + otel_metrics._deriver_meter = None # pyright: ignore[reportPrivateUsage] + otel_metrics._dialectic_meter = None # pyright: ignore[reportPrivateUsage] + otel_metrics._dreamer_meter = None # pyright: ignore[reportPrivateUsage] + + # Reset all counters + otel_metrics._api_requests = None # pyright: ignore[reportPrivateUsage] + otel_metrics._messages_created = None # pyright: ignore[reportPrivateUsage] + otel_metrics._dialectic_calls = None # pyright: ignore[reportPrivateUsage] + otel_metrics._deriver_queue_items = None # pyright: ignore[reportPrivateUsage] + otel_metrics._deriver_tokens = None # pyright: ignore[reportPrivateUsage] + otel_metrics._dialectic_tokens = None # pyright: ignore[reportPrivateUsage] + otel_metrics._dreamer_tokens = None # pyright: ignore[reportPrivateUsage] @pytest.fixture -def enable_metrics(monkeypatch: pytest.MonkeyPatch): - """Enable prometheus metrics with a test namespace.""" - monkeypatch.setattr("src.prometheus.METRICS_ENABLED", True) - monkeypatch.setattr("src.config.settings.METRICS.NAMESPACE", "test") - yield +def otel_test_setup( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[tuple[InMemoryMetricReader, OTelMetricChecker]]: + """Set up OTel metrics with in-memory reader for testing. + + The OTel SDK only allows set_meter_provider() to be called once per process. + To work around this for testing, we patch get_meter() to return meters + from our test provider directly. + + Yields: + Tuple of (reader, checker) for verifying metrics + """ + # Create in-memory reader + reader = InMemoryMetricReader() + + # Create a test meter provider + provider = MeterProvider(metric_readers=[reader]) + + # Patch get_meter to return meters from our test provider + # This is necessary because set_meter_provider() can only be called once per process + def test_get_meter(name: str, version: str = "") -> Meter: + return provider.get_meter(name, version) + + monkeypatch.setattr("src.telemetry.otel.metrics.get_meter", test_get_meter) + + # Reset the otel_metrics singleton instance so it reinitializes with our test provider + _reset_otel_metrics_singleton() + + # Enable OTEL in settings and set namespace for test assertions + monkeypatch.setattr("src.config.settings.OTEL.ENABLED", True) + monkeypatch.setattr("src.config.settings.OTEL.SERVICE_NAMESPACE", "test") + + checker = OTelMetricChecker(reader) + + yield reader, checker + + # Cleanup: reset singleton so other tests aren't affected + _reset_otel_metrics_singleton() + + +@pytest.fixture +def metric_checker( + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], +) -> OTelMetricChecker: + """Fixture providing a metric checker instance.""" + return otel_test_setup[1] # ============================================================================= @@ -188,7 +278,6 @@ def create_mock_dialectic_response( @pytest.mark.asyncio -@pytest.mark.usefixtures("enable_metrics") class TestDeriverIngestionMetrics: """Test token metrics for deriver INGESTION task type.""" @@ -196,11 +285,12 @@ class TestDeriverIngestionMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify OUTPUT_TOTAL tokens match response.output_tokens from LLM.""" from src.deriver.deriver import process_representation_tasks_batch + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -237,6 +327,7 @@ class TestDeriverIngestionMetrics: message_level_configuration=create_test_configuration(), observer=peer.name, observed=peer.name, + queue_items_count=len(messages), ) # Verify output tokens metric @@ -252,12 +343,13 @@ class TestDeriverIngestionMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify PROMPT component is tracked for ingestion input.""" from src.deriver.deriver import process_representation_tasks_batch from src.deriver.prompts import estimate_minimal_deriver_prompt_tokens + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -292,6 +384,7 @@ class TestDeriverIngestionMetrics: message_level_configuration=create_test_configuration(), observer=peer.name, observed=peer.name, + queue_items_count=len(messages), ) metric_checker.assert_delta( @@ -306,11 +399,12 @@ class TestDeriverIngestionMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify MESSAGES component is tracked for ingestion input.""" from src.deriver.deriver import process_representation_tasks_batch + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -347,6 +441,7 @@ class TestDeriverIngestionMetrics: message_level_configuration=create_test_configuration(), observer=peer.name, observed=peer.name, + queue_items_count=len(messages), ) # Verify messages tokens were tracked (should be > 0) @@ -360,7 +455,6 @@ class TestDeriverIngestionMetrics: @pytest.mark.asyncio -@pytest.mark.usefixtures("enable_metrics") class TestDeriverSummaryMetrics: """Test token metrics for deriver SUMMARY task type.""" @@ -368,10 +462,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify OUTPUT_TOTAL tokens are tracked for summary.""" + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -401,7 +496,9 @@ class TestDeriverSummaryMetrics: with ( patch( "src.utils.summarizer._create_summary", - new=AsyncMock(return_value=(mock_summary, False)), # is_fallback=False + new=AsyncMock( + return_value=(mock_summary, False, 100, expected_output_tokens) + ), # (summary, is_fallback, llm_input_tokens, llm_output_tokens) ), patch( "src.utils.summarizer._save_summary", @@ -431,10 +528,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify PROMPT component is tracked for summary input.""" + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -462,7 +560,9 @@ class TestDeriverSummaryMetrics: with ( patch( "src.utils.summarizer._create_summary", - new=AsyncMock(return_value=(mock_summary, False)), + new=AsyncMock( + return_value=(mock_summary, False, 100, 10) + ), # (summary, is_fallback, llm_input_tokens, llm_output_tokens) ), patch( "src.utils.summarizer._save_summary", @@ -491,10 +591,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify MESSAGES component is tracked for summary input.""" + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -531,7 +632,9 @@ class TestDeriverSummaryMetrics: with ( patch( "src.utils.summarizer._create_summary", - new=AsyncMock(return_value=(mock_summary, False)), + new=AsyncMock( + return_value=(mock_summary, False, 100, 10) + ), # (summary, is_fallback, llm_input_tokens, llm_output_tokens) ), patch( "src.utils.summarizer._save_summary", @@ -559,10 +662,11 @@ class TestDeriverSummaryMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify metrics are NOT emitted when _create_summary returns is_fallback=True.""" + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) messages = await create_test_messages( @@ -599,7 +703,9 @@ class TestDeriverSummaryMetrics: with patch( "src.utils.summarizer._create_summary", - new=AsyncMock(return_value=(mock_summary, True)), # is_fallback=True + new=AsyncMock( + return_value=(mock_summary, True, 0, 0) + ), # (summary, is_fallback=True, llm_input_tokens, llm_output_tokens) ): await _create_and_save_summary( workspace_name=workspace.name, @@ -633,7 +739,6 @@ class TestDeriverSummaryMetrics: @pytest.mark.asyncio -@pytest.mark.usefixtures("enable_metrics") class TestDialecticTokenMetrics: """Test token metrics for dialectic calls.""" @@ -641,11 +746,12 @@ class TestDialecticTokenMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify INPUT tokens are tracked from LLM response.""" from src.dialectic.core import DialecticAgent + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -688,11 +794,12 @@ class TestDialecticTokenMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], ): """Verify OUTPUT tokens are tracked from LLM response.""" from src.dialectic.core import DialecticAgent + _, metric_checker = otel_test_setup workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -735,15 +842,16 @@ class TestDialecticTokenMetrics: self, db_session: AsyncSession, sample_data: tuple[Workspace, Peer], - metric_checker: MetricDeltaChecker, + otel_test_setup: tuple[InMemoryMetricReader, OTelMetricChecker], monkeypatch: pytest.MonkeyPatch, ): - """Verify metrics are NOT emitted when METRICS_ENABLED=False.""" + """Verify metrics are NOT emitted when OTEL.ENABLED=False.""" from src.dialectic.core import DialecticAgent - # Explicitly disable metrics - monkeypatch.setattr("src.prometheus.METRICS_ENABLED", False) - monkeypatch.setattr("src.config.settings.METRICS.NAMESPACE", "test") + _, metric_checker = otel_test_setup + + # Explicitly disable OTEL metrics + monkeypatch.setattr("src.config.settings.OTEL.ENABLED", False) workspace, peer = sample_data session = await create_test_session_with_peer(db_session, workspace, peer) @@ -757,11 +865,13 @@ class TestDialecticTokenMetrics: "namespace": "test", "token_type": "input", "component": "total", + "reasoning_level": "low", } output_labels = { "namespace": "test", "token_type": "output", "component": "total", + "reasoning_level": "low", } before_input = metric_checker.capture( "dialectic_tokens_processed", input_labels diff --git a/tests/telemetry/__init__.py b/tests/telemetry/__init__.py new file mode 100644 index 00000000..c7f63d51 --- /dev/null +++ b/tests/telemetry/__init__.py @@ -0,0 +1 @@ +"""Unit tests for the telemetry module.""" diff --git a/tests/telemetry/conftest.py b/tests/telemetry/conftest.py new file mode 100644 index 00000000..6bce95eb --- /dev/null +++ b/tests/telemetry/conftest.py @@ -0,0 +1,458 @@ +# pyright: reportUnknownMemberType=false +"""Fixtures for telemetry unit tests. + +This module provides: +- Sample event fixtures for all 12 event types +- Mock settings fixtures for controlling telemetry configuration +- Mock HTTP client fixtures for testing the emitter without network calls +""" + +from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from src.telemetry.events.agent import ( + AgentIterationEvent, + AgentToolConclusionsCreatedEvent, + AgentToolConclusionsDeletedEvent, + AgentToolPeerCardUpdatedEvent, + AgentToolSummaryCreatedEvent, +) +from src.telemetry.events.base import BaseEvent +from src.telemetry.events.deletion import DeletionCompletedEvent +from src.telemetry.events.dialectic import DialecticCompletedEvent +from src.telemetry.events.dream import DreamRunEvent, DreamSpecialistEvent +from src.telemetry.events.reconciliation import ( + CleanupStaleItemsCompletedEvent, + SyncVectorsCompletedEvent, +) +from src.telemetry.events.representation import RepresentationCompletedEvent + +# ============================================================================= +# Fixed timestamp for deterministic tests +# ============================================================================= + + +@pytest.fixture +def fixed_timestamp() -> datetime: + """Return a fixed timestamp for deterministic event ID generation.""" + return datetime(2024, 1, 15, 12, 0, 0, tzinfo=UTC) + + +# ============================================================================= +# Sample event fixtures for all event types +# ============================================================================= + + +@pytest.fixture +def sample_representation_event( + fixed_timestamp: datetime, +) -> RepresentationCompletedEvent: + """Create a sample RepresentationCompletedEvent for testing.""" + return RepresentationCompletedEvent( + timestamp=fixed_timestamp, + workspace_name="test_workspace", + session_name="test_session", + observed="user_peer", + queue_items_processed=3, + earliest_message_id="msg_001", + latest_message_id="msg_010", + message_count=10, + explicit_conclusion_count=5, + context_preparation_ms=50.0, + llm_call_ms=1200.0, + total_duration_ms=1300.0, + input_tokens=5000, + output_tokens=500, + ) + + +@pytest.fixture +def sample_dream_run_event(fixed_timestamp: datetime) -> DreamRunEvent: + """Create a sample DreamRunEvent for testing.""" + return DreamRunEvent( + timestamp=fixed_timestamp, + run_id="abc12345", + workspace_name="test_workspace", + session_name="test_session", + observer="assistant", + observed="user_peer", + specialists_run=["deduction", "induction"], + deduction_success=True, + induction_success=True, + surprisal_enabled=False, + surprisal_conclusion_count=0, + total_iterations=15, + total_input_tokens=25000, + total_output_tokens=3000, + total_duration_ms=45000.0, + ) + + +@pytest.fixture +def sample_dream_specialist_event(fixed_timestamp: datetime) -> DreamSpecialistEvent: + """Create a sample DreamSpecialistEvent for testing.""" + return DreamSpecialistEvent( + timestamp=fixed_timestamp, + run_id="abc12345", + specialist_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + iterations=8, + tool_calls_count=12, + input_tokens=15000, + output_tokens=2000, + duration_ms=25000.0, + success=True, + ) + + +@pytest.fixture +def sample_dialectic_event(fixed_timestamp: datetime) -> DialecticCompletedEvent: + """Create a sample DialecticCompletedEvent for testing.""" + return DialecticCompletedEvent( + timestamp=fixed_timestamp, + run_id="def67890", + workspace_name="test_workspace", + peer_name="user_peer", + session_name="test_session", + reasoning_level="medium", + total_iterations=3, + prefetched_conclusion_count=10, + tool_calls_count=5, + total_duration_ms=3500.0, + input_tokens=8000, + output_tokens=1200, + cache_read_tokens=500, + cache_creation_tokens=200, + ) + + +@pytest.fixture +def sample_agent_iteration_event(fixed_timestamp: datetime) -> AgentIterationEvent: + """Create a sample AgentIterationEvent for testing.""" + return AgentIterationEvent( + timestamp=fixed_timestamp, + run_id="abc12345", + parent_category="dream", + agent_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + iteration=3, + tool_calls=["search_memory", "create_observations"], + input_tokens=3000, + output_tokens=500, + cache_read_tokens=100, + cache_creation_tokens=50, + ) + + +@pytest.fixture +def sample_conclusions_created_event( + fixed_timestamp: datetime, +) -> AgentToolConclusionsCreatedEvent: + """Create a sample AgentToolConclusionsCreatedEvent for testing.""" + return AgentToolConclusionsCreatedEvent( + timestamp=fixed_timestamp, + run_id="abc12345", + iteration=3, + parent_category="dream", + agent_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + conclusion_count=5, + levels=["explicit", "deductive", "deductive", "explicit", "deductive"], + ) + + +@pytest.fixture +def sample_conclusions_deleted_event( + fixed_timestamp: datetime, +) -> AgentToolConclusionsDeletedEvent: + """Create a sample AgentToolConclusionsDeletedEvent for testing.""" + return AgentToolConclusionsDeletedEvent( + timestamp=fixed_timestamp, + run_id="abc12345", + iteration=5, + parent_category="dream", + agent_type="deduction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + conclusion_count=3, + ) + + +@pytest.fixture +def sample_peer_card_updated_event( + fixed_timestamp: datetime, +) -> AgentToolPeerCardUpdatedEvent: + """Create a sample AgentToolPeerCardUpdatedEvent for testing.""" + return AgentToolPeerCardUpdatedEvent( + timestamp=fixed_timestamp, + run_id="abc12345", + iteration=7, + parent_category="dream", + agent_type="induction", + workspace_name="test_workspace", + observer="assistant", + observed="user_peer", + facts_count=12, + ) + + +@pytest.fixture +def sample_summary_created_event( + fixed_timestamp: datetime, +) -> AgentToolSummaryCreatedEvent: + """Create a sample AgentToolSummaryCreatedEvent for testing.""" + return AgentToolSummaryCreatedEvent( + timestamp=fixed_timestamp, + run_id="ghi11111", + iteration=1, + parent_category="representation", + agent_type="summarizer", + workspace_name="test_workspace", + session_name="test_session", + message_id="msg_020", + message_count=20, + message_seq_in_session=20, + summary_type="short", + input_tokens=4000, + output_tokens=300, + ) + + +@pytest.fixture +def sample_deletion_event(fixed_timestamp: datetime) -> DeletionCompletedEvent: + """Create a sample DeletionCompletedEvent for testing.""" + return DeletionCompletedEvent( + timestamp=fixed_timestamp, + workspace_name="test_workspace", + deletion_type="workspace", + resource_id="ws_123abc", + success=True, + peers_deleted=5, + sessions_deleted=10, + messages_deleted=500, + conclusions_deleted=200, + ) + + +@pytest.fixture +def sample_sync_vectors_event(fixed_timestamp: datetime) -> SyncVectorsCompletedEvent: + """Create a sample SyncVectorsCompletedEvent for testing.""" + return SyncVectorsCompletedEvent( + timestamp=fixed_timestamp, + documents_synced=150, + documents_failed=2, + message_embeddings_synced=500, + message_embeddings_failed=0, + total_duration_ms=12000.0, + ) + + +@pytest.fixture +def sample_cleanup_event(fixed_timestamp: datetime) -> CleanupStaleItemsCompletedEvent: + """Create a sample CleanupStaleItemsCompletedEvent for testing.""" + return CleanupStaleItemsCompletedEvent( + timestamp=fixed_timestamp, + documents_cleaned=25, + queue_items_cleaned=100, + total_duration_ms=5000.0, + ) + + +# ============================================================================= +# All sample events as a collection +# ============================================================================= + + +@pytest.fixture +def all_sample_events( + sample_representation_event: RepresentationCompletedEvent, + sample_dream_run_event: DreamRunEvent, + sample_dream_specialist_event: DreamSpecialistEvent, + sample_dialectic_event: DialecticCompletedEvent, + sample_agent_iteration_event: AgentIterationEvent, + sample_conclusions_created_event: AgentToolConclusionsCreatedEvent, + sample_conclusions_deleted_event: AgentToolConclusionsDeletedEvent, + sample_peer_card_updated_event: AgentToolPeerCardUpdatedEvent, + sample_summary_created_event: AgentToolSummaryCreatedEvent, + sample_deletion_event: DeletionCompletedEvent, + sample_sync_vectors_event: SyncVectorsCompletedEvent, + sample_cleanup_event: CleanupStaleItemsCompletedEvent, +) -> list[BaseEvent]: + """Return all sample events as a list for parametrized tests.""" + return [ + sample_representation_event, + sample_dream_run_event, + sample_dream_specialist_event, + sample_dialectic_event, + sample_agent_iteration_event, + sample_conclusions_created_event, + sample_conclusions_deleted_event, + sample_peer_card_updated_event, + sample_summary_created_event, + sample_deletion_event, + sample_sync_vectors_event, + sample_cleanup_event, + ] + + +# ============================================================================= +# Mock settings fixtures +# ============================================================================= + + +@pytest.fixture +def mock_telemetry_settings(): + """Fixture to mock telemetry settings. + + Returns a context manager that patches settings with configurable values. + """ + + def _configure( + enabled: bool = True, + endpoint: str = "http://test-endpoint:8001/v1/events", + namespace: str = "test_namespace", + batch_size: int = 100, + flush_interval: float = 1.0, + flush_threshold: int = 50, + max_retries: int = 3, + max_buffer_size: int = 10000, + headers: dict[str, str] | None = None, + ): + mock_settings = MagicMock() + mock_settings.TELEMETRY.ENABLED = enabled + mock_settings.TELEMETRY.ENDPOINT = endpoint + mock_settings.TELEMETRY.NAMESPACE = namespace + mock_settings.TELEMETRY.BATCH_SIZE = batch_size + mock_settings.TELEMETRY.FLUSH_INTERVAL_SECONDS = flush_interval + mock_settings.TELEMETRY.FLUSH_THRESHOLD = flush_threshold + mock_settings.TELEMETRY.MAX_RETRIES = max_retries + mock_settings.TELEMETRY.MAX_BUFFER_SIZE = max_buffer_size + mock_settings.TELEMETRY.HEADERS = headers + mock_settings.OTEL.ENABLED = False + return patch("src.telemetry.emitter.settings", mock_settings) + + return _configure + + +# ============================================================================= +# Mock HTTP client fixtures +# ============================================================================= + + +@pytest.fixture +def mock_httpx_client(): + """Create a mock httpx.AsyncClient for testing HTTP calls. + + Returns a factory function that creates configured mocks. + """ + + def _create( + status_code: int = 200, + raise_exception: Exception | None = None, + response_sequence: list[tuple[int, Exception | None]] | None = None, + ): + """Create a mock HTTP client. + + Args: + status_code: Default status code to return + raise_exception: Exception to raise on post() + response_sequence: List of (status_code, exception) for sequential calls + """ + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = status_code + mock_response.raise_for_status = MagicMock() + + if raise_exception: + mock_client.post = AsyncMock(side_effect=raise_exception) + elif response_sequence: + # Create side effects for sequential responses + side_effects = [] + for code, exc in response_sequence: + if exc: + side_effects.append(exc) + else: + resp = MagicMock() + resp.status_code = code + if code >= 400: + import httpx + + resp.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError( + f"HTTP {code}", + request=MagicMock(), + response=resp, + ) + ) + else: + resp.raise_for_status = MagicMock() + side_effects.append(resp) + mock_client.post = AsyncMock(side_effect=side_effects) + else: + if status_code >= 400: + import httpx + + mock_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError( + f"HTTP {status_code}", + request=MagicMock(), + response=mock_response, + ) + ) + mock_client.post = AsyncMock(return_value=mock_response) + + mock_client.aclose = AsyncMock() + return mock_client + + return _create + + +# ============================================================================= +# Emitter fixtures +# ============================================================================= + + +@pytest.fixture +def disabled_emitter(): + """Create a disabled TelemetryEmitter for testing disabled state.""" + from src.telemetry.emitter import TelemetryEmitter + + return TelemetryEmitter( + endpoint=None, + enabled=False, + ) + + +@pytest.fixture +def emitter_with_small_buffer(): + """Create an emitter with a small buffer for testing overflow.""" + from src.telemetry.emitter import TelemetryEmitter + + return TelemetryEmitter( + endpoint="http://test:8001/events", + max_buffer_size=5, + flush_threshold=3, + enabled=True, + ) + + +@pytest.fixture +def emitter_with_low_threshold(): + """Create an emitter with low flush threshold for testing auto-flush.""" + from src.telemetry.emitter import TelemetryEmitter + + return TelemetryEmitter( + endpoint="http://test:8001/events", + flush_threshold=2, + batch_size=10, + enabled=True, + ) diff --git a/tests/telemetry/test_emit_function.py b/tests/telemetry/test_emit_function.py new file mode 100644 index 00000000..6b0d1b5e --- /dev/null +++ b/tests/telemetry/test_emit_function.py @@ -0,0 +1,470 @@ +# pyright: reportPrivateUsage=false +"""Unit tests for emit() function and initialization lifecycle. + +Tests for: +- emit() function behavior with/without initialized emitter +- initialize_telemetry_events() configuration handling +- shutdown_telemetry_events() cleanup +- initialize_telemetry() and initialize_telemetry_async() orchestration +""" + +from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from src.telemetry.events.representation import RepresentationCompletedEvent + + +def create_test_event() -> RepresentationCompletedEvent: + """Create a minimal test event.""" + return RepresentationCompletedEvent( + timestamp=datetime(2024, 1, 15, 12, 0, 0, tzinfo=UTC), + workspace_name="test_workspace", + session_name="test_session", + observed="user_peer", + queue_items_processed=1, + earliest_message_id="msg_001", + latest_message_id="msg_001", + message_count=1, + explicit_conclusion_count=1, + context_preparation_ms=10.0, + llm_call_ms=100.0, + total_duration_ms=110.0, + input_tokens=100, + output_tokens=50, + ) + + +# ============================================================================= +# Tests for emit() function +# ============================================================================= + + +class TestEmitFunction: + """Tests for the emit() function in src.telemetry.events.""" + + def test_emit_when_emitter_is_none(self): + """emit() silently returns when emitter is not initialized.""" + from src.telemetry.events import emit + + # Patch where get_emitter is imported from (inside emit function) + with patch("src.telemetry.emitter.get_emitter", return_value=None): + event = create_test_event() + # Should not raise, just silently return + emit(event) + + def test_emit_calls_emitter_emit(self): + """emit() calls the emitter's emit method when initialized.""" + from src.telemetry.events import emit + + mock_emitter = MagicMock() + + with patch("src.telemetry.emitter.get_emitter", return_value=mock_emitter): + event = create_test_event() + emit(event) + + mock_emitter.emit.assert_called_once_with(event) + + def test_emit_passes_event_unchanged(self): + """emit() passes the event object unchanged to emitter.""" + from src.telemetry.events import emit + + mock_emitter = MagicMock() + + with patch("src.telemetry.emitter.get_emitter", return_value=mock_emitter): + event = create_test_event() + emit(event) + + # Get the event that was passed to emit + passed_event = mock_emitter.emit.call_args[0][0] + assert passed_event is event + + +# ============================================================================= +# Tests for get_emitter() function +# ============================================================================= + + +class TestGetEmitter: + """Tests for the get_emitter() function.""" + + def test_get_emitter_returns_none_when_not_initialized(self): + """get_emitter() returns None before initialization.""" + from src.telemetry import emitter as emitter_module + + # Save original value + original = emitter_module._emitter + + try: + emitter_module._emitter = None + result = emitter_module.get_emitter() + assert result is None + finally: + # Restore original + emitter_module._emitter = original + + def test_get_emitter_returns_emitter_when_initialized(self): + """get_emitter() returns the emitter after initialization.""" + from src.telemetry import emitter as emitter_module + from src.telemetry.emitter import TelemetryEmitter + + # Save original value + original = emitter_module._emitter + + try: + test_emitter = TelemetryEmitter(endpoint="http://test:8001/events") + emitter_module._emitter = test_emitter + + result = emitter_module.get_emitter() + assert result is test_emitter + finally: + # Restore original + emitter_module._emitter = original + + +# ============================================================================= +# Tests for initialize_emitter() function +# ============================================================================= + + +class TestInitializeEmitter: + """Tests for the initialize_emitter() function.""" + + @pytest.mark.asyncio + async def test_initialize_emitter_creates_and_starts(self): + """initialize_emitter() creates emitter and starts it.""" + from src.telemetry import emitter as emitter_module + from src.telemetry.emitter import initialize_emitter + + # Save original value + original = emitter_module._emitter + + # Create a properly async mock client + mock_client = AsyncMock() + mock_client.aclose = AsyncMock() + + try: + emitter_module._emitter = None + + with patch("httpx.AsyncClient", return_value=mock_client): + result = await initialize_emitter( + endpoint="http://test:8001/events", + enabled=True, + ) + + assert result is not None + assert result.endpoint == "http://test:8001/events" + assert result.is_running is True + + # Should be stored in global + assert emitter_module._emitter is result + + # Clean up + await result.shutdown() + finally: + emitter_module._emitter = original + + @pytest.mark.asyncio + async def test_initialize_emitter_with_custom_settings(self): + """initialize_emitter() accepts custom configuration.""" + from src.telemetry import emitter as emitter_module + from src.telemetry.emitter import initialize_emitter + + original = emitter_module._emitter + + # Create a properly async mock client + mock_client = AsyncMock() + mock_client.aclose = AsyncMock() + + try: + emitter_module._emitter = None + + with patch("httpx.AsyncClient", return_value=mock_client): + result = await initialize_emitter( + endpoint="http://custom:9000/events", + headers={"X-API-Key": "secret"}, + batch_size=50, + flush_interval_seconds=2.0, + flush_threshold=25, + max_retries=5, + max_buffer_size=5000, + enabled=True, + ) + + assert result.endpoint == "http://custom:9000/events" + assert result.headers == {"X-API-Key": "secret"} + assert result.batch_size == 50 + assert result.flush_interval == 2.0 + assert result.flush_threshold == 25 + assert result.max_retries == 5 + assert result.max_buffer_size == 5000 + + await result.shutdown() + finally: + emitter_module._emitter = original + + @pytest.mark.asyncio + async def test_initialize_emitter_disabled(self): + """initialize_emitter() creates disabled emitter when enabled=False.""" + from src.telemetry import emitter as emitter_module + from src.telemetry.emitter import initialize_emitter + + original = emitter_module._emitter + + try: + emitter_module._emitter = None + + result = await initialize_emitter( + endpoint="http://test:8001/events", + enabled=False, + ) + + assert result is not None + assert result.enabled is False + assert result.is_running is False + + await result.shutdown() + finally: + emitter_module._emitter = original + + +# ============================================================================= +# Tests for shutdown_emitter() function +# ============================================================================= + + +class TestShutdownEmitter: + """Tests for the shutdown_emitter() function.""" + + @pytest.mark.asyncio + async def test_shutdown_emitter_when_none(self): + """shutdown_emitter() handles None emitter gracefully.""" + from src.telemetry import emitter as emitter_module + from src.telemetry.emitter import shutdown_emitter + + original = emitter_module._emitter + + try: + emitter_module._emitter = None + # Should not raise + await shutdown_emitter() + finally: + emitter_module._emitter = original + + @pytest.mark.asyncio + async def test_shutdown_emitter_calls_shutdown(self): + """shutdown_emitter() calls shutdown on the emitter.""" + from src.telemetry import emitter as emitter_module + from src.telemetry.emitter import shutdown_emitter + + original = emitter_module._emitter + + try: + mock_emitter = AsyncMock() + emitter_module._emitter = mock_emitter + + await shutdown_emitter() + + mock_emitter.shutdown.assert_called_once() + # Should clear global + assert emitter_module._emitter is None + finally: + emitter_module._emitter = original + + +# ============================================================================= +# Tests for initialize_telemetry_events() function +# ============================================================================= + + +class TestInitializeTelemetryEvents: + """Tests for initialize_telemetry_events() in src.telemetry.events.""" + + @pytest.mark.asyncio + async def test_initialize_when_disabled(self): + """initialize_telemetry_events() is no-op when TELEMETRY.ENABLED=False.""" + from src.telemetry.events import initialize_telemetry_events + + with ( + patch("src.config.settings") as mock_settings, + patch("src.telemetry.emitter.initialize_emitter") as mock_init, + ): + mock_settings.TELEMETRY.ENABLED = False + + await initialize_telemetry_events() + + mock_init.assert_not_called() + + @pytest.mark.asyncio + async def test_initialize_when_enabled(self): + """initialize_telemetry_events() initializes emitter when enabled.""" + from src.telemetry.events import initialize_telemetry_events + + with ( + patch("src.config.settings") as mock_settings, + patch("src.telemetry.emitter.initialize_emitter") as mock_init, + ): + mock_settings.TELEMETRY.ENABLED = True + mock_settings.TELEMETRY.ENDPOINT = "http://test:8001/events" + mock_settings.TELEMETRY.HEADERS = {"X-API-Key": "secret"} + mock_settings.TELEMETRY.BATCH_SIZE = 100 + mock_settings.TELEMETRY.FLUSH_INTERVAL_SECONDS = 1.0 + mock_settings.TELEMETRY.FLUSH_THRESHOLD = 50 + mock_settings.TELEMETRY.MAX_RETRIES = 3 + mock_settings.TELEMETRY.MAX_BUFFER_SIZE = 10000 + mock_init.return_value = AsyncMock() + + await initialize_telemetry_events() + + mock_init.assert_called_once_with( + endpoint="http://test:8001/events", + headers={"X-API-Key": "secret"}, + batch_size=100, + flush_interval_seconds=1.0, + flush_threshold=50, + max_retries=3, + max_buffer_size=10000, + enabled=True, + ) + + +# ============================================================================= +# Tests for shutdown_telemetry_events() function +# ============================================================================= + + +class TestShutdownTelemetryEvents: + """Tests for shutdown_telemetry_events() in src.telemetry.events.""" + + @pytest.mark.asyncio + async def test_shutdown_calls_shutdown_emitter(self): + """shutdown_telemetry_events() calls shutdown_emitter.""" + from src.telemetry.events import shutdown_telemetry_events + + with patch( + "src.telemetry.emitter.shutdown_emitter", new_callable=AsyncMock + ) as mock_shutdown: + await shutdown_telemetry_events() + + mock_shutdown.assert_called_once() + + +# ============================================================================= +# Tests for initialize_telemetry() function +# ============================================================================= + + +class TestInitializeTelemetry: + """Tests for initialize_telemetry() in src.telemetry.""" + + def test_initialize_otel_when_enabled(self): + """initialize_telemetry() initializes OTel metrics when enabled.""" + import src.telemetry as telemetry_module + + with ( + patch("src.config.settings") as mock_settings, + # Patch where it's used (in src.telemetry), not where it's defined + patch.object(telemetry_module, "initialize_otel_metrics") as mock_otel_init, + ): + mock_settings.OTEL.ENABLED = True + mock_settings.OTEL.ENDPOINT = "http://otel:9009/metrics" + mock_settings.OTEL.HEADERS = None + mock_settings.OTEL.EXPORT_INTERVAL_MILLIS = 60000 + mock_settings.OTEL.SERVICE_NAME = "honcho" + mock_settings.OTEL.SERVICE_NAMESPACE = "test" + + telemetry_module.initialize_telemetry() + + mock_otel_init.assert_called_once_with( + endpoint="http://otel:9009/metrics", + headers=None, + export_interval_millis=60000, + service_name="honcho", + service_namespace="test", + enabled=True, + ) + + def test_skip_otel_when_disabled(self): + """initialize_telemetry() skips OTel when disabled.""" + import src.telemetry as telemetry_module + + with ( + patch("src.config.settings") as mock_settings, + patch.object(telemetry_module, "initialize_otel_metrics") as mock_otel_init, + ): + mock_settings.OTEL.ENABLED = False + + telemetry_module.initialize_telemetry() + + mock_otel_init.assert_not_called() + + +# ============================================================================= +# Tests for initialize_telemetry_async() function +# ============================================================================= + + +class TestInitializeTelemetryAsync: + """Tests for initialize_telemetry_async() in src.telemetry.""" + + @pytest.mark.asyncio + async def test_initialize_cloudevents_when_enabled(self): + """initialize_telemetry_async() initializes CloudEvents when enabled.""" + from src.telemetry import initialize_telemetry_async + + with ( + patch("src.config.settings") as mock_settings, + patch( + "src.telemetry.events.initialize_telemetry_events", + new_callable=AsyncMock, + ) as mock_ce_init, + ): + mock_settings.TELEMETRY.ENABLED = True + + await initialize_telemetry_async() + + mock_ce_init.assert_called_once() + + @pytest.mark.asyncio + async def test_skip_cloudevents_when_disabled(self): + """initialize_telemetry_async() skips CloudEvents when disabled.""" + from src.telemetry import initialize_telemetry_async + + with ( + patch("src.config.settings") as mock_settings, + patch( + "src.telemetry.events.initialize_telemetry_events", + new_callable=AsyncMock, + ) as mock_ce_init, + ): + mock_settings.TELEMETRY.ENABLED = False + + await initialize_telemetry_async() + + mock_ce_init.assert_not_called() + + +# ============================================================================= +# Tests for shutdown_telemetry() function +# ============================================================================= + + +class TestShutdownTelemetry: + """Tests for shutdown_telemetry() in src.telemetry.""" + + @pytest.mark.asyncio + async def test_shutdown_calls_all_subsystems(self): + """shutdown_telemetry() shuts down all telemetry subsystems.""" + from src.telemetry import shutdown_telemetry + + with ( + patch( + "src.telemetry.events.shutdown_telemetry_events", new_callable=AsyncMock + ) as mock_ce_shutdown, + patch("src.telemetry.shutdown_otel_metrics") as mock_otel_shutdown, + ): + await shutdown_telemetry() + + mock_ce_shutdown.assert_called_once() + mock_otel_shutdown.assert_called_once() diff --git a/tests/telemetry/test_emitter.py b/tests/telemetry/test_emitter.py new file mode 100644 index 00000000..5d1e67a1 --- /dev/null +++ b/tests/telemetry/test_emitter.py @@ -0,0 +1,982 @@ +# pyright: reportPrivateUsage=false, reportUnknownParameterType=false, reportMissingParameterType=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportUnusedParameter=false +"""Unit tests for TelemetryEmitter class. + +Exhaustive tests covering: +- Initialization and configuration +- Disabled state behavior +- Buffer management (capacity, overflow, threshold) +- Flush mechanics (manual, threshold-triggered, periodic) +- HTTP sending (success, retries, batch formats) +- Error scenarios (network, HTTP status codes, timeouts) +- Lifecycle (start, shutdown, concurrent operations) +""" + +import asyncio +import contextlib +import json +from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from src.telemetry.emitter import TelemetryEmitter +from src.telemetry.events.representation import RepresentationCompletedEvent + +# ============================================================================= +# Helper to create minimal test events +# ============================================================================= + + +def create_test_event(message_id: str = "msg_001") -> RepresentationCompletedEvent: + """Create a minimal test event.""" + return RepresentationCompletedEvent( + timestamp=datetime(2024, 1, 15, 12, 0, 0, tzinfo=UTC), + workspace_name="test_workspace", + session_name="test_session", + observed="user_peer", + queue_items_processed=1, + earliest_message_id="msg_001", + latest_message_id=message_id, + message_count=1, + explicit_conclusion_count=1, + context_preparation_ms=10.0, + llm_call_ms=100.0, + total_duration_ms=110.0, + input_tokens=100, + output_tokens=50, + ) + + +# ============================================================================= +# Tests for Initialization +# ============================================================================= + + +class TestEmitterInitialization: + """Tests for TelemetryEmitter initialization.""" + + def test_default_initialization(self): + """Emitter initializes with default values.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + assert emitter.endpoint == "http://test:8001/events" + assert emitter.batch_size == 100 + assert emitter.flush_interval == 1.0 + assert emitter.flush_threshold == 50 + assert emitter.max_retries == 3 + assert emitter.max_buffer_size == 10000 + assert emitter.enabled is True + + def test_custom_initialization(self): + """Emitter initializes with custom values.""" + emitter = TelemetryEmitter( + endpoint="http://custom:9000/events", + headers={"X-API-Key": "secret"}, + batch_size=50, + flush_interval_seconds=2.5, + flush_threshold=25, + max_retries=5, + max_buffer_size=5000, + enabled=True, + ) + + assert emitter.endpoint == "http://custom:9000/events" + assert emitter.headers == {"X-API-Key": "secret"} + assert emitter.batch_size == 50 + assert emitter.flush_interval == 2.5 + assert emitter.flush_threshold == 25 + assert emitter.max_retries == 5 + assert emitter.max_buffer_size == 5000 + + def test_disabled_when_no_endpoint(self): + """Emitter is disabled when endpoint is None.""" + emitter = TelemetryEmitter(endpoint=None, enabled=True) + assert emitter.enabled is False + + def test_disabled_when_explicitly_disabled(self): + """Emitter is disabled when enabled=False.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events", enabled=False) + assert emitter.enabled is False + + def test_initial_state(self): + """Emitter starts in non-running state with empty buffer.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + assert emitter.buffer_size == 0 + assert emitter.is_running is False + assert emitter._client is None + assert emitter._flush_task is None + + +# ============================================================================= +# Tests for Disabled State +# ============================================================================= + + +class TestDisabledEmitter: + """Tests for emitter behavior when disabled.""" + + @pytest.mark.asyncio + async def test_start_when_disabled(self, disabled_emitter: TelemetryEmitter): + """start() is no-op when disabled.""" + await disabled_emitter.start() + + assert disabled_emitter.is_running is False + assert disabled_emitter._client is None + + @pytest.mark.asyncio + async def test_emit_when_disabled(self, disabled_emitter: TelemetryEmitter): + """emit() is no-op when disabled.""" + event = create_test_event() + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + disabled_emitter.emit(event) + + assert disabled_emitter.buffer_size == 0 + + @pytest.mark.asyncio + async def test_flush_when_disabled(self, disabled_emitter: TelemetryEmitter): + """flush() is no-op when disabled.""" + await disabled_emitter.flush() + # Should complete without error + + @pytest.mark.asyncio + async def test_shutdown_when_disabled(self, disabled_emitter: TelemetryEmitter): + """shutdown() is no-op when disabled.""" + await disabled_emitter.shutdown() + # Should complete without error + + +# ============================================================================= +# Tests for Buffer Management +# ============================================================================= + + +class TestBufferManagement: + """Tests for buffer capacity and overflow handling.""" + + def test_emit_adds_to_buffer(self): + """emit() adds events to the buffer.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + emitter._running = True # Simulate started state without full start() + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + + assert emitter.buffer_size == 1 + + def test_buffer_respects_max_size(self): + """Buffer drops oldest events when max size exceeded.""" + # Create an emitter with small buffer but high flush threshold + # to prevent auto-flush from interfering with this synchronous test + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_buffer_size=5, + flush_threshold=100, # High threshold to prevent auto-flush + enabled=True, + ) + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Add more events than buffer can hold + for i in range(10): + event = create_test_event(f"msg_{i:03d}") + emitter.emit(event) + + # Buffer should be at max size (5) - deque drops oldest events + assert emitter.buffer_size == 5 + + def test_buffer_warns_at_80_percent_capacity(self, caplog): + """Warning logged when buffer reaches 80% capacity.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_buffer_size=10, + ) + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Add 8 events (80% of 10) + for i in range(8): + event = create_test_event(f"msg_{i:03d}") + emitter.emit(event) + + # Check warning was logged + assert any( + "80%" in record.message or "capacity" in record.message.lower() + for record in caplog.records + if record.levelname == "WARNING" + ) + + @pytest.mark.asyncio + async def test_flush_threshold_triggers_flush(self): + """Flush is triggered when buffer reaches threshold.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + flush_threshold=3, + max_buffer_size=100, + ) + + # Mock the HTTP client + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Add events up to threshold + for i in range(3): + event = create_test_event(f"msg_{i:03d}") + emitter.emit(event) + + # Give async task time to run + await asyncio.sleep(0.1) + + # Flush should have been triggered + assert mock_client.post.called + + +# ============================================================================= +# Tests for HTTP Sending +# ============================================================================= + + +class TestHttpSending: + """Tests for HTTP request formatting and sending.""" + + @pytest.mark.asyncio + async def test_single_event_uses_structured_format(self): + """Single event is sent as structured CloudEvent JSON.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + + await emitter.flush() + + # Check that post was called + assert mock_client.post.called + call_args = mock_client.post.call_args + + # Single event should NOT have batch header override + headers = call_args.kwargs.get("headers") + assert headers is None # No override for single event + + @pytest.mark.asyncio + async def test_multiple_events_use_batch_format(self): + """Multiple events are sent as CloudEvents batch (JSON array).""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=10, + ) + + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Add multiple events + for i in range(3): + event = create_test_event(f"msg_{i:03d}") + emitter.emit(event) + + await emitter.flush() + + # Check that post was called with batch header + call_args = mock_client.post.call_args + headers = call_args.kwargs.get("headers") + assert headers == {"Content-Type": "application/cloudevents-batch+json"} + + # Content should be a JSON array + content = call_args.kwargs.get("content") + parsed = json.loads(content) + assert isinstance(parsed, list) + assert len(parsed) == 3 + + @pytest.mark.asyncio + async def test_batch_respects_batch_size(self): + """Large buffers are sent in batch_size chunks.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + batch_size=2, # Small batch size + ) + + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + # Add 5 events + for i in range(5): + event = create_test_event(f"msg_{i:03d}") + emitter.emit(event) + + await emitter.flush() + + # Should have made 3 calls (2, 2, 1) + assert mock_client.post.call_count == 3 + + +# ============================================================================= +# Tests for Retry Logic +# ============================================================================= + + +class TestRetryLogic: + """Tests for exponential backoff retry behavior.""" + + @pytest.mark.asyncio + async def test_retry_on_http_error(self): + """Emitter retries on HTTP errors.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=3, + ) + + # First two calls fail, third succeeds + mock_client = AsyncMock() + fail_response = MagicMock() + fail_response.status_code = 500 + fail_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError( + "500", request=MagicMock(), response=fail_response + ) + ) + + success_response = MagicMock() + success_response.status_code = 200 + success_response.raise_for_status = MagicMock() + + mock_client.post = AsyncMock( + side_effect=[fail_response, fail_response, success_response] + ) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), # Skip actual delays + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + # Should have tried 3 times + assert mock_client.post.call_count == 3 + # Buffer should be empty (success on 3rd try) + assert emitter.buffer_size == 0 + + @pytest.mark.asyncio + async def test_retry_on_request_error(self): + """Emitter retries on network/request errors.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=3, + ) + + mock_client = AsyncMock() + success_response = MagicMock() + success_response.status_code = 200 + success_response.raise_for_status = MagicMock() + + # First call raises connection error, second succeeds + mock_client.post = AsyncMock( + side_effect=[ + httpx.ConnectError("Connection refused"), + success_response, + ] + ) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + assert mock_client.post.call_count == 2 + + @pytest.mark.asyncio + async def test_all_retries_exhausted(self): + """Events returned to buffer when all retries fail.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=3, + ) + + mock_client = AsyncMock() + fail_response = MagicMock() + fail_response.status_code = 500 + fail_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError( + "500", request=MagicMock(), response=fail_response + ) + ) + mock_client.post = AsyncMock(return_value=fail_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + # All 3 retries should have been attempted + assert mock_client.post.call_count == 3 + # Event should be back in buffer + assert emitter.buffer_size == 1 + + @pytest.mark.asyncio + async def test_exponential_backoff_timing(self): + """Retry uses exponential backoff delays.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=4, + ) + + mock_client = AsyncMock() + fail_response = MagicMock() + fail_response.status_code = 503 + fail_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError( + "503", request=MagicMock(), response=fail_response + ) + ) + mock_client.post = AsyncMock(return_value=fail_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + sleep_calls = [] + + async def track_sleep(delay): + sleep_calls.append(delay) + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", side_effect=track_sleep), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + # Should have delays of 1, 2, 4 (2^0, 2^1, 2^2) before attempts 2, 3, 4 + assert sleep_calls == [1, 2, 4] + + +# ============================================================================= +# Tests for HTTP Error Scenarios +# ============================================================================= + + +class TestHttpErrorScenarios: + """Tests for various HTTP error status codes.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "status_code,should_retry", + [ + (400, True), # Bad Request - still retries + (401, True), # Unauthorized + (403, True), # Forbidden + (404, True), # Not Found + (429, True), # Too Many Requests + (500, True), # Internal Server Error + (502, True), # Bad Gateway + (503, True), # Service Unavailable + (504, True), # Gateway Timeout + ], + ) + async def test_http_status_codes(self, status_code: int, should_retry: bool): + """Various HTTP status codes trigger retry behavior.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_retries=2, + ) + + mock_client = AsyncMock() + fail_response = MagicMock() + fail_response.status_code = status_code + fail_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError( + f"HTTP {status_code}", request=MagicMock(), response=fail_response + ) + ) + mock_client.post = AsyncMock(return_value=fail_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + if should_retry: + assert mock_client.post.call_count == 2 # max_retries + + +# ============================================================================= +# Tests for Network Error Scenarios +# ============================================================================= + + +class TestNetworkErrorScenarios: + """Tests for various network error conditions.""" + + @pytest.mark.asyncio + async def test_connection_refused(self): + """Connection refused error triggers retry.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events", max_retries=2) + + mock_client = AsyncMock() + mock_client.post = AsyncMock( + side_effect=httpx.ConnectError("Connection refused") + ) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + assert mock_client.post.call_count == 2 + + @pytest.mark.asyncio + async def test_connection_timeout(self): + """Connection timeout error triggers retry.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events", max_retries=2) + + mock_client = AsyncMock() + mock_client.post = AsyncMock( + side_effect=httpx.ConnectTimeout("Connection timed out") + ) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + assert mock_client.post.call_count == 2 + + @pytest.mark.asyncio + async def test_read_timeout(self): + """Read timeout error triggers retry.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events", max_retries=2) + + mock_client = AsyncMock() + mock_client.post = AsyncMock(side_effect=httpx.ReadTimeout("Read timed out")) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with ( + patch("src.config.settings") as mock_settings, + patch("asyncio.sleep", new_callable=AsyncMock), + ): + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + await emitter.flush() + + assert mock_client.post.call_count == 2 + + +# ============================================================================= +# Tests for Lifecycle Management +# ============================================================================= + + +class TestLifecycle: + """Tests for emitter lifecycle (start, shutdown).""" + + @pytest.mark.asyncio + async def test_start_creates_client_and_task(self): + """start() creates HTTP client and periodic flush task.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value = mock_client + + await emitter.start() + + assert emitter.is_running is True + assert emitter._client is not None + assert emitter._flush_task is not None + + # Clean up + emitter._running = False + emitter._flush_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await emitter._flush_task + + @pytest.mark.asyncio + async def test_shutdown_flushes_and_closes(self): + """shutdown() flushes buffer and closes client.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + emitter._flush_task = asyncio.create_task(asyncio.sleep(100)) + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + + assert emitter.buffer_size == 1 + + await emitter.shutdown() + + # Client should be closed + assert mock_client.aclose.called + assert emitter._client is None + assert emitter.is_running is False + + @pytest.mark.asyncio + async def test_shutdown_with_empty_buffer(self): + """shutdown() handles empty buffer gracefully.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + mock_client = AsyncMock() + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + emitter._flush_task = asyncio.create_task(asyncio.sleep(100)) + + await emitter.shutdown() + + assert mock_client.aclose.called + # post should not have been called (nothing to flush) + assert not mock_client.post.called + + @pytest.mark.asyncio + async def test_double_shutdown_is_safe(self, disabled_emitter: TelemetryEmitter): + """Multiple shutdown calls are idempotent.""" + await disabled_emitter.shutdown() + await disabled_emitter.shutdown() + # Should complete without error + + @pytest.mark.asyncio + async def test_emit_after_shutdown(self): + """Emit after shutdown is a no-op (disabled).""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + mock_client = AsyncMock() + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + emitter._flush_task = asyncio.create_task(asyncio.sleep(100)) + + await emitter.shutdown() + + # Emit after shutdown + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + + # Should not be added (emitter no longer running) + # Note: the emit checks self.enabled, not self._running + # After shutdown, enabled is still True but _running is False + + +# ============================================================================= +# Tests for Concurrent Operations +# ============================================================================= + + +class TestConcurrentOperations: + """Tests for concurrent emit and flush operations.""" + + @pytest.mark.asyncio + async def test_concurrent_emits(self): + """Multiple concurrent emits are thread-safe.""" + emitter = TelemetryEmitter( + endpoint="http://test:8001/events", + max_buffer_size=1000, + flush_threshold=1000, # High threshold to prevent auto-flush + ) + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + + # Emit many events concurrently + for i in range(100): + event = create_test_event(f"msg_{i:03d}") + emitter.emit(event) + + assert emitter.buffer_size == 100 + + @pytest.mark.asyncio + async def test_flush_uses_lock(self): + """Concurrent flush calls are serialized via lock.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + mock_client = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.raise_for_status = MagicMock() + + # Slow post to simulate long-running flush + async def slow_post(*args, **kwargs): + await asyncio.sleep(0.1) + return mock_response + + mock_client.post = slow_post + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + for i in range(5): + event = create_test_event(f"msg_{i:03d}") + emitter.emit(event) + + # Start two concurrent flushes + await asyncio.gather(emitter.flush(), emitter.flush()) + + # Both should complete without error + # Buffer should be empty + assert emitter.buffer_size == 0 + + +# ============================================================================= +# Tests for Edge Cases +# ============================================================================= + + +class TestEdgeCases: + """Tests for edge cases and boundary conditions.""" + + @pytest.mark.asyncio + async def test_flush_empty_buffer(self): + """Flushing empty buffer is a no-op.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + mock_client = AsyncMock() + emitter._client = mock_client + emitter._running = True + + await emitter.flush() + + # post should not have been called + assert not mock_client.post.called + + @pytest.mark.asyncio + async def test_flush_without_client(self): + """Flush without client (not started) is a no-op.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + event = create_test_event() + emitter.emit(event) + + # Flush before start (no client) + await emitter.flush() + + # Should complete without error, buffer unchanged + assert emitter.buffer_size == 1 + + def test_buffer_size_property(self): + """buffer_size property returns current buffer length.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + emitter._running = True + + assert emitter.buffer_size == 0 + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test" + emitter.emit(create_test_event("msg_001")) + assert emitter.buffer_size == 1 + + emitter.emit(create_test_event("msg_002")) + assert emitter.buffer_size == 2 + + def test_is_running_property(self): + """is_running property reflects internal state.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + assert emitter.is_running is False + + emitter._running = True + assert emitter.is_running is True + + emitter._running = False + assert emitter.is_running is False + + +# ============================================================================= +# Tests for CloudEvent Format +# ============================================================================= + + +class TestCloudEventFormat: + """Tests for correct CloudEvent formatting.""" + + @pytest.mark.asyncio + async def test_cloudevent_attributes(self): + """CloudEvent has correct attributes.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + captured_content = None + + async def capture_post(url, content=None, headers=None): + nonlocal captured_content + captured_content = content + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + return response + + mock_client = AsyncMock() + mock_client.post = capture_post + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = "test_ns" + event = create_test_event() + emitter.emit(event) + + await emitter.flush() + + # Parse the captured content + assert captured_content is not None + cloud_event = json.loads(captured_content) + + # Check CloudEvent required attributes + assert "id" in cloud_event + assert cloud_event["id"].startswith("evt_") + assert "source" in cloud_event + assert cloud_event["source"] == "/honcho/test_ns/representation" + assert "type" in cloud_event + assert cloud_event["type"] == "representation.completed" + assert "time" in cloud_event + assert "dataschema" in cloud_event + assert "data" in cloud_event + + @pytest.mark.asyncio + async def test_cloudevent_source_without_namespace(self): + """CloudEvent source format when namespace is None.""" + emitter = TelemetryEmitter(endpoint="http://test:8001/events") + + captured_content = None + + async def capture_post(url, content=None, headers=None): + nonlocal captured_content + captured_content = content + response = MagicMock() + response.status_code = 200 + response.raise_for_status = MagicMock() + return response + + mock_client = AsyncMock() + mock_client.post = capture_post + mock_client.aclose = AsyncMock() + + emitter._client = mock_client + emitter._running = True + + with patch("src.config.settings") as mock_settings: + mock_settings.TELEMETRY.NAMESPACE = None + event = create_test_event() + emitter.emit(event) + + await emitter.flush() + + assert captured_content is not None + cloud_event = json.loads(captured_content) + # Without namespace, source should be /honcho/{category} + assert cloud_event["source"] == "/honcho/representation" diff --git a/tests/telemetry/test_events.py b/tests/telemetry/test_events.py new file mode 100644 index 00000000..94f34aaa --- /dev/null +++ b/tests/telemetry/test_events.py @@ -0,0 +1,724 @@ +# pyright: reportUnknownParameterType=false, reportMissingParameterType=false, reportUnusedParameter=false +"""Unit tests for telemetry event classes. + +Tests all 12 event types for: +- Correct instantiation with required fields +- event_type(), schema_version(), category() class methods +- get_resource_id() returns expected format +- generate_id() is deterministic (same inputs = same ID) +- Timestamp defaults to UTC now +""" + +from datetime import UTC, datetime + +import pytest +from pydantic import ValidationError + +from src.telemetry.events.agent import ( + AgentIterationEvent, + AgentToolConclusionsCreatedEvent, + AgentToolConclusionsDeletedEvent, + AgentToolPeerCardUpdatedEvent, + AgentToolSummaryCreatedEvent, +) +from src.telemetry.events.base import BaseEvent, generate_event_id +from src.telemetry.events.deletion import DeletionCompletedEvent +from src.telemetry.events.dialectic import DialecticCompletedEvent +from src.telemetry.events.dream import DreamRunEvent, DreamSpecialistEvent +from src.telemetry.events.reconciliation import ( + CleanupStaleItemsCompletedEvent, + SyncVectorsCompletedEvent, +) +from src.telemetry.events.representation import RepresentationCompletedEvent + +# ============================================================================= +# Tests for generate_event_id function +# ============================================================================= + + +class TestGenerateEventId: + """Tests for the generate_event_id function.""" + + def test_deterministic_output(self, fixed_timestamp: datetime): + """Same inputs always produce the same ID.""" + event_type = "test.event" + resource_id = "resource_123" + + id1 = generate_event_id(event_type, fixed_timestamp, resource_id) + id2 = generate_event_id(event_type, fixed_timestamp, resource_id) + + assert id1 == id2 + + def test_different_event_types_produce_different_ids( + self, fixed_timestamp: datetime + ): + """Different event types produce different IDs.""" + resource_id = "resource_123" + + id1 = generate_event_id("type.a", fixed_timestamp, resource_id) + id2 = generate_event_id("type.b", fixed_timestamp, resource_id) + + assert id1 != id2 + + def test_different_timestamps_produce_different_ids(self): + """Different timestamps produce different IDs.""" + event_type = "test.event" + resource_id = "resource_123" + ts1 = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) + ts2 = datetime(2024, 1, 1, 12, 0, 1, tzinfo=UTC) + + id1 = generate_event_id(event_type, ts1, resource_id) + id2 = generate_event_id(event_type, ts2, resource_id) + + assert id1 != id2 + + def test_different_resource_ids_produce_different_ids( + self, fixed_timestamp: datetime + ): + """Different resource IDs produce different IDs.""" + event_type = "test.event" + + id1 = generate_event_id(event_type, fixed_timestamp, "resource_a") + id2 = generate_event_id(event_type, fixed_timestamp, "resource_b") + + assert id1 != id2 + + def test_id_format(self, fixed_timestamp: datetime): + """Event ID has correct format: evt_{base64_hash}.""" + event_id = generate_event_id("test.event", fixed_timestamp, "resource_123") + + assert event_id.startswith("evt_") + # Base64url encoded 16 bytes = 22 chars (without padding) + assert len(event_id) == 4 + 22 # "evt_" + 22 chars + + +# ============================================================================= +# Tests for BaseEvent class +# ============================================================================= + + +class TestBaseEvent: + """Tests for BaseEvent base class behavior.""" + + def test_timestamp_defaults_to_utc_now(self): + """Events without explicit timestamp get current UTC time.""" + # Create event without timestamp + event = RepresentationCompletedEvent( + workspace_name="test", + session_name="test_session", + observed="user", + queue_items_processed=1, + earliest_message_id="msg_1", + latest_message_id="msg_2", + message_count=2, + explicit_conclusion_count=1, + context_preparation_ms=10.0, + llm_call_ms=100.0, + total_duration_ms=110.0, + input_tokens=100, + output_tokens=50, + ) + + # Timestamp should be set and be UTC + assert event.timestamp is not None + assert event.timestamp.tzinfo == UTC + + def test_get_resource_id_not_implemented_on_base(self): + """BaseEvent.get_resource_id raises NotImplementedError.""" + # Can't instantiate BaseEvent directly, but we can test the method + with pytest.raises(NotImplementedError, match="Subclasses must implement"): + BaseEvent.get_resource_id(BaseEvent(timestamp=datetime.now(UTC))) + + +# ============================================================================= +# Tests for RepresentationCompletedEvent +# ============================================================================= + + +class TestRepresentationCompletedEvent: + """Tests for RepresentationCompletedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert RepresentationCompletedEvent.event_type() == "representation.completed" + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert RepresentationCompletedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert RepresentationCompletedEvent.category() == "representation" + + def test_get_resource_id( + self, sample_representation_event: RepresentationCompletedEvent + ): + """get_resource_id() returns workspace:session:message format.""" + resource_id = sample_representation_event.get_resource_id() + assert resource_id == "test_workspace:test_session:msg_010" + + def test_generate_id_deterministic( + self, sample_representation_event: RepresentationCompletedEvent + ): + """generate_id() produces same ID for same event.""" + id1 = sample_representation_event.generate_id() + id2 = sample_representation_event.generate_id() + assert id1 == id2 + assert id1.startswith("evt_") + + def test_required_fields(self): + """Event requires all mandatory fields.""" + with pytest.raises(ValidationError): + RepresentationCompletedEvent() # pyright: ignore[reportCallIssue] + + def test_model_dump( + self, sample_representation_event: RepresentationCompletedEvent + ): + """Event can be serialized to dict.""" + data = sample_representation_event.model_dump(mode="json") + assert data["workspace_name"] == "test_workspace" + assert data["message_count"] == 10 + assert data["explicit_conclusion_count"] == 5 + + +# ============================================================================= +# Tests for DreamRunEvent +# ============================================================================= + + +class TestDreamRunEvent: + """Tests for DreamRunEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert DreamRunEvent.event_type() == "dream.run" + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert DreamRunEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert DreamRunEvent.category() == "dream" + + def test_get_resource_id(self, sample_dream_run_event: DreamRunEvent): + """get_resource_id() returns run_id.""" + assert sample_dream_run_event.get_resource_id() == "abc12345" + + def test_generate_id_deterministic(self, sample_dream_run_event: DreamRunEvent): + """generate_id() produces same ID for same event.""" + id1 = sample_dream_run_event.generate_id() + id2 = sample_dream_run_event.generate_id() + assert id1 == id2 + + def test_specialists_run_list(self, sample_dream_run_event: DreamRunEvent): + """specialists_run contains expected values.""" + assert "deduction" in sample_dream_run_event.specialists_run + assert "induction" in sample_dream_run_event.specialists_run + + +# ============================================================================= +# Tests for DreamSpecialistEvent +# ============================================================================= + + +class TestDreamSpecialistEvent: + """Tests for DreamSpecialistEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert DreamSpecialistEvent.event_type() == "dream.specialist" + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert DreamSpecialistEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert DreamSpecialistEvent.category() == "dream" + + def test_get_resource_id(self, sample_dream_specialist_event: DreamSpecialistEvent): + """get_resource_id() returns run_id:specialist_type format.""" + assert sample_dream_specialist_event.get_resource_id() == "abc12345:deduction" + + def test_generate_id_deterministic( + self, sample_dream_specialist_event: DreamSpecialistEvent + ): + """generate_id() produces same ID for same event.""" + id1 = sample_dream_specialist_event.generate_id() + id2 = sample_dream_specialist_event.generate_id() + assert id1 == id2 + + +# ============================================================================= +# Tests for DialecticCompletedEvent +# ============================================================================= + + +class TestDialecticCompletedEvent: + """Tests for DialecticCompletedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert DialecticCompletedEvent.event_type() == "dialectic.completed" + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert DialecticCompletedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert DialecticCompletedEvent.category() == "dialectic" + + def test_get_resource_id(self, sample_dialectic_event: DialecticCompletedEvent): + """get_resource_id() returns run_id.""" + assert sample_dialectic_event.get_resource_id() == "def67890" + + def test_generate_id_deterministic( + self, sample_dialectic_event: DialecticCompletedEvent + ): + """generate_id() produces same ID for same event.""" + id1 = sample_dialectic_event.generate_id() + id2 = sample_dialectic_event.generate_id() + assert id1 == id2 + + def test_optional_session_fields(self, fixed_timestamp: datetime): + """session_id and session_name are optional.""" + event = DialecticCompletedEvent( + timestamp=fixed_timestamp, + run_id="test123", + workspace_name="test", + peer_name="user", + reasoning_level="low", + total_duration_ms=1000.0, + input_tokens=500, + output_tokens=100, + ) + assert event.session_name is None + + def test_cache_token_defaults(self, fixed_timestamp: datetime): + """Cache tokens default to 0.""" + event = DialecticCompletedEvent( + timestamp=fixed_timestamp, + run_id="test123", + workspace_name="test", + peer_name="user", + reasoning_level="low", + total_duration_ms=1000.0, + input_tokens=500, + output_tokens=100, + ) + assert event.cache_read_tokens == 0 + assert event.cache_creation_tokens == 0 + + +# ============================================================================= +# Tests for AgentIterationEvent +# ============================================================================= + + +class TestAgentIterationEvent: + """Tests for AgentIterationEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert AgentIterationEvent.event_type() == "agent.iteration" + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert AgentIterationEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert AgentIterationEvent.category() == "agent" + + def test_get_resource_id(self, sample_agent_iteration_event: AgentIterationEvent): + """get_resource_id() returns run_id:iteration format.""" + assert sample_agent_iteration_event.get_resource_id() == "abc12345:3" + + def test_generate_id_deterministic( + self, sample_agent_iteration_event: AgentIterationEvent + ): + """generate_id() produces same ID for same event.""" + id1 = sample_agent_iteration_event.generate_id() + id2 = sample_agent_iteration_event.generate_id() + assert id1 == id2 + + def test_tool_calls_list(self, sample_agent_iteration_event: AgentIterationEvent): + """tool_calls is a list of strings.""" + assert isinstance(sample_agent_iteration_event.tool_calls, list) + assert "search_memory" in sample_agent_iteration_event.tool_calls + + def test_optional_peer_fields(self, fixed_timestamp: datetime): + """observer, observed, peer_id are optional.""" + event = AgentIterationEvent( + timestamp=fixed_timestamp, + run_id="test123", + parent_category="dialectic", + agent_type="dialectic", + workspace_name="test", + iteration=1, + input_tokens=100, + output_tokens=50, + ) + assert event.observer is None + assert event.observed is None + + +# ============================================================================= +# Tests for AgentToolConclusionsCreatedEvent +# ============================================================================= + + +class TestAgentToolConclusionsCreatedEvent: + """Tests for AgentToolConclusionsCreatedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert ( + AgentToolConclusionsCreatedEvent.event_type() + == "agent.tool.conclusions.created" + ) + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert AgentToolConclusionsCreatedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert AgentToolConclusionsCreatedEvent.category() == "agent" + + def test_get_resource_id( + self, sample_conclusions_created_event: AgentToolConclusionsCreatedEvent + ): + """get_resource_id() returns run_id:iteration:conclusions_created format.""" + assert ( + sample_conclusions_created_event.get_resource_id() + == "abc12345:3:conclusions_created" + ) + + def test_levels_list( + self, sample_conclusions_created_event: AgentToolConclusionsCreatedEvent + ): + """levels is a list matching conclusion_count.""" + assert len(sample_conclusions_created_event.levels) == 5 + assert sample_conclusions_created_event.conclusion_count == 5 + + +# ============================================================================= +# Tests for AgentToolConclusionsDeletedEvent +# ============================================================================= + + +class TestAgentToolConclusionsDeletedEvent: + """Tests for AgentToolConclusionsDeletedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert ( + AgentToolConclusionsDeletedEvent.event_type() + == "agent.tool.conclusions.deleted" + ) + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert AgentToolConclusionsDeletedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert AgentToolConclusionsDeletedEvent.category() == "agent" + + def test_get_resource_id( + self, sample_conclusions_deleted_event: AgentToolConclusionsDeletedEvent + ): + """get_resource_id() returns run_id:iteration:conclusions_deleted format.""" + assert ( + sample_conclusions_deleted_event.get_resource_id() + == "abc12345:5:conclusions_deleted" + ) + + +# ============================================================================= +# Tests for AgentToolPeerCardUpdatedEvent +# ============================================================================= + + +class TestAgentToolPeerCardUpdatedEvent: + """Tests for AgentToolPeerCardUpdatedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert ( + AgentToolPeerCardUpdatedEvent.event_type() == "agent.tool.peer_card.updated" + ) + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert AgentToolPeerCardUpdatedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert AgentToolPeerCardUpdatedEvent.category() == "agent" + + def test_get_resource_id( + self, sample_peer_card_updated_event: AgentToolPeerCardUpdatedEvent + ): + """get_resource_id() returns run_id:iteration:peer_card_updated format.""" + assert ( + sample_peer_card_updated_event.get_resource_id() + == "abc12345:7:peer_card_updated" + ) + + +# ============================================================================= +# Tests for AgentToolSummaryCreatedEvent +# ============================================================================= + + +class TestAgentToolSummaryCreatedEvent: + """Tests for AgentToolSummaryCreatedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert AgentToolSummaryCreatedEvent.event_type() == "agent.tool.summary.created" + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert AgentToolSummaryCreatedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert AgentToolSummaryCreatedEvent.category() == "agent" + + def test_get_resource_id( + self, sample_summary_created_event: AgentToolSummaryCreatedEvent + ): + """get_resource_id() returns run_id:iteration:summary_created format.""" + assert ( + sample_summary_created_event.get_resource_id() + == "ghi11111:1:summary_created" + ) + + def test_summary_type_values(self, fixed_timestamp: datetime): + """summary_type accepts 'short' and 'long'.""" + for summary_type in ["short", "long"]: + event = AgentToolSummaryCreatedEvent( + timestamp=fixed_timestamp, + run_id="test123", + iteration=1, + parent_category="representation", + agent_type="summarizer", + workspace_name="test", + session_name="test_session", + message_id="msg_1", + message_count=10, + message_seq_in_session=10, + summary_type=summary_type, + input_tokens=100, + output_tokens=50, + ) + assert event.summary_type == summary_type + + +# ============================================================================= +# Tests for DeletionCompletedEvent +# ============================================================================= + + +class TestDeletionCompletedEvent: + """Tests for DeletionCompletedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert DeletionCompletedEvent.event_type() == "deletion.completed" + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert DeletionCompletedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert DeletionCompletedEvent.category() == "deletion" + + def test_get_resource_id(self, sample_deletion_event: DeletionCompletedEvent): + """get_resource_id() returns workspace:type:resource format.""" + assert ( + sample_deletion_event.get_resource_id() + == "test_workspace:workspace:ws_123abc" + ) + + def test_cascade_counts_default_to_zero(self, fixed_timestamp: datetime): + """Cascade counts default to 0 for non-workspace deletions.""" + event = DeletionCompletedEvent( + timestamp=fixed_timestamp, + workspace_name="test", + deletion_type="session", + resource_id="sess_456", + success=True, + ) + assert event.peers_deleted == 0 + assert event.sessions_deleted == 0 + assert event.messages_deleted == 0 + assert event.conclusions_deleted == 0 + + def test_error_message_optional(self, fixed_timestamp: datetime): + """error_message is optional and defaults to None.""" + event = DeletionCompletedEvent( + timestamp=fixed_timestamp, + workspace_name="test", + deletion_type="session", + resource_id="sess_456", + success=True, + ) + assert event.error_message is None + + def test_failed_deletion_with_error(self, fixed_timestamp: datetime): + """Failed deletion can include error message.""" + event = DeletionCompletedEvent( + timestamp=fixed_timestamp, + workspace_name="test", + deletion_type="session", + resource_id="sess_456", + success=False, + error_message="Foreign key constraint violation", + ) + assert event.success is False + assert event.error_message == "Foreign key constraint violation" + + +# ============================================================================= +# Tests for SyncVectorsCompletedEvent +# ============================================================================= + + +class TestSyncVectorsCompletedEvent: + """Tests for SyncVectorsCompletedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert ( + SyncVectorsCompletedEvent.event_type() + == "reconciliation.sync_vectors.completed" + ) + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert SyncVectorsCompletedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert SyncVectorsCompletedEvent.category() == "reconciliation" + + def test_get_resource_id( + self, sample_sync_vectors_event: SyncVectorsCompletedEvent + ): + """get_resource_id() returns fixed string.""" + assert sample_sync_vectors_event.get_resource_id() == "sync_vectors" + + def test_metrics_default_to_zero(self, fixed_timestamp: datetime): + """Sync metrics default to 0.""" + event = SyncVectorsCompletedEvent( + timestamp=fixed_timestamp, + total_duration_ms=1000.0, + ) + assert event.documents_synced == 0 + assert event.documents_failed == 0 + assert event.message_embeddings_synced == 0 + assert event.message_embeddings_failed == 0 + + +# ============================================================================= +# Tests for CleanupStaleItemsCompletedEvent +# ============================================================================= + + +class TestCleanupStaleItemsCompletedEvent: + """Tests for CleanupStaleItemsCompletedEvent.""" + + def test_event_type(self): + """event_type() returns correct value.""" + assert ( + CleanupStaleItemsCompletedEvent.event_type() + == "reconciliation.cleanup_stale_items.completed" + ) + + def test_schema_version(self): + """schema_version() returns correct value.""" + assert CleanupStaleItemsCompletedEvent.schema_version() == 1 + + def test_category(self): + """category() returns correct value.""" + assert CleanupStaleItemsCompletedEvent.category() == "reconciliation" + + def test_get_resource_id( + self, sample_cleanup_event: CleanupStaleItemsCompletedEvent + ): + """get_resource_id() returns fixed string.""" + assert sample_cleanup_event.get_resource_id() == "cleanup_stale_items" + + def test_cleanup_metrics_default_to_zero(self, fixed_timestamp: datetime): + """Cleanup metrics default to 0.""" + event = CleanupStaleItemsCompletedEvent( + timestamp=fixed_timestamp, + total_duration_ms=500.0, + ) + assert event.documents_cleaned == 0 + assert event.queue_items_cleaned == 0 + + +# ============================================================================= +# Parametrized tests across all event types +# ============================================================================= + + +class TestAllEventTypes: + """Parametrized tests that run across all event types.""" + + def test_all_events_have_timestamp(self, all_sample_events: list[BaseEvent]): + """All events have a timestamp field.""" + for event in all_sample_events: + assert hasattr(event, "timestamp") + assert event.timestamp is not None + + def test_all_events_generate_valid_ids(self, all_sample_events: list[BaseEvent]): + """All events generate valid event IDs.""" + for event in all_sample_events: + event_id = event.generate_id() + assert event_id.startswith("evt_") + assert len(event_id) == 26 # "evt_" + 22 chars + + def test_all_events_have_event_type(self, all_sample_events: list[BaseEvent]): + """All events return a non-empty event_type.""" + for event in all_sample_events: + event_type = event.event_type() + assert event_type + assert isinstance(event_type, str) + + def test_all_events_have_category(self, all_sample_events: list[BaseEvent]): + """All events return a non-empty category.""" + for event in all_sample_events: + category = event.category() + assert category + assert isinstance(category, str) + + def test_all_events_have_schema_version(self, all_sample_events: list[BaseEvent]): + """All events return a positive schema version.""" + for event in all_sample_events: + version = event.schema_version() + assert version >= 1 + assert isinstance(version, int) + + def test_all_events_can_serialize_to_json(self, all_sample_events: list[BaseEvent]): + """All events can be serialized to JSON-compatible dict.""" + for event in all_sample_events: + data = event.model_dump(mode="json") + assert isinstance(data, dict) + # Timestamp should be serialized as ISO string + assert "timestamp" in data + + def test_all_events_have_get_resource_id(self, all_sample_events: list[BaseEvent]): + """All events implement get_resource_id().""" + for event in all_sample_events: + resource_id = event.get_resource_id() + assert resource_id + assert isinstance(resource_id, str) diff --git a/uv.lock b/uv.lock index d3def4f5..cd16eece 100644 --- a/uv.lock +++ b/uv.lock @@ -459,6 +459,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, ] +[[package]] +name = "cloudevents" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/aa/804bdb5f2f021fcc887eeabfa24bad0ffd4b150f60850ae88faa51d393a5/cloudevents-1.12.0.tar.gz", hash = "sha256:ebd5544ceb58c8378a0787b657a2ae895e929b80a82d6675cba63f0e8c5539e0", size = 34494, upload-time = "2025-06-02T18:58:45.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/b6/4e29b74bb40daa7580310a5ff0df5f121a08ce98340e01a960b668468aab/cloudevents-1.12.0-py3-none-any.whl", hash = "sha256:49196267f5f963d87ae156f93fc0fa32f4af69485f2c8e62e0db8b0b4b8b8921", size = 55762, upload-time = "2025-06-02T18:58:44.013Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -1018,6 +1030,7 @@ source = { virtual = "." } dependencies = [ { name = "alembic" }, { name = "cashews", extra = ["redis"] }, + { name = "cloudevents" }, { name = "fastapi", extra = ["standard"] }, { name = "fastapi-pagination" }, { name = "google-genai" }, @@ -1029,9 +1042,10 @@ dependencies = [ { name = "langfuse" }, { name = "nanoid" }, { name = "openai" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "pdfplumber" }, { name = "pgvector" }, - { name = "prometheus-client" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -1072,6 +1086,7 @@ dev = [ requires-dist = [ { name = "alembic", specifier = ">=1.14.0" }, { name = "cashews", extras = ["redis"], specifier = "==7.4.4" }, + { name = "cloudevents", specifier = ">=1.12.0" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.111.0" }, { name = "fastapi-pagination", specifier = ">=0.12.24" }, { name = "google-genai", specifier = ">=1.32.0" }, @@ -1083,9 +1098,10 @@ requires-dist = [ { name = "langfuse", specifier = ">=3.3.2" }, { name = "nanoid", specifier = ">=2.0.0" }, { name = "openai", specifier = ">=1.99.7" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.36.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.36.0" }, { name = "pdfplumber", specifier = ">=0.11.7" }, { name = "pgvector", specifier = ">=0.2.5" }, - { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.1.19" }, { name = "pydantic", specifier = ">=2.11.7" }, { name = "pydantic-settings", specifier = ">=2.10.1" }, @@ -2130,15 +2146,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/74/a88bf1b1efeae488a0c0b7bdf71429c313722d1fc0f377537fbe554e6180/pre_commit-4.2.0-py2.py3-none-any.whl", hash = "sha256:a009ca7205f1eb497d10b845e52c838a98b6cdd2102a6c8e4540e94ee75c58bd", size = 220707, upload-time = "2025-03-18T21:35:19.343Z" }, ] -[[package]] -name = "prometheus-client" -version = "0.23.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, -] - [[package]] name = "propcache" version = "0.4.1"