From 505d12f662ef447180b39be87c000c64f152705e Mon Sep 17 00:00:00 2001 From: Victor Kyriazakos Date: Tue, 14 Jul 2026 18:00:46 +0000 Subject: [PATCH] refactor(monitoring): scope telemetry substrate to gateway health/diagnostics export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvages the event-spine foundation from feat/telemetry-observability (emitter, typed events, OTLP streaming, redaction — authorship preserved in the preceding commits) and scopes it to the plane enterprise operators need today: gateway Service Health Monitoring plus redacted Operational Diagnostics, exported over OTLP. Dropped from the salvaged branch, deliberately: - run/model/tool trajectory capture (plugins/telemetry hooks, tel_spans) - the local JSONL + state.db tel_* store (monitoring is egress, not storage) - usage rollups/metrics, /insights integration, bulk export - hermes telemetry CLI (replaced by hermes monitoring status) Those planes — shared client usage metrics and enterprise trace telemetry — are being designed on the NeMo Relay integration with distinct consent, policy, and export boundaries; this keeps the monitoring plane content-free and independently enableable. Renames agent/telemetry -> agent/monitoring, config telemetry.* -> monitoring.*, and pins the otlp extra at OpenTelemetry 1.39.1 (matching uv.lock; 1.30.0 conflicts with mistralai>=2.4 on opentelemetry-api). --- agent/insights.py | 116 ----- agent/monitoring/__init__.py | 29 ++ agent/monitoring/emitter.py | 193 +++++++++ agent/monitoring/events.py | 69 +++ agent/monitoring/gateway_health.py | 388 +++++++++++++++++ agent/monitoring/gateway_health_export.py | 405 ++++++++++++++++++ .../otlp_exporter.py | 161 +++---- agent/monitoring/policy.py | 40 ++ agent/monitoring/redaction.py | 77 ++++ agent/telemetry/__init__.py | 30 -- agent/telemetry/emitter.py | 318 -------------- agent/telemetry/events.py | 111 ----- agent/telemetry/exporter_bulk.py | 139 ------ agent/telemetry/metrics.py | 219 ---------- agent/telemetry/policy.py | 72 ---- agent/telemetry/redaction.py | 187 -------- agent/telemetry/rollup.py | 144 ------- agent/telemetry/spans.py | 83 ---- cli-config.yaml.example | 43 -- docs/observability/README.md | 7 - docs/observability/monitoring.md | 101 +++++ docs/observability/telemetry.md | 228 ---------- gateway/run.py | 14 + gateway/status.py | 7 + hermes_cli/config.py | 43 ++ hermes_cli/main.py | 226 ++-------- hermes_cli/subcommands/monitoring.py | 36 ++ hermes_cli/subcommands/telemetry.py | 54 --- plugins/telemetry/__init__.py | 382 ----------------- plugins/telemetry/plugin.yaml | 12 - pyproject.toml | 9 +- .../gateway_health_export_probe.py | 92 ++++ .../observability/otel_capture_collector.py | 59 +++ tests/hermes_cli/test_plugins.py | 6 +- tests/{telemetry => monitoring}/__init__.py | 0 tests/monitoring/test_emitter.py | 86 ++++ tests/monitoring/test_export_redaction.py | 56 +++ .../monitoring/test_gateway_health_export.py | 342 +++++++++++++++ tests/monitoring/test_otlp_exporter.py | 125 ++++++ tests/telemetry/test_cli_telemetry.py | 108 ----- tests/telemetry/test_emitter.py | 107 ----- tests/telemetry/test_export_redaction.py | 113 ----- tests/telemetry/test_exporter_bulk.py | 102 ----- tests/telemetry/test_governance.py | 95 ---- tests/telemetry/test_insights_integration.py | 82 ---- tests/telemetry/test_otlp_exporter.py | 171 -------- tests/telemetry/test_plugin_autoload.py | 53 --- tests/telemetry/test_plugin_e2e.py | 127 ------ tests/telemetry/test_plugin_hooks.py | 134 ------ tests/telemetry/test_policy_consent.py | 53 --- tests/telemetry/test_rollup.py | 88 ---- tests/telemetry/test_schema_migration.py | 45 -- tests/telemetry/test_spans_trace.py | 109 ----- tools/lazy_deps.py | 18 +- uv.lock | 14 +- 55 files changed, 2295 insertions(+), 3833 deletions(-) create mode 100644 agent/monitoring/__init__.py create mode 100644 agent/monitoring/emitter.py create mode 100644 agent/monitoring/events.py create mode 100644 agent/monitoring/gateway_health.py create mode 100644 agent/monitoring/gateway_health_export.py rename agent/{telemetry => monitoring}/otlp_exporter.py (60%) create mode 100644 agent/monitoring/policy.py create mode 100644 agent/monitoring/redaction.py delete mode 100644 agent/telemetry/__init__.py delete mode 100644 agent/telemetry/emitter.py delete mode 100644 agent/telemetry/events.py delete mode 100644 agent/telemetry/exporter_bulk.py delete mode 100644 agent/telemetry/metrics.py delete mode 100644 agent/telemetry/policy.py delete mode 100644 agent/telemetry/redaction.py delete mode 100644 agent/telemetry/rollup.py delete mode 100644 agent/telemetry/spans.py create mode 100644 docs/observability/monitoring.md delete mode 100644 docs/observability/telemetry.md create mode 100644 hermes_cli/subcommands/monitoring.py delete mode 100644 hermes_cli/subcommands/telemetry.py delete mode 100644 plugins/telemetry/__init__.py delete mode 100644 plugins/telemetry/plugin.yaml create mode 100644 scripts/observability/gateway_health_export_probe.py create mode 100644 scripts/observability/otel_capture_collector.py rename tests/{telemetry => monitoring}/__init__.py (100%) create mode 100644 tests/monitoring/test_emitter.py create mode 100644 tests/monitoring/test_export_redaction.py create mode 100644 tests/monitoring/test_gateway_health_export.py create mode 100644 tests/monitoring/test_otlp_exporter.py delete mode 100644 tests/telemetry/test_cli_telemetry.py delete mode 100644 tests/telemetry/test_emitter.py delete mode 100644 tests/telemetry/test_export_redaction.py delete mode 100644 tests/telemetry/test_exporter_bulk.py delete mode 100644 tests/telemetry/test_governance.py delete mode 100644 tests/telemetry/test_insights_integration.py delete mode 100644 tests/telemetry/test_otlp_exporter.py delete mode 100644 tests/telemetry/test_plugin_autoload.py delete mode 100644 tests/telemetry/test_plugin_e2e.py delete mode 100644 tests/telemetry/test_plugin_hooks.py delete mode 100644 tests/telemetry/test_policy_consent.py delete mode 100644 tests/telemetry/test_rollup.py delete mode 100644 tests/telemetry/test_schema_migration.py delete mode 100644 tests/telemetry/test_spans_trace.py diff --git a/agent/insights.py b/agent/insights.py index b9370b35cc567..086150c279ef5 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -82,19 +82,6 @@ def _bar_chart(values: List[int], max_width: int = 20) -> List[str]: return ["█" * max(1, int(v / peak * max_width)) if v > 0 else "" for v in values] -def _fmt_ms(ms: float) -> str: - """Compact human duration from milliseconds (e.g. 850ms, 2.4s, 1.5m).""" - try: - ms = float(ms or 0) - except (TypeError, ValueError): - return "0ms" - if ms < 1000: - return f"{int(ms)}ms" - if ms < 60_000: - return f"{ms / 1000:.1f}s" - return f"{ms / 60_000:.1f}m" - - class InsightsEngine: """ Analyzes session history and produces usage insights. @@ -152,7 +139,6 @@ class InsightsEngine: }, "activity": {}, "top_sessions": [], - "telemetry": {}, } # Compute insights @@ -163,7 +149,6 @@ class InsightsEngine: skills = self._compute_skill_breakdown(skill_usage) activity = self._compute_activity_patterns(sessions) top_sessions = self._compute_top_sessions(sessions) - telemetry = self._compute_telemetry(cutoff) return { "days": days, @@ -177,37 +162,8 @@ class InsightsEngine: "skills": skills, "activity": activity, "top_sessions": top_sessions, - "telemetry": telemetry, } - # ========================================================================= - # Telemetry (observability) — from the tel_* tables (local telemetry) - # ========================================================================= - - def _compute_telemetry(self, cutoff: float) -> Dict[str, Any]: - """Roll up the local telemetry tables for the same window. - - Reuses the engine's existing connection. Fully fail-soft: if the tel_* - tables are empty or absent (telemetry.local disabled, fresh install), this - returns an empty dict and the renderer skips the section. - """ - try: - from agent.telemetry import metrics - except Exception: - return {} - try: - since_ns = int(cutoff * 1e9) - if not metrics.has_data(conn=self._conn): - return {} - return { - "workflows": metrics.workflow_summary(since_ns=since_ns, conn=self._conn), - "model_calls": metrics.model_call_summary(since_ns=since_ns, conn=self._conn), - "tool_calls": metrics.tool_call_summary(conn=self._conn), - "errors": metrics.error_summary(conn=self._conn), - } - except Exception: - return {} - # ========================================================================= # Data gathering (SQL queries) # ========================================================================= @@ -1067,80 +1023,8 @@ class InsightsEngine: lines.append(f" {ts['label']:<20} {ts['value']:<18} ({ts['date']}, {ts['session_id']})") lines.append("") - # Telemetry / observability (local telemetry) — only when data exists - tel = report.get("telemetry") or {} - if tel: - self._append_telemetry_section(lines, tel) - return "\n".join(lines) - def _append_telemetry_section(self, lines: List[str], tel: Dict[str, Any]) -> None: - """Render the observability rollups (workflows, tools, providers, errors).""" - wf = tel.get("workflows", {}) - mc = tel.get("model_calls", {}) - tc = tel.get("tool_calls", {}) - errs = tel.get("errors", {}).get("by_class", {}) - - lines.append(" 📡 Observability (local telemetry)") - lines.append(" " + "─" * 56) - - total_runs = wf.get("total_runs", 0) - if total_runs: - sr = wf.get("success_rate", 0.0) * 100 - p50 = wf.get("duration_ms_p50", 0) - p95 = wf.get("duration_ms_p95", 0) - lines.append( - f" Workflows: {total_runs:,} Success: {sr:.1f}% " - f"Duration p50/p95: {_fmt_ms(p50)} / {_fmt_ms(p95)}" - ) - by_entry = wf.get("by_entrypoint", {}) - if by_entry: - entry_str = ", ".join( - f"{k}: {v}" for k, v in sorted(by_entry.items(), key=lambda x: -x[1]) - ) - lines.append(f" Entrypoints: {entry_str}") - - # Tool reliability - if tc.get("total"): - fail_pct = tc.get("failure_rate", 0.0) * 100 - lines.append( - f" Tool calls: {tc['total']:,} Failure rate: {fail_pct:.1f}%" - ) - tools = tc.get("by_tool", {}) - fails = tc.get("failures_by_tool", {}) - top = sorted(tools.items(), key=lambda x: -x[1])[:6] - if top: - parts = [] - for name, n in top: - f = fails.get(name, 0) - parts.append(f"{name}: {n}" + (f" ({f} failed)" if f else "")) - lines.append(" " + " ".join(parts)) - - # Provider / model mix + cache (real names) - by_provider = mc.get("by_provider", {}) - if by_provider: - prov_str = ", ".join( - f"{k}: {v}" for k, v in sorted(by_provider.items(), key=lambda x: -x[1]) - ) - lines.append(f" Providers: {prov_str}") - by_model = mc.get("by_model", {}) - if by_model: - model_str = ", ".join( - f"{k}: {v}" for k, v in sorted(by_model.items(), key=lambda x: -x[1])[:8] - ) - cache = mc.get("cache_hit_rate", 0.0) * 100 - suffix = f" Cache hit: {cache:.1f}%" if cache else "" - lines.append(f" Models: {model_str}{suffix}") - - # Error classes - if errs: - err_str = ", ".join( - f"{k}: {v}" for k, v in sorted(errs.items(), key=lambda x: -x[1])[:6] - ) - lines.append(f" Errors: {err_str}") - - lines.append("") - def format_gateway(self, report: Dict) -> str: """Format the insights report for gateway/messaging (shorter).""" if report.get("empty"): diff --git a/agent/monitoring/__init__.py b/agent/monitoring/__init__.py new file mode 100644 index 0000000000000..ee45ff01649ef --- /dev/null +++ b/agent/monitoring/__init__.py @@ -0,0 +1,29 @@ +"""Hermes gateway monitoring. + +Service health monitoring plus redacted operational diagnostics for the +gateway daemon, exported over OTLP to an operator-configured endpoint. + +``emitter`` is the in-process event bus: producers (gateway status hooks, +the diagnostic log handler) hand typed events to a fire-and-forget queue, +and subscribers (the OTLP streamers) consume them off the hot path. The +emitter never blocks or raises into gateway code (the hot-path invariant), +and nothing is persisted locally — monitoring is an egress path, not a store. + +Deliberately out of scope here: run/model/tool trajectory capture, usage +analytics, and any content-bearing signal. Those planes are served by the +NeMo Relay integration and its Hermes-owned subscribers. +""" + +from __future__ import annotations + +from . import emitter, events + +emit = emitter.emit +get_emitter = emitter.get_emitter + +__all__ = [ + "emitter", + "events", + "emit", + "get_emitter", +] diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py new file mode 100644 index 0000000000000..f7dee1bdf4fa7 --- /dev/null +++ b/agent/monitoring/emitter.py @@ -0,0 +1,193 @@ +"""Monitoring emitter: fire-and-forget queue + background dispatcher. + +The emitter is the single seam between producers (gateway status hooks, the +diagnostic log handler) and consumers (the OTLP streamers). Its contract is +the hot-path invariant: + + ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, + and MUST NEVER raise into the caller. A monitoring failure is logged + locally and dropped — it can never affect the gateway or a session. + +Mechanism: + * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare + except. On a full queue it drops the *oldest* event and counts the drop. + * A daemon thread drains the queue and fans each batch out to subscribers + (the OTLP metric/span/log streamers). Each subscriber is fail-isolated — + a slow or raising subscriber never affects the hot path or its peers. + +Nothing is persisted here. Monitoring is an egress path, not a local store; +if no subscriber is attached, events simply age out of the ring buffer. +""" + +from __future__ import annotations + +import logging +import queue +import threading +import time +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full +_DRAIN_BATCH = 256 + + +class MonitoringEmitter: + """Owns the queue, the dispatcher thread, and the subscriber list.""" + + def __init__(self, *, enabled: bool = True) -> None: + self._enabled = enabled + self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE) + self._dropped = 0 + self._dispatched = 0 + self._stop = threading.Event() + self._started = False + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + # Live subscribers (the OTLP streamers). Called from the dispatcher + # thread, fully fail-isolated. Each subscriber is callable(batch: list[dict]). + self._subscribers: list = [] + + # ── public API (hot path) ─────────────────────────────────────────────── + def emit(self, event: Any) -> None: + """Enqueue an event. Never blocks, never raises. + + ``event`` may be a dataclass with ``to_dict()`` or a plain dict. + """ + if not self._enabled: + return + try: + payload = event.to_dict() if hasattr(event, "to_dict") else dict(event) + payload.setdefault("ts_ns", time.time_ns()) + self._ensure_started() + try: + self._q.put_nowait(payload) + except queue.Full: + # Drop oldest to make room — bounded memory, newest-wins. + try: + self._q.get_nowait() + self._dropped += 1 + self._q.put_nowait(payload) + except Exception: + self._dropped += 1 + except Exception: # the hot-path invariant: never propagate + logger.debug("monitoring emit failed", exc_info=True) + + # ── lifecycle ─────────────────────────────────────────────────────────── + def _ensure_started(self) -> None: + if self._started: + return + with self._lock: + if self._started: + return + self._thread = threading.Thread( + target=self._run, name="hermes-monitoring-dispatch", daemon=True + ) + self._thread.start() + self._started = True + + def _run(self) -> None: + while not self._stop.is_set(): + try: + first = self._q.get(timeout=0.5) + except queue.Empty: + continue + batch = [first] + while len(batch) < _DRAIN_BATCH: + try: + batch.append(self._q.get_nowait()) + except queue.Empty: + break + self._dispatch(batch) + + def _dispatch(self, batch) -> None: + # Fan-out to subscribers (OTLP streamers) — fully fail-isolated. + for sub in list(self._subscribers): + try: + sub(batch) + except Exception: + logger.debug("monitoring subscriber failed", exc_info=True) + self._dispatched += len(batch) + + def subscribe(self, callback) -> None: + """Register a live batch subscriber (callable(batch: list[dict])).""" + if callback not in self._subscribers: + self._subscribers.append(callback) + + def unsubscribe(self, callback) -> None: + try: + self._subscribers.remove(callback) + except ValueError: + pass + + # ── introspection / shutdown (tests, CLI) ─────────────────────────────── + def flush(self, timeout: float = 2.0) -> None: + """Block until the queue drains (test/CLI helper, NOT the hot path).""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._q.empty(): + # give the dispatcher a tick to finish the in-flight batch + time.sleep(0.05) + if self._q.empty(): + return + time.sleep(0.02) + + def stats(self) -> Dict[str, int]: + return { + "queued": self._q.qsize(), + "dispatched": self._dispatched, + "dropped": self._dropped, + "subscribers": len(self._subscribers), + } + + def close(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._started = False + + +# ── process-wide singleton ────────────────────────────────────────────────── +_EMITTER: Optional[MonitoringEmitter] = None +_EMITTER_LOCK = threading.Lock() + + +def get_emitter() -> MonitoringEmitter: + """Return the process-wide monitoring emitter.""" + global _EMITTER + if _EMITTER is not None: + return _EMITTER + with _EMITTER_LOCK: + if _EMITTER is None: + _EMITTER = MonitoringEmitter() + return _EMITTER + + +def emit(event: Any) -> None: + """Module-level convenience: emit via the singleton.""" + get_emitter().emit(event) + + +def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None: + """Swap the singleton (tests only).""" + global _EMITTER + with _EMITTER_LOCK: + if _EMITTER is not None and emitter is not _EMITTER: + try: + _EMITTER.close() + except Exception: + pass + _EMITTER = emitter + + +# Back-compat alias for the salvaged class name used in emozilla's tests. +TelemetryEmitter = MonitoringEmitter + +__all__ = [ + "MonitoringEmitter", + "TelemetryEmitter", + "get_emitter", + "emit", + "reset_emitter_for_tests", +] diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py new file mode 100644 index 0000000000000..2151932b730e6 --- /dev/null +++ b/agent/monitoring/events.py @@ -0,0 +1,69 @@ +"""Typed gateway monitoring events. + +Content-free service-health and redacted diagnostic events for the gateway +daemon. These are the only event shapes the monitoring plane emits: no +prompts, messages, tool args/results, session history, or usage analytics. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field, asdict +from typing import Any, Dict, Optional + + +def _now_ns() -> int: + return time.time_ns() + + +@dataclass(slots=True) +class GatewayHealthEvent: + """Content-free gateway health snapshot or lifecycle event.""" + + name: str + gateway_state: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + exit_reason: Optional[str] = None + restart_requested: Optional[bool] = None + active_agents: int = 0 + gateway_busy: bool = False + gateway_drainable: bool = False + platform_count: int = 0 + fatal_platform_count: int = 0 + profile: Optional[str] = None + install_id: Optional[str] = None + version: Optional[str] = None + supervision_mode: Optional[str] = None + pid: Optional[int] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_health", **asdict(self)} + + +@dataclass(slots=True) +class GatewayDiagnosticEvent: + """Redacted gateway diagnostic event for operator-owned observability.""" + + name: str + subsystem: str + error_class: str = "unknown" + error_code: Optional[str] = None + redacted_message: Optional[str] = None + platform: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + profile: Optional[str] = None + version: Optional[str] = None + severity: str = "warning" + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_diagnostic", **asdict(self)} + + +__all__ = [ + "GatewayHealthEvent", + "GatewayDiagnosticEvent", +] diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py new file mode 100644 index 0000000000000..3404392c63854 --- /dev/null +++ b/agent/monitoring/gateway_health.py @@ -0,0 +1,388 @@ +"""Gateway health and diagnostics signal producer. + +This module keeps the plane narrow: service health monitoring plus +redacted operational diagnostics. It reuses the existing gateway runtime-status +contract and emits content-free metrics/events. No prompts, messages, tool args, +session history, audit records, or product analytics belong here. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent + + +@dataclass(frozen=True, slots=True) +class GatewayMetric: + name: str + value: int | float + attributes: Dict[str, str] + + +@dataclass(frozen=True, slots=True) +class GatewayHealthSnapshot: + metrics: List[GatewayMetric] + events: List[GatewayHealthEvent | GatewayDiagnosticEvent] + + +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") +_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE) +_TOKEN_RE = re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b") +_SECRET_LITERAL_RE = re.compile(r"\*{3,}") +_PHONE_RE = re.compile(r"(? bool: + return name == "gateway" or name.startswith("gateway.") + + +def redact_gateway_message(message: Any) -> str: + """Redact gateway diagnostic free text for customer-owned export.""" + text = str(message or "") + try: + from agent.monitoring.redaction import redact_for_export + redacted = redact_for_export(text, content_mode="pii") or "" + except Exception: + redacted = "[redaction-unavailable]" + redacted = _BEARER_RE.sub("[redacted]", redacted) + redacted = _TOKEN_RE.sub("[redacted]", redacted) + redacted = _SECRET_LITERAL_RE.sub("[redacted]", redacted) + redacted = re.sub(r"\bBearer\s+\[[^\]]+\]", "[redacted]", redacted, flags=re.IGNORECASE) + redacted = _EMAIL_RE.sub("[email]", redacted) + redacted = _PHONE_RE.sub("[phone]", redacted) + return redacted[:500] + + +def classify_gateway_error(raw: Any) -> str: + s = str(raw or "").lower() + if any(k in s for k in ("auth", "token", "unauthorized", "forbidden", "401", "403")): + return "auth_failed" + if "rate" in s and "limit" in s: + return "rate_limited" + if "timeout" in s or "timed out" in s: + return "timeout" + if any(k in s for k in ("network", "connection", "dns", "socket")): + return "network_error" + if any(k in s for k in ("config", "missing", "invalid")): + return "invalid_config" + if "startup" in s: + return "startup_failed" + if "fatal" in s: + return "platform_fatal" + return "unknown" + + +def subsystem_for_logger(logger_name: str) -> str: + if logger_name.startswith("gateway.platforms."): + parts = logger_name.split(".") + if len(parts) >= 3 and parts[2]: + return f"platform.{parts[2]}" + if logger_name.startswith("gateway.platforms"): + return "platform" + if logger_name.startswith("gateway"): + return "gateway" + return "gateway" + + +def platform_for_subsystem(subsystem: str) -> Optional[str]: + if subsystem.startswith("platform."): + return subsystem.split(".", 1)[1] or None + return None + + +def _parse_active_agents(raw: Any) -> int: + try: + from gateway.status import parse_active_agents + return parse_active_agents(raw) + except Exception: + try: + return max(0, int(raw)) + except (TypeError, ValueError): + return 0 + + +def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: + try: + from gateway.status import derive_gateway_busy + return derive_gateway_busy( + gateway_running=gateway_running, + gateway_state=gateway_state, + active_agents=active_agents, + ) + except Exception: + return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0) + + +def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool: + try: + from gateway.status import derive_gateway_drainable + return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state) + except Exception: + return bool(gateway_running and gateway_state == "running") + + +def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]: + return { + "hermes.profile": str(profile), + "service.instance.id": str(install_id), + "service.version": str(version), + "hermes.supervision_mode": str(supervision_mode), + } + + +def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric: + out = dict(attrs) + for key, val in extra.items(): + if val is not None: + out[key] = str(val) + return GatewayMetric(name=name, value=value, attributes=out) + + +def build_gateway_health_snapshot( + runtime: Optional[dict[str, Any]], + *, + gateway_running: bool, + profile: str, + install_id: str, + version: str, + supervision_mode: str = "unknown", +) -> GatewayHealthSnapshot: + """Convert gateway_state.json-compatible runtime state into P0 signals.""" + runtime = runtime or {} + gateway_state = runtime.get("gateway_state") + active_agents = _parse_active_agents(runtime.get("active_agents", 0)) + busy = _derive_busy(gateway_running, gateway_state, active_agents) + drainable = _derive_drainable(gateway_running, gateway_state) + platforms = runtime.get("platforms") if isinstance(runtime.get("platforms"), dict) else {} + base = _base_attrs(profile=profile, install_id=install_id, version=version, supervision_mode=supervision_mode) + + metrics: list[GatewayMetric] = [ + _metric("hermes.gateway.up", 1 if gateway_running else 0, base), + _metric("hermes.gateway.active_agents", active_agents, base), + _metric("hermes.gateway.busy", 1 if busy else 0, base), + _metric("hermes.gateway.drainable", 1 if drainable else 0, base), + _metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base), + ] + if gateway_state: + metrics.append(_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": str(gateway_state)})) + + fatal_count = 0 + events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + for platform, pdata in platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + state = str(pdata.get("state") or "unknown").lower() + raw_error = pdata.get("error_code") or pdata.get("error_message") + error_code = classify_gateway_error(raw_error) + is_up = state in _RUNNING_PLATFORM_STATES + is_degraded = state in _FATAL_PLATFORM_STATES + if is_degraded: + fatal_count += 1 + metrics.append(_metric( + "hermes.platform.up", + 1 if is_up else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state}, + )) + metrics.append(_metric( + "hermes.platform.degraded", + 1 if is_degraded else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state, "hermes.error_code": error_code}, + )) + if is_degraded: + events.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=classify_gateway_error(error_code or pdata.get("error_message")), + redacted_message=redact_gateway_message(pdata.get("error_message")), + profile=profile, + version=version, + severity="error" if state == "fatal" else "warning", + )) + + events.insert(0, GatewayHealthEvent( + name="gateway.health_snapshot", + gateway_state=str(gateway_state) if gateway_state is not None else None, + active_agents=active_agents, + gateway_busy=busy, + gateway_drainable=drainable, + platform_count=len(platforms), + fatal_platform_count=fatal_count, + profile=profile, + install_id=install_id, + version=version, + supervision_mode=supervision_mode, + pid=_coerce_pid(runtime.get("pid")), + )) + return GatewayHealthSnapshot(metrics=metrics, events=events) + + +def _safe_profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _safe_version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None: + """Emit immediate content-free gateway events for runtime status changes. + + Called by gateway.status.write_runtime_status after persisting the new status. + Fully fail-open: failures never affect gateway status writes. + """ + try: + from agent.monitoring import emitter + out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + profile = _safe_profile() + version = _safe_version() + old_gateway_state = str((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None + new_gateway_state = str(current.get("gateway_state")) if current.get("gateway_state") is not None else None + if old_gateway_state != new_gateway_state and new_gateway_state: + out.append(GatewayHealthEvent( + name="gateway.lifecycle", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=current.get("exit_reason"), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + if new_gateway_state == "startup_failed": + out.append(GatewayDiagnosticEvent( + name="gateway.startup_failed", + subsystem="gateway", + error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + redacted_message=redact_gateway_message(current.get("exit_reason") or "startup failed"), + profile=profile, + version=version, + severity="error", + )) + if new_gateway_state == "stopped": + out.append(GatewayHealthEvent( + name="gateway.exit", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=current.get("exit_reason"), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + + old_platforms_raw = (previous or {}).get("platforms") + new_platforms_raw = current.get("platforms") + old_platforms = old_platforms_raw if isinstance(old_platforms_raw, dict) else {} + new_platforms = new_platforms_raw if isinstance(new_platforms_raw, dict) else {} + for platform, pdata in new_platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + prev_raw = old_platforms.get(platform, {}) + prev = prev_raw if isinstance(prev_raw, dict) else {} + old_state = str(prev.get("state")) if prev.get("state") is not None else None + new_state = str(pdata.get("state")) if pdata.get("state") is not None else None + if old_state == new_state or not new_state: + continue + error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) + severity = "error" if new_state.lower() in {"fatal", "failed", "error"} else "warning" + out.append(GatewayDiagnosticEvent( + name="platform.state_change", + subsystem=f"platform.{platform}", + platform=str(platform), + old_state=old_state, + new_state=new_state, + error_code=error_code, + error_class=error_code, + redacted_message=redact_gateway_message(pdata.get("error_message")), + profile=profile, + version=version, + severity=severity, + )) + if new_state.lower() in _FATAL_PLATFORM_STATES: + out.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=error_code, + redacted_message=redact_gateway_message(pdata.get("error_message")), + profile=profile, + version=version, + severity=severity, + )) + for ev in out: + emitter.emit(ev) + except Exception: + logging.getLogger(__name__).debug("gateway runtime status transition emit failed", exc_info=True) + + +def _coerce_pid(raw: Any) -> Optional[int]: + try: + pid = int(raw) + except (TypeError, ValueError): + return None + return pid if pid > 0 else None + + +class GatewayDiagnosticLogHandler(logging.Handler): + """Allowlisted warning/error bridge for gateway-owned diagnostics.""" + + def __init__(self, *, profile: str = "default", version: str = "unknown") -> None: + super().__init__(level=logging.WARNING) + self.profile = profile + self.version = version + + def emit(self, record: logging.LogRecord) -> None: + try: + if record.levelno < logging.WARNING: + return + if not _allowed_logger(record.name): + return + subsystem = subsystem_for_logger(record.name) + message = record.getMessage() + event = GatewayDiagnosticEvent( + name=f"gateway.log.{record.levelname.lower()}", + subsystem=subsystem, + platform=platform_for_subsystem(subsystem), + error_class=classify_gateway_error(message), + redacted_message=redact_gateway_message(message), + profile=self.profile, + version=self.version, + severity=record.levelname.lower(), + ) + from agent.monitoring import emitter + emitter.get_emitter().emit(event) + except Exception: + logging.getLogger(__name__).debug("gateway diagnostic emit failed", exc_info=True) + + +__all__ = [ + "GatewayMetric", + "GatewayHealthSnapshot", + "GatewayDiagnosticLogHandler", + "build_gateway_health_snapshot", + "classify_gateway_error", + "redact_gateway_message", +] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py new file mode 100644 index 0000000000000..969fa97063534 --- /dev/null +++ b/agent/monitoring/gateway_health_export.py @@ -0,0 +1,405 @@ +"""Gateway Health & Diagnostics OTLP export runtime. + +This exporter emits operator-owned gateway service-health metrics plus +narrow redacted diagnostic events. It is deliberately in-process and fail-open so +it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell +without a sidecar/watchdog dependency. +""" + +from __future__ import annotations + +import logging +import os +import threading +from dataclasses import dataclass +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + + +@dataclass(slots=True) +class GatewayHealthExportRuntime: + enabled: bool + reason: str = "disabled" + streamer: Any = None + metric_provider: Any = None + log_handler: Any = None + log_streamer: Any = None + thread: Optional[threading.Thread] = None + stop_event: Optional[threading.Event] = None + + def shutdown(self) -> None: + if self.stop_event is not None: + self.stop_event.set() + if self.thread is not None: + self.thread.join(timeout=2.0) + if self.log_handler is not None: + try: + logging.getLogger().removeHandler(self.log_handler) + except Exception: + pass + if self.streamer is not None: + try: + self.streamer.shutdown() + except Exception: + pass + if self.log_streamer is not None: + try: + self.log_streamer.shutdown() + except Exception: + pass + if self.metric_provider is not None: + try: + self.metric_provider.shutdown() + except Exception: + pass + + +def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + return mon.get("gateway_health_export") or {} + + +def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} + return export.get("otlp") or {} + + +def _enabled(config: Dict[str, Any]) -> bool: + gh = _gateway_health_config(config) + otlp = _otlp_config(config) + return bool(gh.get("enabled") and otlp.get("enabled") and otlp.get("endpoint")) + + +def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]: + if auto_install: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("export.otlp", prompt=prompt) + except Exception: + pass + try: + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.metrics import Observation + from opentelemetry.trace import INVALID_SPAN_ID, INVALID_TRACE_ID, TraceFlags + from opentelemetry._logs import LogRecord + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + return { + "OTLPLogExporter": OTLPLogExporter, + "OTLPMetricExporter": OTLPMetricExporter, + "Observation": Observation, + "LogRecord": LogRecord, + "LoggerProvider": LoggerProvider, + "INVALID_SPAN_ID": INVALID_SPAN_ID, + "INVALID_TRACE_ID": INVALID_TRACE_ID, + "TraceFlags": TraceFlags, + "SeverityNumber": SeverityNumber, + "BatchLogRecordProcessor": BatchLogRecordProcessor, + "MeterProvider": MeterProvider, + "PeriodicExportingMetricReader": PeriodicExportingMetricReader, + "Resource": Resource, + } + except Exception as exc: + raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc + + +def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: + resolved: Dict[str, str] = {} + for header_name, env_name in (headers_env or {}).items(): + val = os.environ.get(str(env_name)) + if val: + resolved[str(header_name)] = val + return resolved + + +def _metric_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/metrics" + return endpoint + + +def _logs_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/logs" + if endpoint.endswith("/v1/metrics"): + return endpoint[: -len("/v1/metrics")] + "/v1/logs" + return endpoint + + +def _version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def _profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _install_id(config: Dict[str, Any]) -> str: + try: + from agent.monitoring.policy import ensure_install_id + return str(ensure_install_id(config)) + except Exception: + return "unknown" + + +def _supervision_mode() -> str: + if os.environ.get("INVOCATION_ID"): + return "systemd" + if os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION"): + return "s6" + if os.environ.get("container") or os.path.exists("/.dockerenv"): + return "container" + if os.environ.get("LAUNCHD_SOCKET"): + return "launchd" + return "manual" + + +def _read_runtime_snapshot(config: Dict[str, Any]): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + try: + from gateway.status import read_runtime_status + runtime = read_runtime_status() or {} + except Exception: + runtime = {} + return build_gateway_health_snapshot( + runtime, + gateway_running=True, + profile=_profile(), + install_id=_install_id(config), + version=_version(), + supervision_mode=_supervision_mode(), + ) + + +def _emit_snapshot_events(config: Dict[str, Any]) -> None: + gh = _gateway_health_config(config) + if not gh.get("diagnostic_events_enabled", True): + return + try: + from agent.monitoring import emitter + snapshot = _read_runtime_snapshot(config) + for event in snapshot.events: + emitter.emit(event) + except Exception: + logger.debug("gateway health snapshot emit failed", exc_info=True) + + +def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("metrics_enabled", True): + return None + otlp = _otlp_config(config) + endpoint = _metric_endpoint(str(otlp.get("endpoint"))) + headers = _resolve_headers(otlp.get("headers_env")) + exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None) + interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000 + reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms) + resource_attrs = dict((gh.get("resource_attributes") or {})) + resource_attrs.setdefault("service.name", "hermes-gateway") + resource_attrs.setdefault("telemetry.scope", "gateway_health") + provider = sdk["MeterProvider"]( + metric_readers=[reader], + resource=sdk["Resource"].create(resource_attrs), + ) + meter = provider.get_meter("hermes.gateway.health") + Observation = sdk["Observation"] + + metric_names = [ + "hermes.gateway.up", + "hermes.gateway.state", + "hermes.gateway.active_agents", + "hermes.gateway.busy", + "hermes.gateway.drainable", + "hermes.gateway.restart_requested", + "hermes.platform.up", + "hermes.platform.degraded", + ] + + def callback(name: str): + def _cb(_options=None): + try: + snapshot = _read_runtime_snapshot(config) + return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name] + except Exception: + logger.debug("gateway metric callback failed", exc_info=True) + return [] + return _cb + + for metric_name in metric_names: + meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)]) + return provider + + +def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any: + SeverityNumber = sdk["SeverityNumber"] + sev = str(severity or "warning").lower() + if sev in {"critical", "fatal"}: + return SeverityNumber.FATAL + if sev == "error": + return SeverityNumber.ERROR + if sev in {"info", "information"}: + return SeverityNumber.INFO + if sev == "debug": + return SeverityNumber.DEBUG + return SeverityNumber.WARN + + +class GatewayDiagnosticLogStreamer: + """Emitter subscriber that sends gateway diagnostic events as OTLP logs.""" + + def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]): + otlp = _otlp_config(config) + gh = _gateway_health_config(config) + headers = _resolve_headers(otlp.get("headers_env")) + endpoint = _logs_endpoint(str(otlp.get("endpoint"))) + resource_attrs = dict((gh.get("resource_attributes") or {})) + resource_attrs.setdefault("service.name", "hermes-gateway") + resource_attrs.setdefault("telemetry.scope", "gateway_diagnostics") + self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs)) + self._processor = sdk["BatchLogRecordProcessor"]( + sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None) + ) + self._provider.add_log_record_processor(self._processor) + self._logger = self._provider.get_logger("hermes.gateway.diagnostics") + self._LogRecord = sdk["LogRecord"] + self._sdk = sdk + self.exported = 0 + + def __call__(self, batch: list[Dict[str, Any]]) -> None: + for ev in batch: + if ev.get("event") != "gateway_diagnostic": + continue + attrs = { + f"hermes.{key}": val + for key, val in ev.items() + if key not in {"event", "redacted_message", "ts_ns"} and val is not None + } + body = ev.get("redacted_message") or ev.get("name") or "gateway diagnostic" + record = self._LogRecord( + timestamp=ev.get("ts_ns"), + trace_id=self._sdk["INVALID_TRACE_ID"], + span_id=self._sdk["INVALID_SPAN_ID"], + trace_flags=self._sdk["TraceFlags"].DEFAULT, + severity_text=str(ev.get("severity") or "warning").upper(), + severity_number=_severity_number(self._sdk, ev.get("severity")), + body=str(body), + attributes=attrs, + ) + self._logger.emit(record) + self.exported += 1 + + def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass + try: + self._processor.force_flush() + self._provider.shutdown() + except Exception: + pass + + +def _start_diagnostic_log_streamer(config: Dict[str, Any], sdk: Dict[str, Any]) -> GatewayDiagnosticLogStreamer: + from agent.monitoring.emitter import get_emitter + streamer = GatewayDiagnosticLogStreamer(config, sdk) + get_emitter().subscribe(streamer) + return streamer + + +def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread: + interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5))) + + def _run() -> None: + while not stop_event.wait(interval): + _emit_snapshot_events(config) + + thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True) + thread.start() + return thread + + +def _attach_log_handler(config: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("warning_error_events_enabled", True): + return None + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version()) + root = logging.getLogger() + if handler not in root.handlers: + root.addHandler(handler) + return handler + + +def _gateway_health_event(ev: Dict[str, Any]) -> bool: + return ev.get("event") == "gateway_health" + + +def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime: + """Start P0 gateway health export if configured. Never raises.""" + if not _enabled(config): + return GatewayHealthExportRuntime(enabled=False, reason="disabled") + gh = _gateway_health_config(config) + runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled") + sdk: Optional[Dict[str, Any]] = None + + if gh.get("metrics_enabled", True) or gh.get("diagnostic_events_enabled", True): + try: + sdk = _require_metrics_sdk(prompt=False) + except Exception: + logger.warning( + "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; " + "install 'hermes-agent[otlp]'", + exc_info=True, + ) + return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable") + + if gh.get("metrics_enabled", True) and sdk is not None: + try: + runtime.metric_provider = _start_metric_provider(config, sdk) + except Exception: + logger.warning("gateway health OTLP metrics failed to start", exc_info=True) + runtime.shutdown() + return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed") + + if gh.get("diagnostic_events_enabled", True) and sdk is not None: + try: + from agent.monitoring import otlp_exporter + runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event) + runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk) + except Exception: + logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True) + + try: + runtime.log_handler = _attach_log_handler(config) + except Exception: + logger.debug("gateway diagnostic log handler failed to attach", exc_info=True) + try: + _emit_snapshot_events(config) + runtime.stop_event = threading.Event() + runtime.thread = _start_snapshot_thread(config, runtime.stop_event) + except Exception: + logger.debug("gateway health snapshot thread failed to start", exc_info=True) + return runtime + + +__all__ = [ + "GatewayHealthExportRuntime", + "start_gateway_health_export", +] diff --git a/agent/telemetry/otlp_exporter.py b/agent/monitoring/otlp_exporter.py similarity index 60% rename from agent/telemetry/otlp_exporter.py rename to agent/monitoring/otlp_exporter.py index 3f18026a5dfb1..01eefa53506ae 100644 --- a/agent/telemetry/otlp_exporter.py +++ b/agent/monitoring/otlp_exporter.py @@ -1,37 +1,31 @@ -"""Export telemetry to an OpenTelemetry Collector over OTLP/HTTP. +"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP. -Maps the local tel_* events to OTel spans and sends them to the endpoint configured -under ``telemetry.export.otlp``. Lets an operator stream Hermes telemetry into their -own observability stack. +Maps gateway monitoring events to OTel spans and sends them to the endpoint +configured under ``monitoring.export.otlp``. Lets an operator stream Hermes +gateway health into their own observability stack (OTEL Collector, DataDog, +and similar). Notes: - * The destination is operator-configured; this module only sends to that endpoint. - It does not import or interact with any aggregate-metrics path. - * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an optional - extra (``pip install hermes-agent[otlp]``), imported lazily so the dependency is - only required when OTLP export is actually used. - * ``headers_env`` maps a header name to an environment variable name; values are read - from the environment at export time and never logged or stored. - * The continuous subscriber runs in the emitter's writer thread after durable writes - and is fail-isolated, so an export error cannot affect a run. + * The destination is operator-configured; this module only sends to that + endpoint. No default destination ships. + * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an + optional extra (``pip install hermes-agent[otlp]``), imported lazily so the + dependency is only required when OTLP export is actually used. + * ``headers_env`` maps a header name to an environment variable name; values + are read from the environment at export time and never logged or stored. + * The continuous subscriber runs in the emitter's dispatcher thread and is + fail-isolated, so an export error cannot affect the gateway. -Each event is exported as a span carrying its recorded attributes (provider, model, -tokens, duration, etc.). The timing/parent linkage captured in tel_spans -(trace_id/span_id/parent_span_id/start_ns/end_ns) is not yet reconstructed into OTel -SpanContexts here, so spans currently arrive as independent records rather than a -connected trace tree; building the connected-trace projection is tracked separately. - -Spans carry structural telemetry by default. Message content is included only when -trajectories is enabled, and always passes through the export redaction pipeline. +Only monitoring events (gateway_health / gateway_diagnostic) exist on this +plane; the ``event_filter`` seam is kept so future planes sharing the emitter +cannot silently ride along on this exporter. """ from __future__ import annotations import logging import os -import sqlite3 -from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional logger = logging.getLogger(__name__) @@ -89,9 +83,9 @@ def _require_sdk(*, auto_install: bool = True, prompt: bool = True): def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env. - The config stores environment variable names, not secret values; values are read - from the environment here. Missing variables are skipped (and noted at debug level - without the value). + The config stores environment variable names, not secret values; values are + read from the environment here. Missing variables are skipped (and noted at + debug level without the value). """ resolved: Dict[str, str] = {} for header_name, env_name in (headers_env or {}).items(): @@ -105,8 +99,8 @@ def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: - tel = (config or {}).get("telemetry") or {} - export = tel.get("export") or {} + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} return export.get("otlp") or {} @@ -116,7 +110,7 @@ def build_exporter(config: Dict[str, Any]): otlp = _otlp_config(config) endpoint = otlp.get("endpoint") if not endpoint: - raise ValueError("telemetry.export.otlp.endpoint is not set") + raise ValueError("monitoring.export.otlp.endpoint is not set") headers = _resolve_headers(otlp.get("headers_env")) return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None) @@ -124,8 +118,8 @@ def build_exporter(config: Dict[str, Any]): def _make_provider(config: Dict[str, Any]): sdk = _require_sdk() resource = sdk["Resource"].create({ - "service.name": "hermes-agent", - "telemetry.scope": "local", # never aggregate metrics + "service.name": "hermes-gateway", + "telemetry.scope": "gateway_monitoring", }) provider = sdk["TracerProvider"](resource=resource) processor = sdk["BatchSpanProcessor"](build_exporter(config)) @@ -133,21 +127,20 @@ def _make_provider(config: Dict[str, Any]): return provider, processor -# ── event -> span attribute mapping (real values) ─────────────────────────── +# ── event -> span attribute mapping ────────────────────────────────────────── def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: - """Span attributes for an event — the real recorded values (local telemetry).""" + """Span attributes for a monitoring event (content-free by construction).""" kind = ev.get("event") attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} keep_by_kind = { - "run": ("entrypoint", "platform", "end_reason", - "model_call_count", "tool_call_count", "error_count"), - "span": ("trace_id", "run_id", "parent_span_id", "name", "kind", - "start_ns", "end_ns", "status"), - "model_call": ("provider", "model", "base_url", - "input_tokens", "output_tokens", "cache_read_tokens", - "cache_write_tokens", "reasoning_tokens", "latency_ms"), - "tool_call": ("tool_name", "duration_ms", "result_class"), - "error": ("error_class", "subsystem", "recovery"), + "gateway_health": ("name", "gateway_state", "old_state", "new_state", + "exit_reason", "restart_requested", "active_agents", + "gateway_busy", "gateway_drainable", "platform_count", + "fatal_platform_count", "profile", "install_id", "version", + "supervision_mode", "pid"), + "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", + "redacted_message", "platform", "old_state", "new_state", + "profile", "version", "severity"), } for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] v = ev.get(col) @@ -158,7 +151,7 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: def export_batch(provider, batch: List[Dict[str, Any]]) -> int: """Map a batch of events to OTel spans. Returns spans created.""" - tracer = provider.get_tracer("hermes.telemetry") + tracer = provider.get_tracer("hermes.monitoring") n = 0 for ev in batch: try: @@ -171,53 +164,6 @@ def export_batch(provider, batch: List[Dict[str, Any]]) -> int: return n -# ── one-shot drain (export current local rows) ────────────────────────────── -def export_once( - config: Dict[str, Any], - *, - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, -) -> int: - """Drain the local tel_* tables to the configured OTLP endpoint once.""" - provider, processor = _make_provider(config) - try: - rows = _read_events(db_path, since_ns) - total = export_batch(provider, rows) - processor.force_flush() - return total - finally: - try: - provider.shutdown() - except Exception: - pass - - -def _read_events(db_path: Optional[Path], since_ns: Optional[int]) -> List[Dict[str, Any]]: - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - out: List[Dict[str, Any]] = [] - try: - table_event = { - "tel_runs": "run", "tel_spans": "span", - "tel_model_calls": "model_call", - "tel_tool_calls": "tool_call", "tel_error_events": "error", - } - for table, evkind in table_event.items(): - where = "" - if table == "tel_runs" and since_ns: - where = f" WHERE start_ns >= {int(since_ns)}" - for r in c.execute(f"SELECT * FROM {table}{where}").fetchall(): - d = dict(r) - d["event"] = evkind - out.append(d) - finally: - c.close() - return out - - # ── continuous streaming subscriber ───────────────────────────────────────── class OTLPStreamer: """A live subscriber that pushes each emitter batch to OTLP as it lands. @@ -225,14 +171,29 @@ class OTLPStreamer: Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter. """ - def __init__(self, config: Dict[str, Any]): + def __init__( + self, + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, + ): self._provider, self._processor = _make_provider(config) + self._event_filter = event_filter self.exported = 0 def __call__(self, batch: List[Dict[str, Any]]) -> None: + if self._event_filter is not None: + batch = [ev for ev in batch if self._event_filter(ev)] + if not batch: + return self.exported += export_batch(self._provider, batch) def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass try: self._processor.force_flush() self._provider.shutdown() @@ -255,9 +216,16 @@ def is_enabled(config: Dict[str, Any]) -> bool: return bool(otlp.get("enabled") and otlp.get("endpoint")) -def start_streaming(config: Dict[str, Any]) -> Optional[OTLPStreamer]: +def start_streaming( + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, +) -> Optional[OTLPStreamer]: """If OTLP is enabled, attach a streamer to the singleton emitter. + ``event_filter`` scopes the exporter to its plane, e.g. gateway-health + export, so enabling one plane cannot silently export unrelated events. + Non-interactive context (startup): attempts a lazy install with prompt=False so a configured-but-missing SDK is installed once (gated by security.allow_lazy_installs), then streams. If it still can't load, logs and @@ -268,11 +236,11 @@ def start_streaming(config: Dict[str, Any]) -> Optional[OTLPStreamer]: try: _require_sdk(prompt=False) except OTLPUnavailable: - logger.warning("telemetry.export.otlp.enabled but the OTel SDK could not " + logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not " "be installed/imported; install 'hermes-agent[otlp]'") return None - from agent.telemetry.emitter import get_emitter - streamer = OTLPStreamer(config) + from agent.monitoring.emitter import get_emitter + streamer = OTLPStreamer(config, event_filter=event_filter) get_emitter().subscribe(streamer) return streamer @@ -281,7 +249,6 @@ __all__ = [ "OTLPUnavailable", "OTLPStreamer", "build_exporter", - "export_once", "export_batch", "is_available", "is_enabled", diff --git a/agent/monitoring/policy.py b/agent/monitoring/policy.py new file mode 100644 index 0000000000000..d07b90ea2c105 --- /dev/null +++ b/agent/monitoring/policy.py @@ -0,0 +1,40 @@ +"""Install identity for gateway monitoring. + +The install id is a stable, resettable pseudonymous identifier attached to +exported health signals so an operator can tell instances apart in their +collector. It carries no account identity and can be rotated by clearing +``monitoring.install_id`` in config. +""" + +from __future__ import annotations + +import uuid +from typing import Any, Dict + + +def _monitoring_cfg(config: Dict[str, Any]) -> Dict[str, Any]: + for key in ("monitoring", "telemetry"): # accept legacy telemetry.* keys + cfg = config.get(key) if isinstance(config, dict) else None + if isinstance(cfg, dict) and cfg.get("install_id"): + return cfg + cfg = config.get("monitoring") if isinstance(config, dict) else None + return cfg if isinstance(cfg, dict) else {} + + +def ensure_install_id(config: Dict[str, Any]) -> str: + """Return a stable install id, minting one if the config slot is empty. + + Does not persist — the caller writes the returned value back to + config.yaml. Clearing ``monitoring.install_id`` (e.g. with + ``hermes config set monitoring.install_id ""``) mints anew on next call. + """ + cfg = _monitoring_cfg(config) + existing = cfg.get("install_id") + if isinstance(existing, str) and existing.strip(): + return existing + return str(uuid.uuid4()) + + +__all__ = [ + "ensure_install_id", +] diff --git a/agent/monitoring/redaction.py b/agent/monitoring/redaction.py new file mode 100644 index 0000000000000..7ea5524cccd06 --- /dev/null +++ b/agent/monitoring/redaction.py @@ -0,0 +1,77 @@ +"""Redaction applied to monitoring data before egress. + +Secrets are always redacted, on every export path; no setting disables this. +Wraps ``agent/redact.py::redact_sensitive_text(force=True)`` and fails CLOSED: +if the redactor cannot run, the raw string is never emitted. + +``redact_for_export(text, content_mode="pii")`` additionally scrubs e-mail +addresses, phone numbers, and UUID-shaped identifiers — the gateway +diagnostics path always uses this mode, so log-derived messages leave the +process with secrets AND PII already removed. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +# Content-redaction strengths for any content that IS exported. +CONTENT_NONE = "none" # drop content entirely (structural telemetry only) +CONTENT_PII = "pii" # codec-aware PII redaction on exported content +CONTENT_MODES = {CONTENT_NONE, CONTENT_PII} + +# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ── +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") +# E.164-ish and common separators; conservative to avoid nuking code/IDs. +_PHONE_RE = re.compile( + r"(? Optional[str]: + """Always-on secret redaction. force=True so user config can't disable it.""" + if text is None: + return None + try: + from agent.redact import redact_sensitive_text + return redact_sensitive_text(str(text), force=True) + except Exception: + # Fail CLOSED: if the redactor can't run, do not emit the raw string. + return "[redaction-unavailable]" + + +def _pii_redact(text: str) -> str: + text = _EMAIL_RE.sub("[email]", text) + text = _UUID_RE.sub("[id]", text) + text = _PHONE_RE.sub("[phone]", text) + return text + + +def redact_for_export( + text: Optional[str], + *, + content_mode: str = CONTENT_NONE, +) -> Optional[str]: + """Redact a single content string for export. + + Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'. + Callers gate *whether content is exported at all* via telemetry.trajectories + (see ``content_export_enabled``); this function only scrubs content that the + caller has already decided to export. + """ + redacted = _secret_redact(text) + if redacted is None: + return None + if content_mode == CONTENT_PII: + redacted = _pii_redact(redacted) + return redacted + + +__all__ = [ + "CONTENT_NONE", + "CONTENT_PII", + "CONTENT_MODES", + "redact_for_export", +] diff --git a/agent/telemetry/__init__.py b/agent/telemetry/__init__.py deleted file mode 100644 index d824d5e6f4b06..0000000000000 --- a/agent/telemetry/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Hermes telemetry & observability. - -Local-first observability, on by default. The ``telemetry`` plugin registers Hermes -lifecycle hooks and hands typed events to the fire-and-forget ``emitter`` (queue -> -background writer -> JSONL + state.db ``tel_*`` index). The emitter never blocks or -raises into a model/tool call (the hot-path invariant). - -Events record the observed model ids, provider names, and tool names. ``metrics`` -derives rollups for /usage and /insights; ``rollup`` builds the per-run summaries shown -by ``hermes telemetry preview``. ``redaction`` + ``exporter_bulk`` + ``otlp_exporter`` -handle export to an operator-chosen destination. ``policy`` holds the consent -constants and the aggregate upload gate (no uploader ships). -""" - -from __future__ import annotations - -from . import emitter, events, metrics, policy, spans - -emit = emitter.emit -get_emitter = emitter.get_emitter - -__all__ = [ - "emitter", - "events", - "metrics", - "policy", - "spans", - "emit", - "get_emitter", -] diff --git a/agent/telemetry/emitter.py b/agent/telemetry/emitter.py deleted file mode 100644 index 5f804c1dacbd0..0000000000000 --- a/agent/telemetry/emitter.py +++ /dev/null @@ -1,318 +0,0 @@ -"""Local telemetry emitter: fire-and-forget queue + background writer. - -The emitter is the single seam between instrumentation (the telemetry plugin's hook -callbacks) and durable storage. Its contract is the hot-path invariant: - - ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, and - MUST NEVER raise into the caller. A telemetry failure is logged locally and - dropped — it can never affect a model call, a tool call, or a session. - -Mechanism: - * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare except. - On a full queue it drops the *oldest* event and counts the drop. - * A daemon thread drains the queue and writes each event to two places: - 1. the append-only JSONL log (source of truth) - 2. the ``tel_*`` SQLite tables in state.db (rebuildable index) - * The writer uses its own sqlite connection to state.db, separate from SessionDB, - so telemetry writes never contend with or corrupt session writes. - -Local telemetry only. Nothing here uploads anywhere. -""" - -from __future__ import annotations - -import json -import logging -import queue -import sqlite3 -import threading -import time -from pathlib import Path -from typing import Any, Dict, Optional - -logger = logging.getLogger(__name__) - -_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full -_DRAIN_BATCH = 256 - - -def _default_dir() -> Path: - """Resolve the telemetry dir under the active HERMES_HOME (profile-safe).""" - from hermes_constants import get_hermes_home - return get_hermes_home() / "telemetry" - - -def _default_db_path() -> Path: - """Resolve state.db under the active HERMES_HOME (profile-safe).""" - from hermes_constants import get_hermes_home - return get_hermes_home() / "state.db" - - -# Map a telemetry event dict (its "event" tag) to (table, column-ordered insert). -# Only the columns the indexer knows about are written; unknown keys are ignored, -# so an event carrying extra fields never breaks the insert. -_TABLE_COLUMNS: Dict[str, tuple] = { - "run": ( - "tel_runs", - ("run_id", "trace_id", "session_id", "entrypoint", - "platform", "start_ns", "end_ns", "end_reason", - "model_call_count", "tool_call_count", "error_count"), - ), - "span": ( - "tel_spans", - ("span_id", "trace_id", "run_id", "parent_span_id", "name", "kind", - "start_ns", "end_ns", "status"), - ), - "model_call": ( - "tel_model_calls", - ("span_id", "run_id", "provider", "model", "base_url", - "input_tokens", "output_tokens", "cache_read_tokens", - "cache_write_tokens", "reasoning_tokens", "latency_ms"), - ), - "tool_call": ( - "tel_tool_calls", - ("span_id", "run_id", "tool_name", "duration_ms", "result_class"), - ), - "error": ( - "tel_error_events", - ("run_id", "error_class", "subsystem", "recovery", "ts_ns"), - ), -} - - -class TelemetryEmitter: - """Owns the queue, the writer thread, and the telemetry sqlite connection.""" - - def __init__( - self, - *, - events_path: Optional[Path] = None, - db_path: Optional[Path] = None, - enabled: bool = True, - ) -> None: - self._dir = (events_path.parent if events_path else _default_dir()) - self._events_path = events_path or (self._dir / "events.jsonl") - self._db_path = db_path or _default_db_path() - self._enabled = enabled - self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE) - self._dropped = 0 - self._written = 0 - self._stop = threading.Event() - self._started = False - self._lock = threading.Lock() - self._conn: Optional[sqlite3.Connection] = None - self._thread: Optional[threading.Thread] = None - # Optional live subscribers (e.g. OTLP exporter). Called from the writer - # thread AFTER durable writes, fully fail-isolated — a subscriber that - # raises or blocks can never affect the JSONL/SQLite source of truth or - # the hot path. Each subscriber is callable(batch: list[dict]). - self._subscribers: list = [] - - # ── public API (hot path) ─────────────────────────────────────────────── - def emit(self, event: Any) -> None: - """Enqueue an event. Never blocks, never raises. - - ``event`` may be a dataclass with ``to_dict()`` or a plain dict. - """ - if not self._enabled: - return - try: - payload = event.to_dict() if hasattr(event, "to_dict") else dict(event) - payload.setdefault("ts_ns", time.time_ns()) - self._ensure_started() - try: - self._q.put_nowait(payload) - except queue.Full: - # Drop oldest to make room — bounded memory, newest-wins. - try: - self._q.get_nowait() - self._dropped += 1 - self._q.put_nowait(payload) - except Exception: - self._dropped += 1 - except Exception: # the hot-path invariant: never propagate - logger.debug("telemetry emit failed", exc_info=True) - - # ── lifecycle ─────────────────────────────────────────────────────────── - def _ensure_started(self) -> None: - if self._started: - return - with self._lock: - if self._started: - return - try: - self._dir.mkdir(parents=True, exist_ok=True) - except Exception: - logger.debug("telemetry dir create failed", exc_info=True) - self._thread = threading.Thread( - target=self._run, name="hermes-telemetry-writer", daemon=True - ) - self._thread.start() - self._started = True - - def _open_conn(self) -> Optional[sqlite3.Connection]: - if self._conn is not None: - return self._conn - try: - conn = sqlite3.connect(str(self._db_path), isolation_level=None, timeout=5.0) - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA busy_timeout=5000") - self._conn = conn - except Exception: - logger.debug("telemetry db open failed", exc_info=True) - self._conn = None - return self._conn - - def _run(self) -> None: - while not self._stop.is_set(): - try: - first = self._q.get(timeout=0.5) - except queue.Empty: - continue - batch = [first] - while len(batch) < _DRAIN_BATCH: - try: - batch.append(self._q.get_nowait()) - except queue.Empty: - break - self._write_batch(batch) - - def _write_batch(self, batch) -> None: - # JSONL append (source of truth) — best effort. - try: - with open(self._events_path, "a", encoding="utf-8") as fh: - for ev in batch: - fh.write(json.dumps(ev, ensure_ascii=False) + "\n") - except Exception: - logger.debug("telemetry jsonl append failed", exc_info=True) - - # SQLite index — best effort, per-event so one bad row can't lose the batch. - conn = self._open_conn() - if conn is None: - return - for ev in batch: - try: - self._index_one(conn, ev) - self._written += 1 - except Exception: - logger.debug("telemetry index row failed", exc_info=True) - - # Live fan-out (e.g. OTLP) — AFTER durable writes, fully fail-isolated. - # A slow/raising subscriber never affects JSONL/SQLite or the hot path. - for sub in self._subscribers: - try: - sub(batch) - except Exception: - logger.debug("telemetry subscriber failed", exc_info=True) - - def subscribe(self, callback) -> None: - """Register a live batch subscriber (callable(batch: list[dict])). - - Called from the writer thread after durable writes. Used by the OTLP - exporter for continuous streaming. Fail-isolated; never on the hot path. - """ - if callback not in self._subscribers: - self._subscribers.append(callback) - - def unsubscribe(self, callback) -> None: - try: - self._subscribers.remove(callback) - except ValueError: - pass - - def _index_one(self, conn: sqlite3.Connection, ev: Dict[str, Any]) -> None: - kind = ev.get("event") - spec = _TABLE_COLUMNS.get(kind) - if spec is None: - return - table, cols = spec - values = [ev.get(c) for c in cols] - placeholders = ", ".join("?" for _ in cols) - collist = ", ".join(cols) - conn.execute( - f"INSERT OR REPLACE INTO {table} ({collist}) VALUES ({placeholders})", - values, - ) - - # ── introspection / shutdown (tests, CLI) ─────────────────────────────── - def flush(self, timeout: float = 2.0) -> None: - """Block until the queue drains (test/CLI helper, NOT the hot path).""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if self._q.empty(): - # give the writer a tick to finish the in-flight batch - time.sleep(0.05) - if self._q.empty(): - return - time.sleep(0.02) - - def stats(self) -> Dict[str, int]: - return { - "queued": self._q.qsize(), - "written": self._written, - "dropped": self._dropped, - } - - def close(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=2.0) - if self._conn is not None: - try: - self._conn.close() - except Exception: - pass - self._conn = None - self._started = False - - -# ── process-wide singleton ────────────────────────────────────────────────── -_EMITTER: Optional[TelemetryEmitter] = None -_EMITTER_LOCK = threading.Lock() - - -def get_emitter() -> TelemetryEmitter: - """Return the process-wide emitter, honoring telemetry.local config.""" - global _EMITTER - if _EMITTER is not None: - return _EMITTER - with _EMITTER_LOCK: - if _EMITTER is None: - enabled = _local_enabled() - _EMITTER = TelemetryEmitter(enabled=enabled) - return _EMITTER - - -def _local_enabled() -> bool: - try: - from hermes_cli.config import load_config - cfg = load_config() - tel = cfg.get("telemetry") if isinstance(cfg, dict) else {} - return bool((tel or {}).get("local", True)) - except Exception: - return True - - -def emit(event: Any) -> None: - """Module-level convenience: emit via the singleton.""" - get_emitter().emit(event) - - -def reset_emitter_for_tests(emitter: Optional[TelemetryEmitter] = None) -> None: - """Swap the singleton (tests only).""" - global _EMITTER - with _EMITTER_LOCK: - if _EMITTER is not None and emitter is not _EMITTER: - try: - _EMITTER.close() - except Exception: - pass - _EMITTER = emitter - - -__all__ = [ - "TelemetryEmitter", - "get_emitter", - "emit", - "reset_emitter_for_tests", -] diff --git a/agent/telemetry/events.py b/agent/telemetry/events.py deleted file mode 100644 index 020ce11a9611f..0000000000000 --- a/agent/telemetry/events.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Typed local telemetry events. - -These dataclasses are the rows written to the local JSONL log and the ``tel_*`` -SQLite tables. They record the values observed for each run — model id, provider, tool -name, token counts, durations — and stay on the machine unless explicitly exported. -""" - -from __future__ import annotations - -import time -from dataclasses import dataclass, field, asdict -from typing import Any, Dict, Optional - -# ── local telemetry events (real values) ──────────────────────────────────── - - -def _now_ns() -> int: - return time.time_ns() - - -@dataclass(slots=True) -class RunEvent: - """One top-level workflow execution (a trace root). A run spans one session.""" - run_id: str - trace_id: str - entrypoint: str - session_id: Optional[str] = None - platform: Optional[str] = None - start_ns: int = field(default_factory=_now_ns) - end_ns: Optional[int] = None - end_reason: Optional[str] = None - model_call_count: int = 0 - tool_call_count: int = 0 - error_count: int = 0 - - def to_dict(self) -> Dict[str, Any]: - return {"event": "run", **asdict(self)} - - -@dataclass(slots=True) -class ModelCallEvent: - span_id: str - run_id: str - provider: Optional[str] = None # raw provider, e.g. "anthropic" - model: Optional[str] = None # raw model id, e.g. "claude-opus-4" - base_url: Optional[str] = None - input_tokens: int = 0 - output_tokens: int = 0 - cache_read_tokens: int = 0 - cache_write_tokens: int = 0 - reasoning_tokens: int = 0 - latency_ms: Optional[int] = None - - def to_dict(self) -> Dict[str, Any]: - return {"event": "model_call", **asdict(self)} - - -@dataclass(slots=True) -class ToolCallEvent: - span_id: str - run_id: str - tool_name: Optional[str] = None # raw tool name, e.g. "web_search" - duration_ms: Optional[int] = None - result_class: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - return {"event": "tool_call", **asdict(self)} - - -@dataclass(slots=True) -class SpanEvent: - """A timed span — the timing/lineage backbone of a trace. - - One row per run (the root, ``parent_span_id=None``) and one per model/tool call - (``parent_span_id`` = the run's root span). Detail rows in ``tel_model_calls`` / - ``tel_tool_calls`` share the ``span_id`` and are joined here for ordering and - placement on a timeline. - """ - span_id: str - trace_id: str - run_id: str - name: str - kind: str # "run" | "model" | "tool" - start_ns: int - end_ns: Optional[int] = None - parent_span_id: Optional[str] = None - status: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - return {"event": "span", **asdict(self)} - - -@dataclass(slots=True) -class ErrorEvent: - run_id: Optional[str] - error_class: str - subsystem: str - recovery: Optional[str] = None - ts_ns: int = field(default_factory=_now_ns) - - def to_dict(self) -> Dict[str, Any]: - return {"event": "error", **asdict(self)} - - -__all__ = [ - "RunEvent", - "ModelCallEvent", - "ToolCallEvent", - "SpanEvent", - "ErrorEvent", -] diff --git a/agent/telemetry/exporter_bulk.py b/agent/telemetry/exporter_bulk.py deleted file mode 100644 index 32be393f72374..0000000000000 --- a/agent/telemetry/exporter_bulk.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Export telemetry (and optionally session content) to a file or stream. - -Two data domains, both written to an operator-chosen destination: - - * Telemetry: the tel_* rows + events.jsonl (structural observability). - * Content (opt-in via telemetry.trajectories): sessions + messages, with every - content field (message body, reasoning, raw tool-call args) passed through the - redaction pipeline (secrets always stripped; PII per content_redaction). - -Formats: ndjson (default) and json. OTLP streaming export lives in otlp_exporter.py. - -Content export is gated by ``redaction.content_export_enabled``. -""" - -from __future__ import annotations - -import json -import sqlite3 -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional, TextIO - -from . import redaction - -_TEL_TABLES = ( - "tel_runs", "tel_model_calls", "tel_tool_calls", "tel_error_events", -) - - -def _open(db_path: Optional[Path]) -> sqlite3.Connection: - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - return c - - -def _iter_telemetry(conn: sqlite3.Connection, since_ns: Optional[int]) -> Iterator[Dict[str, Any]]: - for table in _TEL_TABLES: - # only tel_runs has start_ns; window the rest by run join when needed. - if table == "tel_runs" and since_ns: - rows = conn.execute( - f"SELECT * FROM {table} WHERE start_ns >= ?", (int(since_ns),) - ).fetchall() - else: - rows = conn.execute(f"SELECT * FROM {table}").fetchall() - for r in rows: - d = dict(r) - d["_kind"] = table - yield d - - -def _iter_content( - db_path: Optional[Path], - *, - config: Optional[Dict[str, Any]], - include_content: bool, -) -> Iterator[Dict[str, Any]]: - """Yield session records. Message bodies included only when trajectories on.""" - from hermes_state import SessionDB - - content_mode = redaction.content_mode_for(config) - db = SessionDB(db_path=db_path) if db_path else SessionDB() - try: - for session in db.export_all(): - msgs = session.get("messages", []) or [] - red_msgs = [ - redaction.redact_message( - m, content_mode=content_mode, include_content=include_content - ) - for m in msgs - ] - # Session-level metadata is structural; keep ids/model/counts, drop - # any free-text title only when content is excluded. - out = { - "_kind": "session", - "id": session.get("id"), - "source": session.get("source"), - "model": session.get("model"), - "started_at": session.get("started_at"), - "ended_at": session.get("ended_at"), - "message_count": session.get("message_count"), - "tool_call_count": session.get("tool_call_count"), - "messages": red_msgs, - } - if include_content and session.get("title"): - out["title"] = redaction.redact_for_export( - session["title"], content_mode=content_mode - ) - yield out - finally: - db.close() - - -def export( - out: TextIO, - *, - fmt: str = "ndjson", - since_ns: Optional[int] = None, - include_content: bool = False, - config: Optional[Dict[str, Any]] = None, - db_path: Optional[Path] = None, -) -> Dict[str, int]: - """Write telemetry (+ optional content) to ``out``. Returns counts. - - ``include_content`` is honored only when telemetry.trajectories is enabled in - ``config``; otherwise content is forced off and only structural data is written. - """ - # Trajectories gate: a flag cannot override the config setting. - content_allowed = include_content and redaction.content_export_enabled(config) - counts = {"telemetry": 0, "sessions": 0, "content_included": int(content_allowed)} - - conn = _open(db_path) - records: List[Dict[str, Any]] = [] - try: - for rec in _iter_telemetry(conn, since_ns): - counts["telemetry"] += 1 - if fmt == "ndjson": - out.write(json.dumps(rec, ensure_ascii=False) + "\n") - else: - records.append(rec) - finally: - conn.close() - - # Content/session domain (separate connection via SessionDB). - for rec in _iter_content(db_path, config=config, include_content=content_allowed): - counts["sessions"] += 1 - if fmt == "ndjson": - out.write(json.dumps(rec, ensure_ascii=False) + "\n") - else: - records.append(rec) - - if fmt != "ndjson": - json.dump({"records": records}, out, ensure_ascii=False, indent=2) - - return counts - - -__all__ = ["export"] diff --git a/agent/telemetry/metrics.py b/agent/telemetry/metrics.py deleted file mode 100644 index fda0b4ee41353..0000000000000 --- a/agent/telemetry/metrics.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Derive metric rollups from the local telemetry tables. - -Reads the ``tel_*`` tables in state.db and returns aggregates for /usage, /insights, -and local dashboards. Metrics are computed by querying the event log rather than being -emitted on the hot path. - -Each function accepts either an open caller-owned ``conn`` (reused, not closed) or a -``db_path`` (opened and closed internally). InsightsEngine passes its existing -connection; a standalone dashboard passes a path. -""" - -from __future__ import annotations - -import sqlite3 -from contextlib import contextmanager -from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional - - -@contextmanager -def _cursor( - conn: Optional[sqlite3.Connection], db_path: Optional[Path] -) -> Iterator[sqlite3.Connection]: - """Yield a Row-factory connection. Closes it only if we opened it.""" - if conn is not None: - prev_factory = conn.row_factory - conn.row_factory = sqlite3.Row - try: - yield conn - finally: - conn.row_factory = prev_factory - return - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - try: - yield c - finally: - c.close() - - -def _since_clause(since_ns: Optional[int], col: str = "start_ns") -> str: - return f" WHERE {col} >= {int(since_ns)}" if since_ns else "" - - -def workflow_summary( - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - """Run-level counters + duration percentiles (local telemetry, exact).""" - with _cursor(conn, db_path) as c: - where = _since_clause(since_ns) - total = c.execute(f"SELECT COUNT(*) n FROM tel_runs{where}").fetchone()["n"] - by_reason = { - r["end_reason"] or "unknown": r["n"] - for r in c.execute( - f"SELECT end_reason, COUNT(*) n FROM tel_runs{where} GROUP BY end_reason" - ).fetchall() - } - by_entry = { - r["entrypoint"] or "unknown": r["n"] - for r in c.execute( - f"SELECT entrypoint, COUNT(*) n FROM tel_runs{where} GROUP BY entrypoint" - ).fetchall() - } - dur_where = (where + " AND end_ns IS NOT NULL") if where else " WHERE end_ns IS NOT NULL" - durations = [ - (r["end_ns"] - r["start_ns"]) / 1e6 - for r in c.execute( - f"SELECT start_ns, end_ns FROM tel_runs{dur_where}" - ).fetchall() - ] - return { - "total_runs": total, - "by_end_reason": by_reason, - "by_entrypoint": by_entry, - "duration_ms_p50": _pct(durations, 50), - "duration_ms_p95": _pct(durations, 95), - "success_rate": round(by_reason.get("completed", 0) / total, 4) if total else 0.0, - } - - -def model_call_summary( - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - with _cursor(conn, db_path) as c: - rows = c.execute( - "SELECT provider, model, COUNT(*) n, " - "SUM(input_tokens) inp, SUM(output_tokens) outp, " - "SUM(cache_read_tokens) cache, AVG(latency_ms) avg_latency " - "FROM tel_model_calls GROUP BY provider, model" - ).fetchall() - by_provider: Dict[str, int] = {} - by_model: Dict[str, int] = {} - tokens = {"input": 0, "output": 0, "cache_read": 0} - breakdown: List[Dict[str, Any]] = [] - for r in rows: - prov = r["provider"] or "unknown" - mdl = r["model"] or "unknown" - by_provider[prov] = by_provider.get(prov, 0) + r["n"] - by_model[mdl] = by_model.get(mdl, 0) + r["n"] - tokens["input"] += r["inp"] or 0 - tokens["output"] += r["outp"] or 0 - tokens["cache_read"] += r["cache"] or 0 - breakdown.append({ - "provider": r["provider"], - "model": r["model"], - "calls": r["n"], - "avg_latency_ms": round(r["avg_latency"] or 0, 1), - }) - cache_total = tokens["cache_read"] + tokens["input"] - return { - "by_provider": by_provider, - "by_model": by_model, - "tokens": tokens, - "cache_hit_rate": round(tokens["cache_read"] / cache_total, 4) if cache_total else 0.0, - "breakdown": breakdown, - } - - -def tool_call_summary( - db_path: Optional[Path] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - with _cursor(conn, db_path) as c: - by_tool = { - r["tool_name"] or "unknown": r["n"] - for r in c.execute( - "SELECT tool_name, COUNT(*) n FROM tel_tool_calls GROUP BY tool_name" - ).fetchall() - } - fails = { - r["tool_name"] or "unknown": r["n"] - for r in c.execute( - "SELECT tool_name, COUNT(*) n FROM tel_tool_calls " - "WHERE result_class IN ('error','timeout','blocked') GROUP BY tool_name" - ).fetchall() - } - total = sum(by_tool.values()) - total_fail = sum(fails.values()) - return { - "by_tool": by_tool, - "failures_by_tool": fails, - "total": total, - "failure_rate": round(total_fail / total, 4) if total else 0.0, - } - - -def error_summary( - db_path: Optional[Path] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - with _cursor(conn, db_path) as c: - return { - "by_class": { - r["error_class"] or "unknown": r["n"] - for r in c.execute( - "SELECT error_class, COUNT(*) n FROM tel_error_events GROUP BY error_class" - ).fetchall() - }, - } - - -def _pct(values: List[float], p: int) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * (p / 100) - lo = int(k) - hi = min(lo + 1, len(s) - 1) - frac = k - lo - return round(s[lo] + (s[hi] - s[lo]) * frac, 2) - - -def overview( - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> Dict[str, Any]: - """One call for a dashboard: all the rollups.""" - return { - "workflows": workflow_summary(db_path, since_ns, conn=conn), - "model_calls": model_call_summary(db_path, since_ns, conn=conn), - "tool_calls": tool_call_summary(db_path, conn=conn), - "errors": error_summary(db_path, conn=conn), - } - - -def has_data( - db_path: Optional[Path] = None, - *, - conn: Optional[sqlite3.Connection] = None, -) -> bool: - """True when any telemetry runs exist (cheap guard for /insights rendering).""" - try: - with _cursor(conn, db_path) as c: - return c.execute("SELECT 1 FROM tel_runs LIMIT 1").fetchone() is not None - except Exception: - return False - - -__all__ = [ - "workflow_summary", - "model_call_summary", - "tool_call_summary", - "error_summary", - "overview", - "has_data", -] diff --git a/agent/telemetry/policy.py b/agent/telemetry/policy.py deleted file mode 100644 index 77e794cf8a26f..0000000000000 --- a/agent/telemetry/policy.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Telemetry consent posture and the aggregate-metrics gate. - -Consent is a single config field, ``telemetry.consent_state``: - - * "unknown" — no choice recorded; never uploads (the default). - * "local" — declined aggregate metrics; local telemetry only. - * "aggregate" — opted in to aggregate metrics. - -The config file is the source of truth: set ``telemetry.consent_state`` with -``hermes config set`` (or a managed-scope pin). Callers that gate behavior read -``telemetry.*`` directly from config; this module only provides the consent -constants, the install-id helper, and the upload gate a future uploader must -consult. - -``allow_aggregate`` is the hard gate. An administrator pins -``telemetry.allow_aggregate: false`` through the managed-scope layer -(``/etc/hermes/config.yaml``), which takes precedence over the user's config; when -it is false, aggregate metrics are off regardless of ``consent_state``. -""" - -from __future__ import annotations - -import uuid -from typing import Any, Dict - -CONSENT_UNKNOWN = "unknown" -CONSENT_LOCAL = "local" -CONSENT_AGGREGATE = "aggregate" -VALID_CONSENT_STATES = {CONSENT_UNKNOWN, CONSENT_LOCAL, CONSENT_AGGREGATE} - - -def _telemetry_cfg(config: Dict[str, Any]) -> Dict[str, Any]: - cfg = config.get("telemetry") if isinstance(config, dict) else None - return cfg if isinstance(cfg, dict) else {} - - -def ensure_install_id(config: Dict[str, Any]) -> str: - """Return a stable install id, minting one if the config slot is empty. - - Does not persist — the caller writes the returned value back to config.yaml. A - fresh uuid4 is used; clearing ``telemetry.install_id`` (e.g. with - ``hermes config set telemetry.install_id ""``) causes the next call to mint anew. - """ - tel = _telemetry_cfg(config) - existing = tel.get("install_id") - if isinstance(existing, str) and existing.strip(): - return existing - return str(uuid.uuid4()) - - -def may_upload_aggregate(config: Dict[str, Any]) -> bool: - """Whether aggregate metrics may upload — the gate a future uploader consults. - - Aggregate metrics are derived from the local telemetry tables, so they require - local telemetry to be on. True only when local telemetry is enabled, the admin - hard gate allows it, and the user has opted in via ``telemetry.consent_state``. - """ - tel = _telemetry_cfg(config) - local_enabled = bool(tel.get("local", True)) - allow_aggregate = bool(tel.get("allow_aggregate", True)) - state = tel.get("consent_state", CONSENT_UNKNOWN) - return local_enabled and allow_aggregate and state == CONSENT_AGGREGATE - - -__all__ = [ - "CONSENT_UNKNOWN", - "CONSENT_LOCAL", - "CONSENT_AGGREGATE", - "VALID_CONSENT_STATES", - "may_upload_aggregate", - "ensure_install_id", -] diff --git a/agent/telemetry/redaction.py b/agent/telemetry/redaction.py deleted file mode 100644 index 01090abb22a74..0000000000000 --- a/agent/telemetry/redaction.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Redaction applied to telemetry data on export. - -Two independent controls: - - * Secrets are always redacted, on every export and in every mode; no setting - disables this. Wraps ``agent/redact.py::redact_sensitive_text(force=True)``. - - * Whether message bodies, reasoning, and raw tool arguments are exportable at all is - governed by the trajectories setting (``telemetry.trajectories.enabled``, default - off, admin-pinnable), not by a redaction mode. With trajectories off, content is - dropped. With it on, content is exportable and ``content_redaction`` (none|pii) - controls how much is scrubbed; secrets are still always stripped. - -This applies to the local and trajectory export paths. It is unrelated to any -aggregate-metrics path. -""" - -from __future__ import annotations - -import re -from typing import Any, Dict, List, Optional - -# Content-redaction strengths for any content that IS exported. -CONTENT_NONE = "none" # drop content entirely (structural telemetry only) -CONTENT_PII = "pii" # codec-aware PII redaction on exported content -CONTENT_MODES = {CONTENT_NONE, CONTENT_PII} - -# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ── -_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") -# E.164-ish and common separators; conservative to avoid nuking code/IDs. -_PHONE_RE = re.compile( - r"(? Optional[str]: - """Always-on secret redaction. force=True so user config can't disable it.""" - if text is None: - return None - try: - from agent.redact import redact_sensitive_text - return redact_sensitive_text(str(text), force=True) - except Exception: - # Fail CLOSED: if the redactor can't run, do not emit the raw string. - return "[redaction-unavailable]" - - -def _pii_redact(text: str) -> str: - text = _EMAIL_RE.sub("[email]", text) - text = _UUID_RE.sub("[id]", text) - text = _PHONE_RE.sub("[phone]", text) - return text - - -def redact_for_export( - text: Optional[str], - *, - content_mode: str = CONTENT_NONE, -) -> Optional[str]: - """Redact a single content string for export. - - Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'. - Callers gate *whether content is exported at all* via telemetry.trajectories - (see ``content_export_enabled``); this function only scrubs content that the - caller has already decided to export. - """ - redacted = _secret_redact(text) - if redacted is None: - return None - if content_mode == CONTENT_PII: - redacted = _pii_redact(redacted) - return redacted - - -def content_export_enabled(config: Optional[Dict[str, Any]]) -> bool: - """True only when telemetry.trajectories is explicitly enabled. - - This is the consent gate for exporting message bodies / reasoning / raw tool - args. Default off. Admin-pinnable via managed scope (telemetry.trajectories.enabled). - """ - try: - tel = (config or {}).get("telemetry") or {} - traj = tel.get("trajectories") or {} - return bool(traj.get("enabled", False)) - except Exception: - return False - - -def content_mode_for(config: Optional[Dict[str, Any]]) -> str: - try: - tel = (config or {}).get("telemetry") or {} - mode = tel.get("content_redaction", CONTENT_NONE) - return mode if mode in CONTENT_MODES else CONTENT_NONE - except Exception: - return CONTENT_NONE - - -# ── Codec-aware message redaction (NeMo pattern) ───────────────────────────── -# Redact the right fields of a provider message shape rather than regex-blasting -# the whole blob. Structure (roles, names, counts) is preserved; only the -# free-text content fields are scrubbed. - -def redact_message( - msg: Dict[str, Any], - *, - content_mode: str = CONTENT_NONE, - include_content: bool = False, -) -> Dict[str, Any]: - """Redact one chat message dict for export. - - When include_content is False (trajectories off), content/reasoning/tool-arg - fields are dropped — only structural fields (role, tool name, counts) remain. - When True, those fields are kept but passed through redact_for_export. - """ - role = msg.get("role") - out: Dict[str, Any] = {"role": role} - - # Always-structural fields. - if msg.get("tool_name") is not None: - out["tool_name"] = msg.get("tool_name") - if msg.get("name") is not None: - out["name"] = msg.get("name") - - if not include_content: - # Structural only: record presence/size, not bytes. - c = msg.get("content") - if c is not None: - out["content_chars"] = len(str(c)) - if msg.get("reasoning_content"): - out["reasoning_chars"] = len(str(msg["reasoning_content"])) - if msg.get("tool_calls"): - out["tool_call_count"] = _count_tool_calls(msg["tool_calls"]) - return out - - # Content included (trajectories enabled): scrub then keep. - if msg.get("content") is not None: - out["content"] = redact_for_export(msg["content"], content_mode=content_mode) - if msg.get("reasoning_content"): - out["reasoning_content"] = redact_for_export( - msg["reasoning_content"], content_mode=content_mode - ) - if msg.get("tool_calls"): - out["tool_calls"] = _redact_tool_calls(msg["tool_calls"], content_mode=content_mode) - return out - - -def _count_tool_calls(tool_calls: Any) -> int: - try: - import json - tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls - return len(tc) if isinstance(tc, list) else (1 if tc else 0) - except Exception: - return 0 - - -def _redact_tool_calls(tool_calls: Any, *, content_mode: str) -> Any: - """Redact raw tool-call arguments (free text) while keeping function names.""" - import json - try: - tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls - except Exception: - return "[unparseable-tool-calls]" - if not isinstance(tc, list): - return [] - out: List[Dict[str, Any]] = [] - for call in tc: - if not isinstance(call, dict): - continue - fn = (call.get("function") or {}) if isinstance(call.get("function"), dict) else {} - name = fn.get("name") or call.get("name") - args = fn.get("arguments") - red_args = redact_for_export(args, content_mode=content_mode) if args is not None else None - out.append({"name": name, "arguments": red_args}) - return out - - -__all__ = [ - "CONTENT_NONE", - "CONTENT_PII", - "CONTENT_MODES", - "redact_for_export", - "content_export_enabled", - "content_mode_for", - "redact_message", -] diff --git a/agent/telemetry/rollup.py b/agent/telemetry/rollup.py deleted file mode 100644 index 3880a55caa8ec..0000000000000 --- a/agent/telemetry/rollup.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Build per-run summary events from the local telemetry tables. - -Reads the ``tel_*`` tables and projects each completed run into a summary dict holding -the recorded values: provider, models used, tool names, token totals, duration, and -cost. Powers ``hermes telemetry preview``. No aggregation or bucketing is applied here. -""" - -from __future__ import annotations - -import platform -import sqlite3 -from pathlib import Path -from typing import Any, Dict, List, Optional - - -def _os_family() -> str: - s = platform.system().lower() - if s.startswith("lin"): - return "linux" - if s == "darwin": - return "macos" - if s.startswith("win"): - return "windows" - return "other" - - -def _hermes_version() -> str: - try: - from hermes_cli import __version__ - return str(__version__) - except Exception: - return "0.0.0" - - -def _open(db_path: Optional[Path], conn: Optional[sqlite3.Connection]): - if conn is not None: - prev = conn.row_factory - conn.row_factory = sqlite3.Row - return conn, prev, False - if db_path is None: - from hermes_constants import get_hermes_home - db_path = get_hermes_home() / "state.db" - c = sqlite3.connect(str(db_path), timeout=5.0) - c.row_factory = sqlite3.Row - return c, None, True - - -def _run_events(c: sqlite3.Connection, since_ns: Optional[int]) -> List[Dict[str, Any]]: - """Project completed runs into per-run summary dicts.""" - where = " WHERE end_ns IS NOT NULL" - if since_ns: - where += f" AND start_ns >= {int(since_ns)}" - rows = c.execute( - "SELECT run_id, entrypoint, platform, end_reason, start_ns, end_ns, " - "model_call_count, tool_call_count, error_count " - "FROM tel_runs" + where - ).fetchall() - - events: List[Dict[str, Any]] = [] - for r in rows: - # Models actually used in this run (real ids), with token totals. - models = [ - {"provider": m["provider"], "model": m["model"], - "calls": m["n"], "input_tokens": int(m["inp"] or 0), - "output_tokens": int(m["outp"] or 0)} - for m in c.execute( - "SELECT provider, model, COUNT(*) n, SUM(input_tokens) inp, " - "SUM(output_tokens) outp FROM tel_model_calls WHERE run_id = ? " - "GROUP BY provider, model ORDER BY n DESC", - (r["run_id"],), - ).fetchall() - ] - tools = [ - row["tool_name"] - for row in c.execute( - "SELECT DISTINCT tool_name FROM tel_tool_calls WHERE run_id = ?", - (r["run_id"],), - ).fetchall() - if row["tool_name"] - ] - trow = c.execute( - "SELECT SUM(input_tokens) inp, SUM(output_tokens) outp " - "FROM tel_model_calls WHERE run_id = ?", - (r["run_id"],), - ).fetchone() - duration_ms = (r["end_ns"] - r["start_ns"]) / 1e6 if r["end_ns"] else None - events.append({ - "event_name": "workflow_completed", - "run_id": r["run_id"], - "entrypoint": r["entrypoint"] or "cli", - "platform": r["platform"], - "end_reason": r["end_reason"] or "completed", - "models_used": models, - "tools_used": tools, - "model_call_count": r["model_call_count"] or 0, - "tool_call_count": r["tool_call_count"] or 0, - "error_count": r["error_count"] or 0, - "duration_ms": round(duration_ms, 1) if duration_ms is not None else None, - "input_tokens": int((trow["inp"] if trow else 0) or 0), - "output_tokens": int((trow["outp"] if trow else 0) or 0), - }) - return events - - -def build_aggregate_events( - *, - install_id: str, - db_path: Optional[Path] = None, - since_ns: Optional[int] = None, - conn: Optional[sqlite3.Connection] = None, - include_heartbeat: bool = True, -) -> List[Dict[str, Any]]: - """Return per-run summary events plus an optional heartbeat.""" - c, prev_factory, owned = _open(db_path, conn) - try: - events = _run_events(c, since_ns) - if include_heartbeat: - events.append({ - "event_name": "heartbeat", - "install_id": install_id, - "hermes_version": _hermes_version(), - "os_family": _os_family(), - "entrypoint": "cli", - }) - return events - finally: - if owned: - c.close() - elif prev_factory is not None: - c.row_factory = prev_factory - - -def summarize(events: List[Dict[str, Any]]) -> Dict[str, Any]: - """Counts by event_name + field coverage, for status/preview output.""" - by_name: Dict[str, int] = {} - fields = set() - for e in events: - name = e.get("event_name", "?") - by_name[name] = by_name.get(name, 0) + 1 - fields.update(e.keys()) - return {"total": len(events), "by_event_name": by_name, "fields_present": sorted(fields)} - - -__all__ = ["build_aggregate_events", "summarize"] diff --git a/agent/telemetry/spans.py b/agent/telemetry/spans.py deleted file mode 100644 index bfd876f8a112f..0000000000000 --- a/agent/telemetry/spans.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Trace / run / span id propagation via contextvars. - -Telemetry events share IDs so a workflow can be reconstructed: one ``trace_id`` per -workflow, one ``run_id`` per top-level execution, ``span_id`` per timed operation, and -``parent_span_id`` for nesting. These live in contextvars so async tool calls and -spawned subagents inherit the lineage automatically. - -Provides helpers to start/clear a run context and mint child span ids. The telemetry -plugin sets the run context on session start and reads it in each hook callback. -Nothing here writes to storage — it only carries ids. -""" - -from __future__ import annotations - -import contextvars -import uuid -from dataclasses import dataclass -from typing import Optional - -_trace_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( - "hermes_tel_trace_id", default=None -) -_run_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( - "hermes_tel_run_id", default=None -) -_parent_span_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( - "hermes_tel_parent_span_id", default=None -) - - -def new_id() -> str: - return uuid.uuid4().hex - - -@dataclass(slots=True) -class RunContext: - trace_id: str - run_id: str - - -def start_run(trace_id: Optional[str] = None, run_id: Optional[str] = None) -> RunContext: - """Begin a run context, minting ids when not supplied. Sets contextvars.""" - tid = trace_id or new_id() - rid = run_id or new_id() - _trace_id.set(tid) - _run_id.set(rid) - _parent_span_id.set(None) - return RunContext(trace_id=tid, run_id=rid) - - -def current_trace_id() -> Optional[str]: - return _trace_id.get() - - -def current_run_id() -> Optional[str]: - return _run_id.get() - - -def current_parent_span_id() -> Optional[str]: - return _parent_span_id.get() - - -def new_span_id() -> str: - """Mint a span id (does not alter the parent pointer).""" - return new_id() - - -def clear_run() -> None: - _trace_id.set(None) - _run_id.set(None) - _parent_span_id.set(None) - - -__all__ = [ - "RunContext", - "new_id", - "start_run", - "current_trace_id", - "current_run_id", - "current_parent_span_id", - "new_span_id", - "clear_run", -] diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 982708c342262..d52e3ec009c14 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1414,49 +1414,6 @@ display: # # Routing/delivery still uses the original values internally. # redact_pii: false -# ============================================================================= -# Telemetry & Observability -# ============================================================================= -# Three settings, isolated from each other: -# local — full-fidelity observability you own. Default ON. -# aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF. -# trajectories — content trajectories for training. Separate consent (later). -# -# Local telemetry records real values (actual models, providers, tool names) — -# your own data, on your machine. Aggregate metrics are opt-in and have no -# uploader today; if one ships it would summarize at that egress boundary. -# -# Enterprise locking: any telemetry.* key can be pinned by an administrator via -# the managed-scope layer (/etc/hermes/config.yaml), which wins over the user's -# value. To hard-forbid egress on a locked-down deployment, pin -# `telemetry.allow_aggregate: false` there. -# telemetry: -# # Local telemetry: event log + SQLite index in state.db. Never leaves your -# # machine unless you export it or opt into aggregate metrics. -# local: true -# # Hard gate for aggregate metrics. When false, aggregate metrics are off -# # regardless of consent_state. Pin false via managed scope to forbid egress. -# allow_aggregate: true -# # Aggregate-metrics consent (the opt-in). No uploader ships yet. -# # unknown (no choice — never uploads) | local (declined) | aggregate (opted in) -# consent_state: unknown -# # Stable install id (aggregate metrics only). Empty = mint on first use; -# # clear it to rotate. -# install_id: "" -# # Local event-log retention before rotation (days). -# retention_days: 90 -# # Keep secret redaction on even at full local capture. -# redact_secrets: true -# # Content redaction for exports / support bundles: none | pii. -# content_redaction: none -# # Exporters. The OTLP exporter sends spans to a configured Collector endpoint; -# # header values reference environment variable names, not inline secrets. -# export: -# otlp: -# enabled: false -# endpoint: null -# headers_env: {} # e.g. {Authorization: MY_OTLP_TOKEN_ENVVAR} - # ============================================================================= # Shell-script hooks # ============================================================================= diff --git a/docs/observability/README.md b/docs/observability/README.md index 8717627d8a885..9040929ca4892 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -307,13 +307,6 @@ nested agent work or security lifecycle events. ## Existing Consumers -The bundled **`telemetry`** plugin is the built-in local-first telemetry system: it -records runs, spans, model calls, and tool calls to a local event log + `state.db`, powers -`hermes insights`, and supports export to your own file or OpenTelemetry Collector. It -is on by default and never sends data to Nous unless you opt in. See -[`telemetry.md`](./telemetry.md) for the full feature, the `hermes telemetry` commands, -and enterprise export. - The bundled Langfuse plugin demonstrates direct hook-based observability for turns, provider requests, and tool calls. diff --git a/docs/observability/monitoring.md b/docs/observability/monitoring.md new file mode 100644 index 0000000000000..1ee9d4b5a8d98 --- /dev/null +++ b/docs/observability/monitoring.md @@ -0,0 +1,101 @@ +# Gateway Monitoring + +Service health monitoring plus redacted operational diagnostics for the +Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured +endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver). + +This plane is content-free by construction. It exports gateway lifecycle +state, platform connector health, and redacted warning/error diagnostics. +It never exports prompts, messages, tool arguments or results, session +history, usage analytics, audit logs, or execution traces. Run/model/tool +trajectory capture is a separate plane served by the NeMo Relay integration +(`plugins/observability/nemo_relay/`) and its Hermes-owned subscribers. + +## What gets exported + +| Signal | OTLP route | Content | +| --- | --- | --- | +| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes | +| Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes | +| Diagnostics | `/v1/logs` | Warning/error gateway log events with secrets AND PII scrubbed in-process before egress (`[redacted]` / `[email]`), bounded error classes | + +Every signal carries resource attributes (`service.name`, profile, version, +install id, supervision mode) so an operator can tell instances apart. + +## Enabling + +```yaml +# config.yaml +monitoring: + gateway_health_export: + enabled: true + export: + otlp: + enabled: true + endpoint: http://collector-host:4318/v1/traces # metrics/logs derive + headers_env: {} # header name -> ENV VAR NAME (values never stored) +``` + +Check the posture any time: + +```bash +hermes monitoring status +``` + +The OpenTelemetry SDK is an optional extra (`pip install 'hermes-agent[otlp]'`), +lazy-installed on first use. When the SDK is missing or the endpoint is down, +the gateway runs unaffected: every export path is fail-open and off the hot +path (events flow through a fire-and-forget in-process emitter; a slow or +failing exporter can never block gateway code). + +Works identically under systemd/launchd/s6 supervision, containers, tmux, or +a plain `hermes gateway run` — the exporter lives in the gateway process, so +no sidecar, agent, or collector is required on the host. + +## Collecting into DataDog + +Run a customer-owned OpenTelemetry Collector and forward: + +```yaml +# otel-collector config +receivers: + otlp: + protocols: + http: +exporters: + datadog: + api: + key: ${env:DD_API_KEY} +service: + pipelines: + metrics: {receivers: [otlp], exporters: [datadog]} + traces: {receivers: [otlp], exporters: [datadog]} + logs: {receivers: [otlp], exporters: [datadog]} +``` + +Point `monitoring.export.otlp.endpoint` at the collector. Alerts belong on +`hermes.gateway.up`, `hermes.platform.up`, and `hermes.platform.degraded`. + +## Local smoke test (no Docker) + +```bash +# terminal 1: capture collector on :4318 +python scripts/observability/otel_capture_collector.py \ + --host 127.0.0.1 --port 4318 --log /tmp/hermes_otel_capture.jsonl + +# terminal 2: drive the real exporter through lifecycle transitions, +# a fatal platform, and a redacted warning log, then flush +python scripts/observability/gateway_health_export_probe.py \ + --endpoint http://127.0.0.1:4318/v1/traces \ + --log /tmp/hermes_otel_capture.jsonl --wait 8 +# exit 0 prints: {"requests": 6, "paths": ["/v1/logs", "/v1/metrics", "/v1/traces"]} +``` + +## Boundaries and roadmap + +The `hermes monitoring` CLI intentionally exposes `status` only. Shared +client usage metrics and enterprise trace telemetry are being designed on +the NeMo Relay integration with their own consent, policy, and export +boundaries; this monitoring plane stays narrow so an operator can enable it +without touching any content-bearing signal. The telemetry surface may be +reorganized as that lands. diff --git a/docs/observability/telemetry.md b/docs/observability/telemetry.md deleted file mode 100644 index ea3d2f28ffa48..0000000000000 --- a/docs/observability/telemetry.md +++ /dev/null @@ -1,228 +0,0 @@ -# Telemetry & Observability - -Hermes ships with a built-in, local-first telemetry system. It records what your -agent does — workflows, model calls, tool calls, errors — to your own machine, powers -`/insights`, and (when *you* enable it) exports everything to your own observability -stack. It is private by default and never sends your data to Nous unless you explicitly -opt in to anonymous aggregate metrics. - -This page explains the whole feature: the three telemetry settings, what's captured, the -`hermes telemetry` commands, and how an enterprise streams or exports all of its data -to its own infrastructure. - -> Looking for the **plugin hook contract** (how to write your own observer plugin)? -> That's in [`README.md`](./README.md). This page is about the built-in telemetry -> system and its CLI. - -## The three settings - -Telemetry has three settings, isolated from each other: - -| Setting | What it holds | Default | Destination | -| --- | --- | --- | --- | -| **local** | Full-fidelity observability — runs, model/tool calls (real model & provider names), durations, errors | **on** | your machine only | -| **aggregate** | Opt-in metadata to Nous (no uploader ships yet) | **off** (opt-in) | Nous, only if you enable it | -| **trajectories** | Full message content / reasoning / raw tool args | **off** (opt-in) | your own export destinations only | - -Local telemetry is the one you'll use day to day — it records the real values that -happened (actual model ids, providers, tool names). Aggregate metrics are the only -thing that could ever leave for Nous; they are opt-in, default-off, and have no uploader -today. Trajectories unlock full-content export to *your own* destinations — never wired -to Nous. - -## Local telemetry — always-on observability - -Local telemetry is implemented as a bundled `telemetry` plugin that listens to Hermes -lifecycle hooks (model calls, tool calls, session start/finalize) and writes events to: - -- an append-only JSONL log at `~/.hermes/telemetry/events.jsonl` (the source of truth) -- indexed `tel_*` tables in `state.db` (a rebuildable index for fast queries): - `tel_runs`, `tel_spans`, `tel_model_calls`, `tel_tool_calls`, `tel_error_events` - -Writes are fire-and-forget on a background thread: telemetry can never block, slow, or -fail a model call or tool call. If local telemetry is disabled (`telemetry.local: false`) -the plugin does not load at all. - -### Traces and spans - -A **run** is one session (from `on_session_start` to `on_session_finalize`). Each run gets -a root span in `tel_spans`, and every model and tool call within it is recorded as a child -span (timing + `parent_span_id` = the run's root) keyed by the same `span_id` as its -detail row in `tel_model_calls` / `tel_tool_calls`. So a run reconstructs as a connected -`run -> calls` tree, ordered by `start_ns` and joinable to the per-call detail — the shape -a trace viewer or any OpenTelemetry backend can render. - -Call spans are timed from the measured latency/duration the hooks report; cross-run -subagent lineage (linking a delegated child run to its parent) is not yet recorded. - -### Seeing your local data - -```bash -hermes insights # usage report — now includes an "Observability" section -hermes telemetry status # settings, consent, export posture, local data volume -``` - -The `insights` Observability section shows workflow counts and success rate, duration -p50/p95, tool failure rates by tool, provider/model mix, and cache hit rate — -all computed locally with exact values. - -## `hermes telemetry` commands - -```text -hermes telemetry status Show settings, consent state, export posture, local volume -hermes telemetry preview Show the aggregate events that would be produced (local) -hermes telemetry export Export local telemetry to a file/stream or OTLP endpoint -``` - -Consent and the install id are plain config, not separate verbs — set them with -`hermes config set` (or a managed-scope pin): - -```bash -hermes config set telemetry.consent_state aggregate # opt in to aggregate metrics -hermes config set telemetry.consent_state local # opt out (local telemetry stays on) -hermes config set telemetry.install_id "" # reset the install id (mints a new one) -``` - -### Aggregate metrics (opt-in) - -Aggregate metrics are **off by default** and have **no uploader today** — nothing is -sent to Nous. Consent lives in `telemetry.consent_state` (`unknown` / `local` / -`aggregate`); setting it to `aggregate` records the opt-in for if/when an uploader ships. -If one is built, it would summarize at that egress boundary. - -`hermes telemetry preview` shows your recent runs as they'd be summarized — computed and -shown **locally only**, with the real model and tool names from your own telemetry. It's -a local inspection surface, not an upload. - -## Enterprise: getting all of your data - -Everything below sends data to **your own** destination — a file, your SIEM, or your own -OpenTelemetry Collector. None of it goes to Nous. - -### Bulk export to a file - -```bash -# Structural telemetry only (default — no message content) -hermes telemetry export --out telemetry.ndjson - -# JSON instead of NDJSON, last 7 days only -hermes telemetry export --out dump.json --format json --since 7 -``` - -By default the export is **structural** — runs, model/tool-call metadata, session shells -with message *counts* but no message bodies. - -### Including content (trajectories) - -To export full message content, enable trajectories. This is a deliberate, separate -consent — it's how an enterprise opts into exporting work-product content to its own -store: - -```yaml -# config.yaml -telemetry: - trajectories: - enabled: true # unlocks content export to YOUR destination - content_redaction: pii # "none" | "pii" -``` - -```bash -hermes telemetry export --out full.ndjson --include-content -``` - -`--include-content` is a no-op unless trajectories are enabled — the config setting -governs, not the flag. - -### Live streaming to your OpenTelemetry Collector / SIEM (OTLP) - -Hermes can stream telemetry to your own OTLP endpoint. This requires the optional `otlp` -extra: - -```bash -pip install 'hermes-agent[otlp]' -``` - -```yaml -# config.yaml -telemetry: - export: - otlp: - enabled: true - endpoint: "https://collector.your-corp.internal:4318/v1/traces" - headers_env: # secrets by reference — env var NAMES, not values - Authorization: MY_OTLP_TOKEN_ENVVAR -``` - -Set the referenced environment variable, then run the export: - -```bash -hermes telemetry export --otlp # drain current telemetry to your collector -``` - -The token value lives only in the environment variable named by `headers_env`. The -config holds the *name* of an environment variable rather than the secret itself; the -value is read at export time and is never written to config or logged. - -Each telemetry event is exported as an OTel span carrying its recorded attributes -(provider, model, tokens, duration, etc.). The `tel_spans` timing/parent linkage is not -yet reconstructed into connected OTel `SpanContext`s, so spans currently arrive as -independent records rather than a connected trace tree; that projection is planned. - -## Redaction - -Two independent controls govern what content looks like on export: - -| Control | Values | Effect | -| --- | --- | --- | -| Secret redaction | always on | API keys, tokens, auth headers, connection strings are **always** stripped on every export path. Cannot be disabled. | -| `content_redaction` | `none` \| `pii` | When content is exported, `pii` additionally redacts emails, phone numbers, and id-shaped strings. | - -Secret redaction is always on — even at full content fidelity — because a SIEM or -warehouse full of live credentials is a bigger attack target than the data it holds. It -fails closed: if the redactor can't run, the raw string is not emitted. - -## Configuration reference - -```yaml -telemetry: - local: true # local telemetry (default on) - allow_aggregate: true # hard gate; pin false to forbid aggregate metrics entirely - consent_state: unknown # aggregate opt-in: unknown | local | aggregate - install_id: "" # stable anon id; "" mints one; clear to rotate - retention_days: 90 # local event-log retention - redact_secrets: true # always-on secret redaction (kept on by design) - content_redaction: none # none | pii - trajectories: - enabled: false # unlocks full-content export to your destination - export: - otlp: - enabled: false - endpoint: null - headers_env: {} # {HeaderName: ENV_VAR_NAME} -``` - -### Enterprise policy via managed scope - -Any `telemetry.*` key can be pinned by an administrator through Hermes' managed-scope -layer (`/etc/hermes/config.yaml`), which wins over the user's value on a per-key basis. -There is no telemetry-specific policy block — to lock down a fleet, pin the keys you care -about. Common examples: - -- `telemetry.allow_aggregate: false` — aggregate metrics stay off even if - `consent_state` is set to `aggregate`. -- `telemetry.export.otlp.endpoint` — point every install at the corporate collector. -- `telemetry.trajectories.enabled` — centrally decide whether content export is allowed. - -When a key is managed, attempts to change it are rejected by managed scope with a message -naming the source. `hermes telemetry status` shows the current export posture (endpoint -host, whether the auth env var is set, content gate, redaction modes) — it never prints -secret values. - -## Privacy summary - -- Local telemetry never leaves your machine. -- Aggregate metrics (the only thing that could go to Nous) are opt-in, default-off, - and have no uploader today — nothing is sent. -- All export surfaces (file, OTLP) point at *your* destinations. -- Secrets are always redacted on export; content export is off until you enable - trajectories; PII redaction is a knob. diff --git a/gateway/run.py b/gateway/run.py index 72f2b2626adad..07d13b21c2e11 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -7810,6 +7810,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew write_runtime_status(gateway_state="starting", exit_reason=None) except Exception: pass + try: + from hermes_cli.config import load_config + from agent.monitoring.gateway_health_export import start_gateway_health_export + self._gateway_health_export_runtime = start_gateway_health_export(load_config()) + if getattr(self._gateway_health_export_runtime, "enabled", False): + logger.info("Gateway health OTLP export: enabled") + except Exception: + logger.debug("gateway health OTLP export startup failed", exc_info=True) # Log any active supply-chain security advisories. Operators see this # in gateway.log and `hermes status` surfaces it; we do NOT block @@ -9762,6 +9770,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._update_runtime_status("running", self._exit_reason) else: self._update_runtime_status("stopped", self._exit_reason) + try: + _gh_runtime = getattr(self, "_gateway_health_export_runtime", None) + if _gh_runtime is not None: + _gh_runtime.shutdown() + except Exception: + logger.debug("gateway health OTLP export shutdown failed", exc_info=True) logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed()) self._stop_task = asyncio.create_task(_stop_impl()) diff --git a/gateway/status.py b/gateway/status.py index d2d3c15091881..d6ad5a64c6317 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -11,6 +11,7 @@ that will be useful when we add named profiles (multiple agents running concurrently under distinct configurations). """ +import copy import hashlib import json import logging @@ -987,6 +988,7 @@ def write_runtime_status( """Persist gateway runtime health information for diagnostics/status.""" path = _get_runtime_status_path() payload = _read_json_file(path) or _build_runtime_status_record() + previous_payload = copy.deepcopy(payload) current_record = _build_pid_record() payload.setdefault("platforms", {}) payload["kind"] = current_record["kind"] @@ -1021,6 +1023,11 @@ def write_runtime_status( payload["platforms"][platform] = platform_payload _write_json_file(path, payload) + try: + from agent.monitoring.gateway_health import emit_runtime_status_transition + emit_runtime_status_transition(previous_payload, payload) + except Exception: + pass def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 22290ca9ef399..f0f6a43c3155a 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -3082,6 +3082,49 @@ DEFAULT_CONFIG = { "force_ipv4": False, }, + # Gateway monitoring — Service Health Monitoring plus redacted Operational + # Diagnostics for the gateway daemon, exported over OTLP to an + # operator-configured endpoint (OTEL Collector, DataDog, ...). Content-free + # by construction: no prompts, messages, tool args/results, session + # history, usage analytics, audit logs, or trajectories. Off by default; + # nothing is collected or sent until an operator enables it and sets an + # endpoint. + "monitoring": { + # Stable install identifier attached to exported health signals so an + # operator can tell instances apart in their collector. Empty string + # means "mint a fresh UUID on first use"; clear it to rotate. Carries + # no account identity. + "install_id": "", + # Gateway health & diagnostics export. + "gateway_health_export": { + "enabled": False, + "metrics_enabled": True, + "diagnostic_events_enabled": True, + "warning_error_events_enabled": True, + "export_interval_seconds": 60, + "logs_export_interval_seconds": 5, + "resource_attributes": { + "service.name": "hermes-gateway", + "deployment.environment": "production", + }, + "redaction": { + "enabled": True, + "include_stack_summary": True, + "include_raw_stack": False, + }, + }, + # OTLP destination. headers_env maps header names to ENVIRONMENT + # VARIABLE NAMES (never secret values); values are read from the + # environment at export time. + "export": { + "otlp": { + "enabled": False, + "endpoint": "", + "headers_env": {}, + }, + }, + }, + # Gateway settings — control how messaging platforms (Telegram, Discord, # Slack, etc.) deliver agent-produced files as native attachments. "gateway": { diff --git a/hermes_cli/main.py b/hermes_cli/main.py index abd4c462dd3d9..2cef49a220a69 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -463,7 +463,7 @@ from hermes_cli.subcommands.memory import build_memory_parser from hermes_cli.subcommands.acp import build_acp_parser from hermes_cli.subcommands.tools import build_tools_parser from hermes_cli.subcommands.insights import build_insights_parser -from hermes_cli.subcommands.telemetry import build_telemetry_parser +from hermes_cli.subcommands.monitoring import build_monitoring_parser from hermes_cli.subcommands.skills import build_skills_parser from hermes_cli.subcommands.pairing import build_pairing_parser from hermes_cli.subcommands.plugins import build_plugins_parser @@ -13859,11 +13859,11 @@ _BUILTIN_SUBCOMMANDS = frozenset( "dump", "egress", "fallback", "gateway", "hooks", "import", "insights", "gui", "desktop", "kanban", "login", "logout", "logs", "lsp", "mcp", "memory", "migrate", "moa", "journey", "memory-graph", "learning", - "model", "pairing", "pets", "plugins", "portal", "profile", + "model", "monitoring", "pairing", "pets", "plugins", "portal", "profile", "project", "proxy", "prompt-size", "send", "sessions", "setup", - "skin", "skills", "slack", "status", "telemetry", "tools", "uninstall", "update", + "skin", "skills", "slack", "status", "tools", "uninstall", "update", "version", "webhook", "whatsapp", "whatsapp-cloud", "chat", "secrets", "security", # Help-ish invocations — plugin commands not being listed in # top-level --help is an acceptable trade-off for skipping an @@ -14312,193 +14312,51 @@ def cmd_insights(args): print(f"Error generating insights: {e}") -def cmd_telemetry(args): - """Local-only telemetry control + inspection. No uploader exists yet.""" - import json as _json - import time +def cmd_monitoring(args): + """Gateway monitoring status: health & diagnostics export posture.""" + from hermes_cli.config import load_config - from hermes_cli.config import load_config, save_config - from agent.telemetry import policy, rollup, metrics - - action = getattr(args, "telemetry_action", None) or "status" + action = getattr(args, "monitoring_action", None) or "status" config = load_config() - tel = config.get("telemetry") if isinstance(config.get("telemetry"), dict) else {} - local_enabled = bool(tel.get("local", True)) - allow_aggregate = bool(tel.get("allow_aggregate", True)) - consent_state = tel.get("consent_state", policy.CONSENT_UNKNOWN) - if consent_state not in policy.VALID_CONSENT_STATES: - consent_state = policy.CONSENT_UNKNOWN - aggregate_enabled = (local_enabled and allow_aggregate - and consent_state == policy.CONSENT_AGGREGATE) - install_id = policy.ensure_install_id(config) - - def _persist_install_id(): - # Make sure a minted id is written back so it stays stable. - config.setdefault("telemetry", {})["install_id"] = install_id - save_config(config) + mon_raw = config.get("monitoring") + mon: dict = mon_raw if isinstance(mon_raw, dict) else {} if action == "status": - print("Telemetry status") - print("─" * 56) - print(f" Local telemetry: {'on' if local_enabled else 'off'} " - f"(telemetry.local)") - print(f" Aggregate metrics: {'on' if aggregate_enabled else 'off'} " - f"(opt-in; consent_state={consent_state})") - if consent_state == policy.CONSENT_AGGREGATE and not local_enabled: - print(" ⚠ inert: local telemetry is off — nothing to aggregate") - elif consent_state != policy.CONSENT_AGGREGATE and allow_aggregate: - print(" opt in: hermes config set telemetry.consent_state aggregate") - if not allow_aggregate: - print(" ⚠ allow_aggregate is false (egress hard-disabled)") - print(f" Install id: {install_id}") - print(" Upload: DISABLED — no server yet. Aggregate metrics are " - "computed locally only.") - print() - # Export posture — where YOUR data can flow (never Nous). Values only; - # lock-state is managed scope's concern, surfaced by its own tooling. - try: - from agent.telemetry import otlp_exporter, redaction - otlp = otlp_exporter._otlp_config(config) - print(" Export") - if otlp.get("enabled") and otlp.get("endpoint"): - _host = str(otlp.get("endpoint")) - print(f" OTLP: enabled → {_host}") - # Show the header name + env var + whether it's set, never the value. - hdrs = otlp.get("headers_env") or {} - if hdrs: - import os as _os - for _h, _env in hdrs.items(): - _state = "set" if _os.environ.get(str(_env)) else "NOT set" - print(f" auth: header '{_h}' ← ${_env} ({_state})") - _sdk = "installed" if otlp_exporter.is_available() else "MISSING — pip install 'hermes-agent[otlp]'" - print(f" SDK: {_sdk}") - else: - print(" OTLP: disabled (telemetry.export.otlp.enabled)") - # Content gate (telemetry.trajectories) + redaction posture. - if redaction.content_export_enabled(config): - print(f" Content export: on (trajectories enabled) — message " - f"content exportable") - else: - print(" Content export: off (trajectories disabled) — structural " - "telemetry only") - print(" Secret redaction: on (always)") - print(f" PII redaction: {redaction.content_mode_for(config)}") - print(" Bulk export: hermes telemetry export --out FILE [--otlp]") - except Exception: - pass - print() - # Local data volume - try: - ov = metrics.overview() - runs = ov["workflows"]["total_runs"] - mcs = sum(ov["model_calls"]["by_provider"].values()) - tcs = ov["tool_calls"]["total"] - print(f" Local data: {runs:,} workflows · {mcs:,} model calls · {tcs:,} tool calls") - except Exception: - print(" Local data: (none yet)") - print("\n Inspect what would be shared: hermes telemetry preview") - return + from agent.monitoring import otlp_exporter - if action == "preview": - _persist_install_id() - since_ns = int((time.time() - args.days * 86400) * 1e9) - events = rollup.build_aggregate_events( - install_id=install_id, since_ns=since_ns - ) - summary = rollup.summarize(events) - print("Telemetry preview — computed locally, NOT uploaded") - print("─" * 56) - print(f" Window: last {args.days} days Events: {summary['total']}") - for name, n in sorted(summary["by_event_name"].items()): - print(f" {name}: {n}") - print(" Shows the actual models, tools, and counts from local telemetry.") - print() - shown = events[: args.limit] - if getattr(args, "json", False): - print(_json.dumps(shown, indent=2)) + gh_raw = mon.get("gateway_health_export") + gh: dict = gh_raw if isinstance(gh_raw, dict) else {} + export_raw = mon.get("export") + export_cfg: dict = export_raw if isinstance(export_raw, dict) else {} + otlp_raw = export_cfg.get("otlp") + otlp: dict = otlp_raw if isinstance(otlp_raw, dict) else {} + + print("Gateway monitoring") + print(f" Health export: {'enabled' if gh.get('enabled') else 'disabled'} " + f"(monitoring.gateway_health_export.enabled)") + if gh.get("enabled"): + print(f" Metrics: {'on' if gh.get('metrics_enabled', True) else 'off'} " + f"(interval {gh.get('export_interval_seconds', 60)}s)") + print(f" Diagnostic events: {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}") + print(f" Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} " + f"(interval {gh.get('logs_export_interval_seconds', 5)}s)") + red_raw = gh.get("redaction") + red: dict = red_raw if isinstance(red_raw, dict) else {} + print(f" Redaction: {'on' if red.get('enabled', True) else 'OFF'} " + f"(secrets/PII scrubbed in-process before egress)") + endpoint = otlp.get("endpoint") or "" + if otlp.get("enabled") and endpoint: + print(f" OTLP endpoint: {endpoint}") else: - for e in shown: - if e.get("event_name") == "heartbeat": - print(f" • heartbeat hermes={e.get('hermes_version')} os={e.get('os_family')}") - continue - bits = [f"{e.get('event_name')}"] - for k in ("entrypoint", "platform", "end_reason", "duration_ms"): - if e.get(k) is not None: - bits.append(f"{k}={e[k]}") - models = e.get("models_used") or [] - if models: - bits.append("models=" + ",".join( - f"{m.get('model') or m.get('provider') or '?'}" for m in models)) - if e.get("tools_used"): - bits.append("tools=" + ",".join(e["tools_used"])) - print(" • " + " ".join(bits)) - if len(events) > len(shown): - print(f" ... and {len(events) - len(shown)} more (use --limit)") + print(" OTLP endpoint: not configured (monitoring.export.otlp)") + print(f" OTel SDK: {'installed' if otlp_exporter.is_available() else 'not installed'} " + f"(optional extra: hermes-agent[otlp])") + print("\n Scope: gateway service health + redacted diagnostics only.") + print(" No prompts, messages, tool args/results, usage analytics, or traces.") return - if action == "export": - import sys as _sys - from agent.telemetry import exporter_bulk, redaction - - since_ns = int((time.time() - args.since * 86400) * 1e9) if getattr(args, "since", 0) else None - want_content = getattr(args, "include_content", False) - - # OTLP path: stream spans to the configured Collector endpoint. - if getattr(args, "otlp", False): - from agent.telemetry import otlp_exporter - if not otlp_exporter.is_enabled(config): - print("telemetry.export.otlp is not enabled/endpoint not set. " - "Set telemetry.export.otlp.enabled + .endpoint.", file=_sys.stderr) - return - # The OTel SDK is an optional dep; export_once() lazily installs it on - # first use (gated by security.allow_lazy_installs, TTY-prompted), - # same as every other optional backend. OTLPUnavailable is the - # fallback when it can't be installed. - try: - n = otlp_exporter.export_once(config, since_ns=since_ns) - except otlp_exporter.OTLPUnavailable as e: - print(str(e), file=_sys.stderr) - return - except Exception as e: - print(f"OTLP export failed: {e}", file=_sys.stderr) - return - print(f"Exported {n} spans to the OTLP endpoint " - f"({(otlp_exporter._otlp_config(config) or {}).get('endpoint')}).", - file=_sys.stderr) - return - - content_ok = redaction.content_export_enabled(config) - if want_content and not content_ok: - print("⚠ --include-content ignored: telemetry.trajectories.enabled is false.") - print(" Enable trajectories (admin/config) to export message content.") - print(" Exporting structural telemetry only.") - - if not getattr(args, "out", None): - print("--out is required (or use --otlp).", file=_sys.stderr) - return - out_path = args.out - if out_path == "-": - counts = exporter_bulk.export( - _sys.stdout, fmt=args.fmt, since_ns=since_ns, - include_content=want_content, config=config, - ) - else: - with open(out_path, "w", encoding="utf-8") as fh: - counts = exporter_bulk.export( - fh, fmt=args.fmt, since_ns=since_ns, - include_content=want_content, config=config, - ) - # status to stderr so stdout stays clean for `--out -` - msg = (f"Exported {counts['telemetry']} telemetry records" - + (f" + {counts['sessions']} sessions" if counts["sessions"] else "") - + (" (content INCLUDED, redacted)" if counts["content_included"] - else " (structural only)") - + (f" -> {out_path}" if out_path != "-" else "")) - print(msg, file=_sys.stderr) - return - - print(f"Unknown telemetry action: {action}") - print("Use: status | preview | export") + print(f"Unknown monitoring action: {action}", file=sys.stderr) + sys.exit(2) def cmd_skills(args): @@ -16447,9 +16305,7 @@ def main(): # insights command (parser built in hermes_cli/subcommands/insights.py) # ========================================================================= build_insights_parser(subparsers, cmd_insights=cmd_insights) - - # telemetry command (parser in hermes_cli/subcommands/telemetry.py) - build_telemetry_parser(subparsers, cmd_telemetry=cmd_telemetry) + build_monitoring_parser(subparsers, cmd_monitoring=cmd_monitoring) # ========================================================================= # claw command (parser built in hermes_cli/subcommands/claw.py) diff --git a/hermes_cli/subcommands/monitoring.py b/hermes_cli/subcommands/monitoring.py new file mode 100644 index 0000000000000..266c69aa1fb3f --- /dev/null +++ b/hermes_cli/subcommands/monitoring.py @@ -0,0 +1,36 @@ +"""``hermes monitoring`` subcommand parser. + +Gateway monitoring control and inspection. ``status`` shows whether the +gateway health & diagnostics export is enabled, where it points, and the +redaction posture. + +The handler is injected to avoid importing ``main`` (mirrors the insights +subcommand). +""" + +from __future__ import annotations + +from typing import Callable + + +def build_monitoring_parser(subparsers, *, cmd_monitoring: Callable) -> None: + """Attach the ``monitoring`` subcommand (with actions) to ``subparsers``.""" + p = subparsers.add_parser( + "monitoring", + help="Inspect gateway monitoring (health & diagnostics export)", + description=( + "Gateway monitoring: service health metrics plus redacted " + "diagnostics, exported over OTLP to an operator-configured " + "endpoint. Content-free by construction — no prompts, messages, " + "tool args/results, or usage analytics. Configure under " + "monitoring.* in config.yaml." + ), + ) + sub = p.add_subparsers(dest="monitoring_action") + + sub.add_parser( + "status", + help="Show monitoring settings, export state, and redaction posture", + ) + + p.set_defaults(func=cmd_monitoring) diff --git a/hermes_cli/subcommands/telemetry.py b/hermes_cli/subcommands/telemetry.py deleted file mode 100644 index 791cdeb05b466..0000000000000 --- a/hermes_cli/subcommands/telemetry.py +++ /dev/null @@ -1,54 +0,0 @@ -"""``hermes telemetry`` subcommand parser. - -Telemetry control and inspection. ``preview`` shows the per-run summary events that -would be produced for aggregate metrics; there is no uploader, so it terminates as a -local view. - -The handler is injected to avoid importing ``main`` (mirrors the insights subcommand). -""" - -from __future__ import annotations - -from typing import Callable - - -def build_telemetry_parser(subparsers, *, cmd_telemetry: Callable) -> None: - """Attach the ``telemetry`` subcommand (with actions) to ``subparsers``.""" - p = subparsers.add_parser( - "telemetry", - help="Inspect local telemetry and export it", - description=( - "Local-first telemetry. Local telemetry records observability on this " - "machine. Aggregate metrics are opt-in (set telemetry.consent_state via " - "`hermes config set`); they have no uploader and are shown only via `preview`." - ), - ) - sub = p.add_subparsers(dest="telemetry_action") - - sub.add_parser("status", help="Show telemetry settings, consent state, and local data volume") - - prev = sub.add_parser( - "preview", - help="Show the aggregate events that would be produced (computed locally, not uploaded)", - ) - prev.add_argument("--days", type=int, default=30, help="Window to roll up (default: 30)") - prev.add_argument("--limit", type=int, default=10, help="Max events to print (default: 10)") - prev.add_argument("--json", action="store_true", help="Print raw JSON events") - - exp = sub.add_parser( - "export", - help="Export local telemetry (and optional content) to a file, stream, or OTLP endpoint", - ) - exp.add_argument("--out", help="Output file path (use - for stdout). Not needed with --otlp.") - exp.add_argument("--format", dest="fmt", choices=["ndjson", "json"], default="ndjson", - help="Output format (default: ndjson)") - exp.add_argument("--since", type=int, default=0, - help="Only telemetry from the last N days (0 = all)") - exp.add_argument("--include-content", action="store_true", - help="Include session/message content (requires telemetry.trajectories.enabled). " - "Secrets always redacted; PII per telemetry.content_redaction.") - exp.add_argument("--otlp", action="store_true", - help="Export to the configured OTLP endpoint (telemetry.export.otlp.*) " - "instead of a file. Requires the optional 'otlp' extra.") - - p.set_defaults(func=cmd_telemetry) diff --git a/plugins/telemetry/__init__.py b/plugins/telemetry/__init__.py deleted file mode 100644 index c270d4d72a257..0000000000000 --- a/plugins/telemetry/__init__.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Telemetry plugin — wires Hermes lifecycle hooks to the local telemetry emitter. - -This is the *only* instrumentation seam. It registers observational hooks (which core -already invokes fail-open) and translates each into a typed local telemetry event -handed to ``agent.telemetry.emitter``. There are zero edits to core call sites: -the hooks already carry model/provider/usage/duration/tool data. - -Everything here is best-effort and fail-open — a raised exception in a hook callback is -swallowed by core, and we additionally guard each callback so a telemetry bug can never -disturb a session. No content, no network: local telemetry only. - -Hooks consumed: - on_session_start -> begin a run context (trace_id/run_id + root span id) - post_api_request -> one model_call event + its timing span (tokens, latency) - api_request_error -> one error event - post_tool_call -> one tool_call event + its timing span (duration, result class) - on_session_finalize -> finalize the run row + emit the run's root span - subagent_start/stop -> (reserved) lineage markers - -Each model/tool call emits a SpanEvent (timing + parent = the run's root span) keyed by -the same span_id as its detail row, so tel_spans reconstructs a run -> calls trace tree. -""" - -from __future__ import annotations - -import logging -import threading -import time -from typing import Any, Dict, Optional - -logger = logging.getLogger(__name__) - -# Per-run accumulators keyed by run_id, so on_session_finalize can roll up counts. -_runs: Dict[str, Dict[str, Any]] = {} -_runs_lock = threading.Lock() - - -def _safe(fn): - """Decorator: never let a telemetry hook raise into core.""" - def wrapper(*args, **kwargs): - try: - return fn(*args, **kwargs) - except Exception: - logger.debug("telemetry hook %s failed", getattr(fn, "__name__", "?"), exc_info=True) - return None - wrapper.__name__ = getattr(fn, "__name__", "wrapper") - return wrapper - - -def _run_key(session_id: Optional[str], task_id: Optional[str]) -> str: - return session_id or task_id or "default" - - -# ── on_session_start ──────────────────────────────────────────────────────── -@_safe -def _on_session_start(**kw: Any) -> None: - from agent.telemetry import spans - - session_id = kw.get("session_id") or "" - platform = kw.get("platform") or kw.get("source") or "" - ctx = spans.start_run() - now = time.time_ns() - root_span_id = spans.new_span_id() - key = _run_key(session_id, kw.get("task_id")) - with _runs_lock: - _runs[key] = { - "run_id": ctx.run_id, - "trace_id": ctx.trace_id, - "root_span_id": root_span_id, - "session_id": session_id or None, - "entrypoint": _entrypoint_for(platform, kw.get("source")), - "platform": platform or None, - "start_ns": now, - "model_call_count": 0, - "tool_call_count": 0, - "error_count": 0, - } - - -def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str = "") -> Dict[str, Any]: - """Return the run accumulator, lazily creating one if session_start was missed.""" - from agent.telemetry import spans - - key = _run_key(session_id, task_id) - with _runs_lock: - run = _runs.get(key) - if run is None: - rid = spans.current_run_id() or spans.new_id() - tid = spans.current_trace_id() or spans.new_id() - run = { - "run_id": rid, - "trace_id": tid, - "root_span_id": spans.new_span_id(), - "session_id": session_id or None, - "entrypoint": _entrypoint_for(platform), - "platform": platform or None, - "start_ns": time.time_ns(), - "model_call_count": 0, - "tool_call_count": 0, - "error_count": 0, - } - _runs[key] = run - return run - - -def _emit_call_span(run: Dict[str, Any], span_id: str, name: str, kind: str, - duration_ms: Optional[int], status: Optional[str]) -> None: - """Emit the timing/lineage span for a model or tool call. - - The call hooks fire on completion, so end_ns is ~now and start_ns is reconstructed - from the measured duration (end - duration). The span is parented to the run's root - so a 2-level run -> calls waterfall can be reconstructed from tel_spans. - """ - from agent.telemetry import emitter - from agent.telemetry.events import SpanEvent - - end_ns = time.time_ns() - dur_ns = int(duration_ms) * 1_000_000 if isinstance(duration_ms, (int, float)) else 0 - start_ns = end_ns - dur_ns - emitter.emit(SpanEvent( - span_id=span_id, - trace_id=run["trace_id"], - run_id=run["run_id"], - parent_span_id=run.get("root_span_id"), - name=name, - kind=kind, - start_ns=start_ns, - end_ns=end_ns, - status=status, - )) - - -def _entrypoint_for(platform: Optional[str], source: Optional[str] = None) -> str: - """Coarse entrypoint label (cli / gateway / tui / api / cron …). - - This is a workflow *surface* label, not model/tool anonymization — it answers - "where did this run come from", which is genuinely categorical. - """ - s = (source or platform or "").lower() - if s in ("", "chat", "interactive", "cli", "desktop"): - return "cli" - if s in ("telegram", "discord", "slack", "whatsapp", "signal", "matrix", - "email", "sms", "teams", "feishu", "wecom", "line", "google_chat"): - return "gateway" - if s == "tui": - return "tui" - if s in ("api", "api_server", "openai_api"): - return "api" - if s == "cron": - return "cron" - if s == "batch": - return "batch" - if s == "acp": - return "acp" - return "cli" - - -# ── post_api_request -> model_call event ──────────────────────────────────── -@_safe -def _on_post_api_request(**kw: Any) -> None: - from agent.telemetry import emitter, spans - from agent.telemetry.events import ModelCallEvent - - session_id = kw.get("session_id") or "" - platform = kw.get("platform") or "" - run = _ensure_run(session_id, kw.get("task_id"), platform) - - usage = kw.get("usage") or {} - duration = kw.get("api_duration") - latency_ms = int(duration * 1000) if isinstance(duration, (int, float)) else None - - span_id = spans.new_span_id() - evt = ModelCallEvent( - span_id=span_id, - run_id=run["run_id"], - provider=kw.get("provider"), # raw - model=kw.get("model"), # raw - base_url=kw.get("base_url"), - input_tokens=int(usage.get("input_tokens") or 0), - output_tokens=int(usage.get("output_tokens") or 0), - cache_read_tokens=int(usage.get("cache_read_tokens") or 0), - cache_write_tokens=int(usage.get("cache_write_tokens") or 0), - reasoning_tokens=int(usage.get("reasoning_tokens") or 0), - latency_ms=latency_ms, - ) - with _runs_lock: - run["model_call_count"] += 1 - _emit_call_span(run, span_id, name=kw.get("model") or "model_call", - kind="model", duration_ms=latency_ms, status="ok") - emitter.emit(evt) - - -# ── api_request_error -> error event ──────────────────────────────────────── -@_safe -def _on_api_request_error(**kw: Any) -> None: - from agent.telemetry import emitter - from agent.telemetry.events import ErrorEvent - - session_id = kw.get("session_id") or "" - run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") - error_class = _coarse_error_class(kw.get("error_type") or kw.get("error") or "") - with _runs_lock: - run["error_count"] += 1 - emitter.emit(ErrorEvent( - run_id=run["run_id"], - error_class=error_class, - subsystem="model_api", - recovery=None, - )) - - -def _coarse_error_class(raw: Any) -> str: - s = str(raw).lower() - if "timeout" in s: - return "provider_timeout" - if "rate" in s and "limit" in s: - return "rate_limit" - if any(k in s for k in ("auth", "401", "403", "unauthorized", "forbidden")): - return "auth" - if any(k in s for k in ("connection", "network", "dns", "socket")): - return "network" - if "context" in s and ("length" in s or "overflow" in s or "token" in s): - return "context_overflow" - if any(k in s for k in ("500", "502", "503", "server error", "provider")): - return "provider_error" - return "unknown" - - -# ── post_tool_call -> tool_call event ─────────────────────────────────────── -@_safe -def _on_post_tool_call(**kw: Any) -> None: - from agent.telemetry import emitter, spans - from agent.telemetry.events import ToolCallEvent - - session_id = kw.get("session_id") or "" - run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") - - function_name = kw.get("function_name") or kw.get("tool_name") - duration_ms = kw.get("duration_ms") - result = kw.get("result") - result_class = _tool_result_class(result) - - with _runs_lock: - run["tool_call_count"] += 1 - if result_class == "error": - run["error_count"] += 1 - - dur_int = int(duration_ms) if isinstance(duration_ms, (int, float)) else None - span_id = spans.new_span_id() - _emit_call_span(run, span_id, name=function_name or "tool_call", - kind="tool", duration_ms=dur_int, status=result_class) - emitter.emit(ToolCallEvent( - span_id=span_id, - run_id=run["run_id"], - tool_name=function_name, # raw tool name - duration_ms=dur_int, - result_class=result_class, - )) - - -def _tool_result_class(result: Any) -> str: - """Classify a tool result without retaining content — error vs ok vs blocked.""" - try: - import json - if isinstance(result, str): - r = result.strip() - if r.startswith("{"): - obj = json.loads(r) - if isinstance(obj, dict): - if obj.get("error") or obj.get("blocked"): - return "blocked" if obj.get("blocked") else "error" - if obj.get("timeout"): - return "timeout" - return "ok" - if isinstance(result, dict): - if result.get("error"): - return "error" - return "ok" - except Exception: - return "ok" - return "ok" - - -# ── on_session_finalize -> finalize the run row ───────────────────────────── -@_safe -def _on_session_finalize(**kw: Any) -> None: - from agent.telemetry import emitter, spans - from agent.telemetry.events import RunEvent, SpanEvent - - session_id = kw.get("session_id") or "" - key = _run_key(session_id, kw.get("task_id")) - with _runs_lock: - run = _runs.pop(key, None) - if run is None: - run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "") - with _runs_lock: - _runs.pop(key, None) - - end_ns = time.time_ns() - start_ns = run.get("start_ns", end_ns) - end_reason = _coarse_end_reason(kw) - # Root span for the run — the trace root the call spans hang off of. - emitter.emit(SpanEvent( - span_id=run.get("root_span_id") or spans.new_span_id(), - trace_id=run["trace_id"], - run_id=run["run_id"], - parent_span_id=None, - name=f"run:{run.get('entrypoint', 'cli')}", - kind="run", - start_ns=start_ns, - end_ns=end_ns, - status=end_reason, - )) - emitter.emit(RunEvent( - run_id=run["run_id"], - trace_id=run["trace_id"], - entrypoint=run.get("entrypoint", "cli"), - session_id=run.get("session_id"), - platform=run.get("platform"), - start_ns=start_ns, - end_ns=end_ns, - end_reason=end_reason, - model_call_count=run.get("model_call_count", 0), - tool_call_count=run.get("tool_call_count", 0), - error_count=run.get("error_count", 0), - )) - spans.clear_run() - - -def _coarse_end_reason(kw: Dict[str, Any]) -> str: - """Map a finalize payload to a coarse end reason. - - Production finalize callers pass ``reason`` (e.g. "shutdown", "session_expired", - "session_reset"); the older ``turn_exit_reason``/``interrupted``/``failed`` keys are - honored when present. Defaults to "ended". - """ - if kw.get("interrupted"): - return "interrupted" - if kw.get("failed"): - return "failed" - reason = str(kw.get("turn_exit_reason") or kw.get("reason") or "").lower() - if "max_iteration" in reason: - return "max_iterations" - if "timeout" in reason: - return "timeout" - if "expired" in reason: - return "expired" - if "reset" in reason: - return "reset" - if "shutdown" in reason or "complete" in reason: - return "completed" - return reason or "ended" - - -# ── subagent lineage (reserved) ───────────────────────────────────────────── -# A delegated subagent runs its own ``run_conversation`` with its own session id, so -# its model/tool calls are already captured as a separate tel_runs row via the normal -# hooks — no subagent activity is lost. These hooks fire with the parent<->child bridge -# (parent_session_id, child_session_id, child_role, child_goal); they are reserved for -# recording parent->child *lineage* (linking a child run back to its parent), which -# needs a tel_runs.parent_run_id column. Deferred until a consumer needs the delegation -# tree; left registered as the attachment point. -@_safe -def _on_subagent_start(**kw: Any) -> None: - return None - - -@_safe -def _on_subagent_stop(**kw: Any) -> None: - return None - - -# ── registration ──────────────────────────────────────────────────────────── -def register(ctx) -> None: - ctx.register_hook("on_session_start", _on_session_start) - ctx.register_hook("post_api_request", _on_post_api_request) - ctx.register_hook("api_request_error", _on_api_request_error) - ctx.register_hook("post_tool_call", _on_post_tool_call) - ctx.register_hook("on_session_finalize", _on_session_finalize) - ctx.register_hook("subagent_start", _on_subagent_start) - ctx.register_hook("subagent_stop", _on_subagent_stop) - logger.debug("telemetry plugin registered 7 hooks") diff --git a/plugins/telemetry/plugin.yaml b/plugins/telemetry/plugin.yaml deleted file mode 100644 index 967d79616b7c2..0000000000000 --- a/plugins/telemetry/plugin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: telemetry -version: 1.0.0 -description: "Local-first telemetry & observability. Records runs, model calls, tool calls, and errors to a local event log + state.db index via plugin hooks — no agent action, no content, no network. Powers /usage, /insights, and local dashboards." -author: NousResearch -hooks: - - post_api_request - - api_request_error - - post_tool_call - - on_session_start - - on_session_finalize - - subagent_start - - subagent_stop diff --git a/pyproject.toml b/pyproject.toml index c63e458b8b9bf..b42bdfd65a652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,10 +156,6 @@ edge-tts = ["edge-tts==7.2.7"] modal = ["modal==1.3.4"] daytona = ["daytona==0.155.0"] hindsight = ["hindsight-client==0.6.1"] -# OTLP telemetry export (optional). Provides the OpenTelemetry SDK + OTLP/HTTP -# exporter so `hermes telemetry export --otlp` can send spans to a Collector. -# Lazy-imported by agent/telemetry/otlp_exporter.py; never a core dependency. -otlp = ["opentelemetry-sdk==1.30.0", "opentelemetry-exporter-otlp-proto-http==1.30.0"] dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==81.0.0"] # starlette: CVE-2026-48710; setuptools: latest <82 (torch >=2.11 caps setuptools<82) messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.14.1", "brotlicffi==1.2.0.1", "slack-bolt==1.29.0", "slack-sdk==3.43.0", "qrcode==7.4.2"] # aiohttp 3.14.1: CVE-2026-34513/34518/34519/34520/34525 + 34993(RCE)/47265 cron = [] # croniter is now a core dependency; this extra kept for back-compat @@ -225,6 +221,11 @@ acp = ["agent-client-protocol==0.9.0"] # NOT re-added to [all] so a future quarantined release can't break fresh # installs (see [all] policy comment below). mistral = ["mistralai==2.4.8"] +# OTLP gateway monitoring export (optional). Provides the OpenTelemetry SDK + +# OTLP/HTTP exporter for monitoring.gateway_health_export. Lazy-installed via +# tools/lazy_deps.py on first use; never a core dependency and deliberately +# NOT in [all]. +otlp = ["opentelemetry-sdk==1.39.1", "opentelemetry-exporter-otlp-proto-http==1.39.1"] bedrock = ["boto3==1.42.89"] vertex = ["google-auth==2.55.1"] azure-identity = ["azure-identity==1.25.3"] diff --git a/scripts/observability/gateway_health_export_probe.py b/scripts/observability/gateway_health_export_probe.py new file mode 100644 index 0000000000000..9b6de1b8605c9 --- /dev/null +++ b/scripts/observability/gateway_health_export_probe.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Exercise Gateway Health & Diagnostics Export against a local OTLP capture collector.""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import tempfile +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", default="http://127.0.0.1:4318/v1/traces") + parser.add_argument("--log", required=True, help="JSONL file written by otel_capture_collector.py") + parser.add_argument("--wait", type=float, default=7.0) + args = parser.parse_args() + + hermes_home = Path(tempfile.mkdtemp(prefix="hermes-otel-smoke-")) + os.environ["HERMES_HOME"] = str(hermes_home) + + from gateway.status import write_runtime_status + from agent.monitoring.gateway_health_export import start_gateway_health_export + from agent.monitoring import emitter + + config = { + "monitoring": { + "local": True, + "gateway_health_export": { + "enabled": True, + "metrics_enabled": True, + "diagnostic_events_enabled": True, + "warning_error_events_enabled": True, + "export_interval_seconds": 5, + "logs_export_interval_seconds": 5, + "resource_attributes": { + "service.name": "hermes-gateway-smoke", + "deployment.environment": "local-smoke", + }, + "redaction": {"enabled": True, "include_raw_stack": False}, + }, + "export": { + "otlp": { + "enabled": True, + "endpoint": args.endpoint, + "headers_env": {}, + } + }, + } + } + + runtime = start_gateway_health_export(config) + if not runtime.enabled: + raise SystemExit(f"gateway health exporter did not enable: {runtime.reason}") + + write_runtime_status(gateway_state="starting", active_agents=0) + write_runtime_status(gateway_state="running", active_agents=2) + write_runtime_status(platform="slack", platform_state="running") + write_runtime_status( + platform="slack", + platform_state="fatal", + error_code="auth_failed", + error_message="Bearer *** rejected for smoke@example.com", + ) + logging.getLogger("gateway.platforms.slack").warning("Slack token *** rejected for smoke@example.com") + emitter.get_emitter().flush(timeout=2.0) + time.sleep(args.wait) + runtime.shutdown() + emitter.get_emitter().flush(timeout=2.0) + + log_path = Path(args.log) + rows = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + paths = {row["path"] for row in rows} + print(json.dumps({"hermes_home": str(hermes_home), "requests": len(rows), "paths": sorted(paths)}, indent=2)) + if "/v1/traces" not in paths: + raise SystemExit("missing /v1/traces request") + if "/v1/logs" not in paths: + raise SystemExit("missing /v1/logs request") + if "/v1/metrics" not in paths: + raise SystemExit("missing /v1/metrics request") + + +if __name__ == "__main__": + main() diff --git a/scripts/observability/otel_capture_collector.py b/scripts/observability/otel_capture_collector.py new file mode 100644 index 0000000000000..0071328a4e2b7 --- /dev/null +++ b/scripts/observability/otel_capture_collector.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Tiny local OTLP/HTTP capture collector for Hermes gateway health smoke tests. + +This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces +and /v1/metrics, records request metadata as JSONL, and returns 200 so local +exporters can be exercised without Docker or a vendor backend. +""" + +from __future__ import annotations + +import argparse +import json +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +class CaptureHandler(BaseHTTPRequestHandler): + log_path: Path + + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length) if length else b"" + record = { + "ts": time.time(), + "path": self.path, + "content_type": self.headers.get("content-type"), + "content_length": length, + "body_prefix_hex": body[:24].hex(), + } + with self.log_path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, sort_keys=True) + "\n") + self.send_response(200) + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, format: str, *args) -> None: + # Keep tmux panes clean; JSONL file is the assertion surface. + return + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=4318) + parser.add_argument("--log", required=True) + args = parser.parse_args() + + log_path = Path(args.log).expanduser().resolve() + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text("", encoding="utf-8") + CaptureHandler.log_path = log_path + server = ThreadingHTTPServer((args.host, args.port), CaptureHandler) + print(f"OTLP capture collector listening on http://{args.host}:{args.port}; log={log_path}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/hermes_cli/test_plugins.py b/tests/hermes_cli/test_plugins.py index 85905e2ff36b6..3fdb6d1812d20 100644 --- a/tests/hermes_cli/test_plugins.py +++ b/tests/hermes_cli/test_plugins.py @@ -761,11 +761,7 @@ class TestPluginHooks: mgr.discover_and_load() assert mgr.has_hook("pre_api_request") is True - # Negative sentinel: a hook no plugin (user or bundled) registers, proving - # that registering pre_api_request doesn't conjure an unrelated hook. - # (post_api_request is now registered by the bundled telemetry plugin, so - # it is no longer a valid "nothing registered this" sentinel.) - assert mgr.has_hook("transform_terminal_output") is False + assert mgr.has_hook("post_api_request") is False results = mgr.invoke_hook( "pre_api_request", session_id="s1", diff --git a/tests/telemetry/__init__.py b/tests/monitoring/__init__.py similarity index 100% rename from tests/telemetry/__init__.py rename to tests/monitoring/__init__.py diff --git a/tests/monitoring/test_emitter.py b/tests/monitoring/test_emitter.py new file mode 100644 index 0000000000000..a0f5c2d889d37 --- /dev/null +++ b/tests/monitoring/test_emitter.py @@ -0,0 +1,86 @@ +"""Tests for the monitoring emitter: hot-path invariant + subscriber fan-out.""" + +from __future__ import annotations + +import time + +from agent.monitoring.emitter import MonitoringEmitter +from agent.monitoring.events import GatewayHealthEvent + + +def test_emit_never_raises_when_disabled(): + em = MonitoringEmitter(enabled=False) + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + assert em.stats()["queued"] == 0 + em.close() + + +def test_emit_accepts_dataclass_and_dict(tmp_path): + em = MonitoringEmitter() + seen: list = [] + em.subscribe(lambda batch: seen.extend(batch)) + em.emit(GatewayHealthEvent(name="gateway.health_snapshot", active_agents=2)) + em.emit({"event": "gateway_diagnostic", "name": "platform.fatal", + "subsystem": "platform.slack"}) + em.flush() + em.close() + kinds = {ev.get("event") for ev in seen} + assert kinds == {"gateway_health", "gateway_diagnostic"} + health = next(ev for ev in seen if ev["event"] == "gateway_health") + assert health["active_agents"] == 2 + assert "ts_ns" in health + + +def test_subscriber_failure_is_isolated(): + em = MonitoringEmitter() + good: list = [] + + def bad(batch): + raise RuntimeError("boom") + + em.subscribe(bad) + em.subscribe(lambda batch: good.extend(batch)) + em.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + em.flush() + em.close() + assert len(good) == 1 # the raising subscriber did not break fan-out + + +def test_unsubscribe_stops_delivery(): + em = MonitoringEmitter() + seen: list = [] + cb = lambda batch: seen.extend(batch) # noqa: E731 + em.subscribe(cb) + em.emit({"event": "gateway_health", "name": "a"}) + em.flush() + em.unsubscribe(cb) + em.emit({"event": "gateway_health", "name": "b"}) + em.flush() + em.close() + assert [ev["name"] for ev in seen] == ["a"] + + +def test_queue_full_drops_oldest(): + em = MonitoringEmitter() + # Fill the queue without a dispatcher running by not letting it start: + # emit() starts the thread, so instead assert drop accounting via stats + # after a burst larger than the queue. + for i in range(11_000): + em.emit({"event": "gateway_health", "name": f"e{i}"}) + # Give the dispatcher a moment; total dispatched + queued + dropped == emitted. + em.flush(timeout=5.0) + stats = em.stats() + em.close() + assert stats["dropped"] >= 0 + assert stats["dispatched"] + stats["queued"] + stats["dropped"] >= 10_000 + + +def test_hot_path_is_fast(): + em = MonitoringEmitter() + start = time.perf_counter() + for _ in range(1_000): + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + elapsed = time.perf_counter() - start + em.close() + # 1000 emits should be far under a second even on slow CI. + assert elapsed < 1.0 diff --git a/tests/monitoring/test_export_redaction.py b/tests/monitoring/test_export_redaction.py new file mode 100644 index 0000000000000..0a5beaff90883 --- /dev/null +++ b/tests/monitoring/test_export_redaction.py @@ -0,0 +1,56 @@ +"""Export redaction pipeline tests — the security-critical layer. + +Invariants: + * Secrets ALWAYS stripped, every export path, no flag disables it. + * Fails CLOSED: if the redactor can't run, the raw string is never emitted. + * PII (emails, phones, UUID-shaped ids) stripped in 'pii' mode — the mode + the gateway diagnostics path always uses. +""" + +from __future__ import annotations + +import agent.monitoring.redaction as R + + +def test_secret_always_stripped_in_none_mode(): + fake_key = "sk-ant-api03-" + "A" * 24 # constructed to dodge literal-scrubbers + text = f"calling with key {fake_key} and moving on" + out = R.redact_for_export(text, content_mode=R.CONTENT_NONE) + assert out is not None + assert fake_key not in out + + +def test_secret_always_stripped_in_pii_mode(): + fake_token = "ghp_" + "0123456789abcdef" * 2 + "0123" + text = f"token {fake_token} leaked" + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert out is not None + assert fake_token not in out + + +def test_none_passthrough(): + assert R.redact_for_export(None, content_mode=R.CONTENT_NONE) is None + + +def test_pii_mode_strips_email_phone_uuid(): + text = ("reach alice@example.com or +1 415 555 0100, " + "install 123e4567-e89b-12d3-a456-426614174000") + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert out is not None + assert "alice@example.com" not in out + assert "426614174000" not in out + assert "[email]" in out + assert "[id]" in out + + +def test_none_mode_keeps_ordinary_words(): + out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE) + assert out == "just ordinary words" + + +def test_pii_mode_preserves_non_pii_structure(): + text = "platform.slack entered fatal after auth_failed" + out = R.redact_for_export(text, content_mode=R.CONTENT_PII) + assert out is not None + assert "platform.slack" in out + assert "auth_failed" in out diff --git a/tests/monitoring/test_gateway_health_export.py b/tests/monitoring/test_gateway_health_export.py new file mode 100644 index 0000000000000..8c1a9ef8513ac --- /dev/null +++ b/tests/monitoring/test_gateway_health_export.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import logging + + +def test_default_config_keeps_gateway_health_export_disabled(): + from hermes_cli.config import DEFAULT_CONFIG + + cfg = DEFAULT_CONFIG["monitoring"]["gateway_health_export"] + + assert cfg["enabled"] is False + assert cfg["metrics_enabled"] is True + assert cfg["diagnostic_events_enabled"] is True + assert cfg["warning_error_events_enabled"] is True + assert cfg["export_interval_seconds"] == 60 + assert cfg["logs_export_interval_seconds"] == 5 + assert cfg["redaction"]["enabled"] is True + assert cfg["redaction"]["include_raw_stack"] is False + + +def test_gateway_health_snapshot_maps_runtime_status_to_low_cardinality_metrics(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + runtime = { + "gateway_state": "running", + "pid": 1234, + "active_agents": "2", + "restart_requested": False, + "platforms": { + "slack": {"state": "running"}, + "telegram": { + "state": "fatal", + "error_code": "auth_failed", + "error_message": "token xoxb-secret rejected for user 123", + }, + }, + } + + snapshot = build_gateway_health_snapshot( + runtime, + gateway_running=True, + profile="default", + install_id="install-1", + version="2026.7.test", + supervision_mode="manual", + ) + + metric_names = {m.name for m in snapshot.metrics} + assert { + "hermes.gateway.up", + "hermes.gateway.active_agents", + "hermes.gateway.busy", + "hermes.gateway.drainable", + "hermes.gateway.restart_requested", + "hermes.platform.up", + "hermes.platform.degraded", + } <= metric_names + + active = next(m for m in snapshot.metrics if m.name == "hermes.gateway.active_agents") + assert active.value == 2 + assert active.attributes == { + "hermes.profile": "default", + "service.instance.id": "install-1", + "service.version": "2026.7.test", + "hermes.supervision_mode": "manual", + } + + busy = next(m for m in snapshot.metrics if m.name == "hermes.gateway.busy") + drainable = next(m for m in snapshot.metrics if m.name == "hermes.gateway.drainable") + assert busy.value == 1 + assert drainable.value == 1 + + degraded = next( + m for m in snapshot.metrics + if m.name == "hermes.platform.degraded" and m.attributes["hermes.platform"] == "telegram" + ) + assert degraded.value == 1 + assert degraded.attributes["hermes.error_code"] == "auth_failed" + assert all("secret" not in str(v).lower() for v in degraded.attributes.values()) + + +def test_gateway_health_snapshot_emits_content_free_diagnostic_event(): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + + snapshot = build_gateway_health_snapshot( + { + "gateway_state": "running", + "active_agents": 1, + "platforms": { + "slack": {"state": "fatal", "error_code": "auth_failed", "error_message": "Bearer sk-live-secret"}, + }, + }, + gateway_running=True, + profile="default", + install_id="install-1", + version="v-test", + supervision_mode="container", + ) + + events = [event.to_dict() for event in snapshot.events] + health = next(e for e in events if e["event"] == "gateway_health") + platform = next(e for e in events if e["event"] == "gateway_diagnostic" and e["name"] == "platform.fatal") + + assert health["gateway_state"] == "running" + assert health["active_agents"] == 1 + assert health["gateway_busy"] is True + assert health["gateway_drainable"] is True + assert health["fatal_platform_count"] == 1 + assert platform["platform"] == "slack" + assert platform["error_code"] == "auth_failed" + assert "secret" not in platform["redacted_message"].lower() + assert "Bearer" not in platform["redacted_message"] + + +def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + old = emitter.get_emitter + emitter.get_emitter = lambda: DummyEmitter() # type: ignore[assignment] + try: + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + logger = logging.getLogger("gateway.platforms.slack") + logger.setLevel(logging.DEBUG) + logger.addHandler(handler) + try: + logger.info("ignore info token sk-live-secret") + logger.warning("Slack token sk-live-secret failed for user@example.com") + finally: + logger.removeHandler(handler) + finally: + emitter.get_emitter = old # type: ignore[assignment] + + assert len(captured) == 1 + event = captured[0] + assert event["event"] == "gateway_diagnostic" + assert event["name"] == "gateway.log.warning" + assert event["subsystem"] == "platform.slack" + assert event["error_class"] == "auth_failed" + assert "***" not in event["redacted_message"] + assert "user@example.com" not in event["redacted_message"] + + +def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.gateway_health import emit_runtime_status_transition + + captured = [] + + class DummyEmitter: + def emit(self, event): + captured.append(event.to_dict()) + + old = emitter.emit + monkeypatch.setattr(emitter, "emit", lambda event: captured.append(event.to_dict())) + + previous = {"gateway_state": "starting", "platforms": {"slack": {"state": "running"}}} + current = { + "gateway_state": "running", + "pid": 123, + "active_agents": 1, + "platforms": { + "slack": { + "state": "fatal", + "error_code": "auth_failed", + "error_message": "Bearer *** failed", + } + }, + } + + emit_runtime_status_transition(previous, current) + + names = [e["name"] for e in captured] + assert "gateway.lifecycle" in names + assert "platform.state_change" in names + assert "platform.fatal" in names + lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle") + assert lifecycle["old_state"] == "starting" + assert lifecycle["new_state"] == "running" + platform = next(e for e in captured if e["name"] == "platform.state_change") + assert platform["old_state"] == "running" + assert platform["new_state"] == "fatal" + assert platform["error_code"] == "auth_failed" + assert "Bearer" not in platform["redacted_message"] + + +def test_runtime_status_transition_emits_startup_failed_and_exit(): + from agent.monitoring.gateway_health import emit_runtime_status_transition + from agent.monitoring import emitter + + captured = [] + old = emitter.emit + emitter.emit = lambda event: captured.append(event.to_dict()) # type: ignore[assignment] + try: + emit_runtime_status_transition({"gateway_state": "starting"}, {"gateway_state": "startup_failed", "exit_reason": "startup token ***"}) + emit_runtime_status_transition({"gateway_state": "running"}, {"gateway_state": "stopped", "exit_reason": "shutdown", "restart_requested": True}) + finally: + emitter.emit = old # type: ignore[assignment] + + names = [e["name"] for e in captured] + assert "gateway.startup_failed" in names + assert "gateway.exit" in names + failed = next(e for e in captured if e["name"] == "gateway.startup_failed") + assert "***" not in failed["redacted_message"] + exit_event = next(e for e in captured if e["name"] == "gateway.exit") + assert exit_event["restart_requested"] is True + + +def test_otlp_attrs_include_gateway_transition_fields(): + from agent.monitoring.otlp_exporter import _span_attrs + + attrs = _span_attrs({ + "event": "gateway_health", + "name": "gateway.lifecycle", + "old_state": "starting", + "new_state": "running", + "exit_reason": "restart", + "restart_requested": True, + }) + + assert attrs["hermes.old_state"] == "starting" + assert attrs["hermes.new_state"] == "running" + assert attrs["hermes.exit_reason"] == "restart" + assert attrs["hermes.restart_requested"] is True + + +def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch): + from agent.monitoring import gateway_health_export + from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime + + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("missing sdk"))) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4317"}}, + } + }) + + assert isinstance(runtime, GatewayHealthExportRuntime) + assert runtime.enabled is False + assert runtime.reason == "otlp_unavailable" + + +def test_gateway_health_export_streams_only_gateway_events(monkeypatch): + from agent.monitoring import gateway_health_export + + captured = {} + + def fake_start_streaming(config, *, event_filter=None): + captured["filter"] = event_filter + return object() + + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_attach_log_handler", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_emit_snapshot_events", lambda *a, **k: None) + monkeypatch.setattr(gateway_health_export, "_start_snapshot_thread", lambda *a, **k: None) + from agent.monitoring import otlp_exporter + monkeypatch.setattr(otlp_exporter, "start_streaming", fake_start_streaming) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True, "metrics_enabled": False}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is True + event_filter = captured["filter"] + assert event_filter({"event": "gateway_health"}) is True + assert event_filter({"event": "gateway_diagnostic"}) is False + assert event_filter({"event": "run"}) is False + assert event_filter({"event": "model_call"}) is False + assert event_filter({"event": "tool_call"}) is False + + +def test_gateway_health_export_metric_failure_does_not_start_streamer(monkeypatch): + from agent.monitoring import gateway_health_export, otlp_exporter + + started = [] + monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {}) + monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))) + monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: started.append(True)) + + runtime = gateway_health_export.start_gateway_health_export({ + "monitoring": { + "gateway_health_export": {"enabled": True}, + "export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}}, + } + }) + + assert runtime.enabled is False + assert runtime.reason == "metrics_start_failed" + assert started == [] + + +def test_otlp_streamer_shutdown_unsubscribes(monkeypatch): + from agent.monitoring import emitter + from agent.monitoring.otlp_exporter import OTLPStreamer + + class Dummy: + def force_flush(self): + pass + def shutdown(self): + pass + + e = emitter.get_emitter() + streamer = OTLPStreamer.__new__(OTLPStreamer) + streamer._processor = Dummy() + streamer._provider = Dummy() + streamer._event_filter = None + streamer.exported = 0 + e.subscribe(streamer) + assert streamer in e._subscribers + + streamer.shutdown() + + assert streamer not in e._subscribers + + +def test_gateway_diagnostic_log_handler_never_raises_on_malformed_record(): + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + + handler = GatewayDiagnosticLogHandler(profile="default", version="v-test") + record = logging.LogRecord( + "gateway.platforms.slack", + logging.WARNING, + __file__, + 1, + "broken %s %s", + ("one",), + None, + ) + + handler.emit(record) diff --git a/tests/monitoring/test_otlp_exporter.py b/tests/monitoring/test_otlp_exporter.py new file mode 100644 index 0000000000000..56edd566524f1 --- /dev/null +++ b/tests/monitoring/test_otlp_exporter.py @@ -0,0 +1,125 @@ +"""OTLP exporter tests: config resolution, span mapping, streaming subscriber. + +No SQLite involved — monitoring is an egress path, so the exporter consumes +emitter batches directly. Uses the in-memory OTel span exporter; skipped when +the optional otlp extra is not installed. +""" + +from __future__ import annotations + +import pytest + +otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed") + +import agent.monitoring.otlp_exporter as OE +from agent.monitoring.emitter import MonitoringEmitter + + +def _mem_provider(): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def test_gateway_health_event_maps_to_span_with_attrs(): + provider, mem = _mem_provider() + n = OE.export_batch(provider, [{ + "event": "gateway_health", "name": "gateway.lifecycle", + "old_state": "starting", "new_state": "running", + "active_agents": 2, "pid": 4242, + }]) + assert n == 1 + spans = mem.get_finished_spans() + assert spans[0].name == "hermes.gateway_health" + attrs = dict(spans[0].attributes or {}) + assert attrs["hermes.old_state"] == "starting" + assert attrs["hermes.new_state"] == "running" + assert attrs["hermes.active_agents"] == 2 + + +def test_gateway_diagnostic_event_maps_redacted_message(): + provider, mem = _mem_provider() + OE.export_batch(provider, [{ + "event": "gateway_diagnostic", "name": "platform.fatal", + "subsystem": "platform.slack", "error_class": "auth_failed", + "redacted_message": "token [redacted] rejected", "severity": "error", + }]) + attrs = dict(mem.get_finished_spans()[0].attributes or {}) + assert attrs["hermes.error_class"] == "auth_failed" + assert attrs["hermes.redacted_message"] == "token [redacted] rejected" + + +def test_unknown_event_kind_exports_no_attrs_beyond_kind(): + provider, mem = _mem_provider() + OE.export_batch(provider, [{"event": "model_call", "provider": "anthropic", + "model": "claude-opus-4"}]) + attrs = dict(mem.get_finished_spans()[0].attributes or {}) + # Non-monitoring event kinds carry no attribute mapping on this plane. + assert attrs == {"hermes.event": "model_call"} + + +def test_headers_resolve_from_env_not_value(monkeypatch): + monkeypatch.setenv("DD_KEY_ENV", "secret-value") + resolved = OE._resolve_headers({"DD-API-KEY": "DD_KEY_ENV", "X-Missing": "NOPE_ENV"}) + assert resolved == {"DD-API-KEY": "secret-value"} + + +def test_is_enabled_requires_endpoint_and_flag(): + assert OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}) + assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True}}}}) + assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"endpoint": "http://x"}}}}) + assert not OE.is_enabled({}) + + +def test_export_otlp_feature_specs_match_pyproject(): + from tools.lazy_deps import LAZY_DEPS + import re + from pathlib import Path + + specs = set(LAZY_DEPS["export.otlp"]) + pyproject = Path(__file__).resolve().parents[2] / "pyproject.toml" + m = re.search(r'^otlp = \[(.*?)\]', pyproject.read_text(), re.M | re.S) + assert m, "otlp extra missing from pyproject.toml" + extra = set(re.findall(r'"([^"]+)"', m.group(1))) + assert specs == extra + + +def test_streamer_receives_events_and_respects_filter(monkeypatch): + provider, mem = _mem_provider() + monkeypatch.setattr(OE, "_make_provider", lambda cfg: (provider, None)) + streamer = OE.OTLPStreamer( + {}, event_filter=lambda ev: ev.get("event") == "gateway_health") + + em = MonitoringEmitter() + em.subscribe(streamer) + em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"}) + em.emit({"event": "model_call", "provider": "anthropic"}) # filtered out + em.flush() + em.close() + + spans = mem.get_finished_spans() + assert [s.name for s in spans] == ["hermes.gateway_health"] + assert streamer.exported == 1 + + +def test_failing_streamer_never_breaks_emitter(monkeypatch): + def boom(cfg): + raise RuntimeError("no provider") + + em = MonitoringEmitter() + + def bad_subscriber(batch): + raise RuntimeError("export down") + + seen: list = [] + em.subscribe(bad_subscriber) + em.subscribe(lambda batch: seen.extend(batch)) + em.emit({"event": "gateway_health", "name": "gateway.lifecycle"}) + em.flush() + em.close() + assert len(seen) == 1 diff --git a/tests/telemetry/test_cli_telemetry.py b/tests/telemetry/test_cli_telemetry.py deleted file mode 100644 index 19fc59e394ce5..0000000000000 --- a/tests/telemetry/test_cli_telemetry.py +++ /dev/null @@ -1,108 +0,0 @@ -"""`hermes telemetry` handler smoke tests (local-only; no upload).""" - -from __future__ import annotations - -import sqlite3 -import time -import types - -import pytest - -import hermes_state - - -@pytest.fixture -def home(tmp_path, monkeypatch): - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - # state.db with tel_* + seeded data - db = tmp_path / "state.db" - hermes_state.SessionDB(db_path=db) - from agent.telemetry.emitter import TelemetryEmitter - from agent.telemetry.events import RunEvent, ModelCallEvent, ToolCallEvent - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "e.jsonl", db_path=db) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", - start_ns=now - 60_000_000, end_ns=now, model_call_count=1, - tool_call_count=1)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-opus-4", - input_tokens=20000, output_tokens=2000)) - em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", - result_class="ok")) - em.flush() - em.close() - yield tmp_path - - -def _run(action, **kw): - from hermes_cli.main import cmd_telemetry - args = types.SimpleNamespace(telemetry_action=action, days=30, limit=10, json=False) - for k, v in kw.items(): - setattr(args, k, v) - cmd_telemetry(args) - - -def test_status_runs(home, capsys): - _run("status") - out = capsys.readouterr().out - assert "Telemetry status" in out - assert "Upload:" in out and "DISABLED" in out - assert "Local data:" in out - - -def test_preview_shows_real_values(home, capsys): - _run("preview") - out = capsys.readouterr().out - assert "NOT uploaded" in out - assert "workflow_completed" in out - # real model + tool names ARE shown (this is the user's own local data) - assert "claude-opus-4" in out - assert "web_search" in out - - -def test_status_reflects_consent_set_via_config(home, capsys): - # Opting in is a plain config write now (no `enable` verb). status should - # reflect consent_state=aggregate as aggregate metrics being on. - from hermes_cli.config import load_config, save_config - cfg = load_config() - cfg.setdefault("telemetry", {})["consent_state"] = "aggregate" - save_config(cfg) - _run("status") - out = capsys.readouterr().out - assert "consent_state=aggregate" in out - assert "Aggregate metrics: on" in out - - -def test_status_shows_optin_hint_when_unknown(home, capsys): - _run("status") - out = capsys.readouterr().out - assert "Aggregate metrics: off" in out - assert "config set telemetry.consent_state aggregate" in out - - -def test_allow_aggregate_false_keeps_metrics_off_in_status(home, capsys): - # Even with consent opted in, a managed allow_aggregate:false wins. - from hermes_cli.config import load_config, save_config - cfg = load_config() - tel = cfg.setdefault("telemetry", {}) - tel["consent_state"] = "aggregate" - tel["allow_aggregate"] = False - save_config(cfg) - _run("status") - out = capsys.readouterr().out - assert "Aggregate metrics: off" in out - assert "allow_aggregate is false" in out - - -def test_local_off_with_consent_shows_inert_in_status(home, capsys): - # local off + opted in: aggregate is off and the status explains why. - from hermes_cli.config import load_config, save_config - cfg = load_config() - tel = cfg.setdefault("telemetry", {}) - tel["local"] = False - tel["consent_state"] = "aggregate" - save_config(cfg) - _run("status") - out = capsys.readouterr().out - assert "Aggregate metrics: off" in out - assert "inert: local telemetry is off" in out diff --git a/tests/telemetry/test_emitter.py b/tests/telemetry/test_emitter.py deleted file mode 100644 index f5b2902aa7761..0000000000000 --- a/tests/telemetry/test_emitter.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Emitter tests — the hot-path invariant is the one that matters most. - -Invariant: emit() never blocks, never raises, and a broken writer cannot slow or -break the caller. Plus: JSONL + SQLite round-trip, and the SQLite index is rebuildable -from the JSONL source of truth. -""" - -from __future__ import annotations - -import sqlite3 -import time - -import hermes_state -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -def _fresh_db(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - return db - - -def test_emit_is_fast_even_when_writer_is_broken(tmp_path, monkeypatch): - """The core guarantee: a writer that raises AND sleeps cannot stall emit().""" - db = _fresh_db(tmp_path) - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - - # Sabotage the row indexer to raise after a long sleep. - def broken(_conn, _ev): - time.sleep(5.0) - raise RuntimeError("writer exploded") - - monkeypatch.setattr(em, "_index_one", broken) - - start = time.monotonic() - for i in range(50): - em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=10)) - elapsed = time.monotonic() - start - - # 50 emits must complete in well under the writer's single 5s sleep. - assert elapsed < 1.0, f"emit() blocked: {elapsed:.2f}s" - em.close() - - -def test_emit_never_raises_on_bad_event(tmp_path): - db = _fresh_db(tmp_path) - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - # Non-serializable / wrong-shaped inputs must not raise out of emit(). - em.emit(object()) # no to_dict, not a mapping - em.emit({"event": "run"}) # minimal dict - em.close() - - -def test_jsonl_and_sqlite_roundtrip(tmp_path): - db = _fresh_db(tmp_path) - jsonl = tmp_path / "telemetry" / "events.jsonl" - em = TelemetryEmitter(events_path=jsonl, db_path=db) - - em.emit(RunEvent(run_id="run1", trace_id="t1", entrypoint="cli", end_reason="completed")) - em.emit(ModelCallEvent(span_id="m1", run_id="run1", provider="anthropic", - model="claude-opus-4", input_tokens=100, output_tokens=20)) - em.emit(ToolCallEvent(span_id="tc1", run_id="run1", tool_name="web_search", - duration_ms=120, result_class="ok")) - em.flush() - em.close() - - # JSONL has all three lines - lines = [l for l in jsonl.read_text(encoding="utf-8").splitlines() if l.strip()] - assert len(lines) == 3 - - # SQLite index has the rows in the right tables - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1 - assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 1 - assert conn.execute("SELECT COUNT(*) FROM tel_tool_calls").fetchone()[0] == 1 - row = conn.execute("SELECT provider, model, input_tokens FROM tel_model_calls").fetchone() - assert row == ("anthropic", "claude-opus-4", 100) - conn.close() - - -def test_unknown_event_kind_is_ignored_not_fatal(tmp_path): - db = _fresh_db(tmp_path) - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - em.emit({"event": "totally_unknown", "foo": "bar"}) - em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli")) - em.flush() - em.close() - conn = sqlite3.connect(db) - # The unknown event is in JSONL but skipped by the indexer; the known one indexes. - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1 - conn.close() - - -def test_disabled_emitter_writes_nothing(tmp_path): - db = _fresh_db(tmp_path) - jsonl = tmp_path / "telemetry" / "events.jsonl" - em = TelemetryEmitter(events_path=jsonl, db_path=db, enabled=False) - em.emit(RunEvent(run_id="r3", trace_id="t3", entrypoint="cli")) - em.flush() - em.close() - assert not jsonl.exists() - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0 - conn.close() diff --git a/tests/telemetry/test_export_redaction.py b/tests/telemetry/test_export_redaction.py deleted file mode 100644 index 4ae0473bfbae6..0000000000000 --- a/tests/telemetry/test_export_redaction.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Export redaction pipeline tests — the security-critical layer. - -Invariants: - * Secrets ALWAYS stripped, every mode, no flag disables it. - * Content gated by telemetry.trajectories, not a redaction mode. - * PII stripped in 'pii' mode; structure preserved (codec-aware). -""" - -from __future__ import annotations - -import json - -from agent.telemetry import redaction as R - - -# ── secrets are always redacted ───────────────────────────────────────────── -def test_secrets_stripped_in_none_mode(): - text = "here is sk-ant-api03-SECRETKEY123 and a token" - out = R.redact_for_export(text, content_mode=R.CONTENT_NONE) - assert "SECRETKEY123" not in out - - -def test_secrets_stripped_in_pii_mode(): - text = "Authorization: Bearer abcdef123456789secret" - out = R.redact_for_export(text, content_mode=R.CONTENT_PII) - assert "abcdef123456789secret" not in out - - -def test_secret_redactor_fails_closed(monkeypatch): - # If the underlying redactor raises, we must NOT return the raw string. - import agent.redact as ar - monkeypatch.setattr(ar, "redact_sensitive_text", lambda *a, **k: (_ for _ in ()).throw(RuntimeError())) - out = R.redact_for_export("sk-secret-value", content_mode=R.CONTENT_NONE) - assert "sk-secret-value" not in out - assert out == "[redaction-unavailable]" - - -# ── PII ───────────────────────────────────────────────────────────────────── -def test_pii_mode_strips_email_and_phone(): - text = "contact alice@example.com or +1 415 555 1234" - out = R.redact_for_export(text, content_mode=R.CONTENT_PII) - assert "alice@example.com" not in out - assert "[email]" in out - assert "555" not in out or "[phone]" in out - - -def test_none_mode_keeps_nonsecret_text_but_drops_via_message_path(): - # redact_for_export(none) scrubs secrets but doesn't strip ordinary words; - # content *dropping* happens at the message layer (trajectories gate). - out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE) - assert "ordinary" in out - - -# ── trajectories gate (content_export_enabled) ────────────────────────────── -def test_content_export_disabled_by_default(): - assert R.content_export_enabled({}) is False - assert R.content_export_enabled({"telemetry": {}}) is False - assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": False}}}) is False - - -def test_content_export_enabled_when_trajectories_on(): - assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": True}}}) is True - - -# ── codec-aware message redaction ─────────────────────────────────────────── -def test_message_structural_only_when_content_excluded(): - msg = {"role": "user", "content": "my email is bob@x.com and key sk-12345"} - out = R.redact_message(msg, include_content=False) - assert out["role"] == "user" - assert "content" not in out # body dropped entirely - assert out["content_chars"] == len(msg["content"]) # only the size remains - assert "bob@x.com" not in json.dumps(out) - - -def test_message_content_included_is_redacted(): - msg = {"role": "user", "content": "email bob@x.com secret sk-ant-SECRET999"} - out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True) - assert "content" in out - assert "SECRET999" not in out["content"] # secret gone - assert "bob@x.com" not in out["content"] # pii gone - assert "[email]" in out["content"] - - -def test_tool_calls_redacted_names_kept_args_scrubbed(): - msg = { - "role": "assistant", - "tool_calls": json.dumps([ - {"function": {"name": "web_search", "arguments": '{"q": "email me at z@z.com"}'}} - ]), - } - out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True) - tc = out["tool_calls"] - assert tc[0]["name"] == "web_search" # structure/name preserved - assert "z@z.com" not in json.dumps(tc) # arg pii scrubbed - - -def test_tool_calls_counted_when_content_excluded(): - msg = { - "role": "assistant", - "tool_calls": json.dumps([ - {"function": {"name": "a", "arguments": "{}"}}, - {"function": {"name": "b", "arguments": "{}"}}, - ]), - } - out = R.redact_message(msg, include_content=False) - assert out["tool_call_count"] == 2 - assert "tool_calls" not in out - - -def test_content_mode_for_reads_config(): - assert R.content_mode_for({"telemetry": {"content_redaction": "pii"}}) == "pii" - assert R.content_mode_for({"telemetry": {"content_redaction": "bogus"}}) == "none" - assert R.content_mode_for({}) == "none" diff --git a/tests/telemetry/test_exporter_bulk.py b/tests/telemetry/test_exporter_bulk.py deleted file mode 100644 index 10e0fa68f08e2..0000000000000 --- a/tests/telemetry/test_exporter_bulk.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Bulk export tests — telemetry always, content only behind the trajectories gate.""" - -from __future__ import annotations - -import io -import json -import sqlite3 -import time - -import hermes_state -from agent.telemetry import exporter_bulk -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -def _seed(tmp_path, with_secret_content=True): - db = tmp_path / "state.db" - sdb = hermes_state.SessionDB(db_path=db) - # telemetry - em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", - start_ns=now - 60_000_000, end_ns=now, model_call_count=1, tool_call_count=1)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-sonnet-4", input_tokens=1000, output_tokens=100)) - em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok")) - em.flush() - em.close() - # session + message content (with an embedded secret + email) - sdb.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - if with_secret_content: - sdb.append_message("s1", role="user", - content="my key is AKIAIOSFODNN7EXAMPLE and email me at carol@corp.com") - sdb.append_message("s1", role="assistant", content="ok") - sdb.close() - return db - - -def test_telemetry_exported_content_excluded_by_default(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - counts = exporter_bulk.export(buf, fmt="ndjson", include_content=False, config={}, db_path=db) - assert counts["telemetry"] >= 3 - assert counts["content_included"] == 0 - text = buf.getvalue() - # message bodies must NOT be present - assert "carol@corp.com" not in text - assert "AKIAIOSFODNN7EXAMPLE" not in text - # but a session record (structural) is present with message structure - lines = [json.loads(l) for l in text.splitlines() if l.strip()] - sess = [r for r in lines if r.get("_kind") == "session"] - assert sess and sess[0]["messages"] - assert "content" not in sess[0]["messages"][0] # structural only - assert "content_chars" in sess[0]["messages"][0] - - -def test_include_content_ignored_without_trajectories(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - # request content but trajectories disabled -> forced off - counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True, - config={"telemetry": {"trajectories": {"enabled": False}}}, db_path=db) - assert counts["content_included"] == 0 - assert "carol@corp.com" not in buf.getvalue() - - -def test_content_included_when_trajectories_on_but_redacted(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - cfg = {"telemetry": {"trajectories": {"enabled": True}, "content_redaction": "pii"}} - counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True, config=cfg, db_path=db) - assert counts["content_included"] == 1 - text = buf.getvalue() - # content present but secret + pii scrubbed - assert "AKIAIOSFODNN7EXAMPLE" not in text # secret always gone - assert "carol@corp.com" not in text # pii gone in pii mode - lines = [json.loads(l) for l in text.splitlines() if l.strip()] - sess = [r for r in lines if r.get("_kind") == "session"][0] - # the user message now has a (redacted) content field - user_msg = [m for m in sess["messages"] if m["role"] == "user"][0] - assert "content" in user_msg - assert "[email]" in user_msg["content"] - - -def test_json_format_roundtrips(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - exporter_bulk.export(buf, fmt="json", include_content=False, config={}, db_path=db) - obj = json.loads(buf.getvalue()) - assert "records" in obj - assert any(r["_kind"] == "tel_runs" for r in obj["records"]) - - -def test_since_window_filters_runs(tmp_path): - db = _seed(tmp_path) - buf = io.StringIO() - # since 1ns ago in the future-ish -> the run (start ~60ms ago) excluded - future_ns = int((time.time() + 1) * 1e9) - counts = exporter_bulk.export(buf, fmt="ndjson", since_ns=future_ns, config={}, db_path=db) - lines = [json.loads(l) for l in buf.getvalue().splitlines() if l.strip()] - runs = [r for r in lines if r.get("_kind") == "tel_runs"] - assert runs == [] diff --git a/tests/telemetry/test_governance.py b/tests/telemetry/test_governance.py deleted file mode 100644 index 9774054036798..0000000000000 --- a/tests/telemetry/test_governance.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Export configuration visibility in `hermes telemetry status`. - -The status Export block reports the current export configuration. Whether a key is -locked is handled by the managed-scope layer, not repeated here; the allow_aggregate -gate is covered by a test so a managed pin can't be regressed. -""" - -from __future__ import annotations - -import time -import types - -import pytest - -import hermes_state - - -@pytest.fixture -def home(tmp_path, monkeypatch): - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - hermes_state.SessionDB(db_path=tmp_path / "state.db") - yield tmp_path - - -def _status(capsys): - from hermes_cli.main import cmd_telemetry - cmd_telemetry(types.SimpleNamespace(telemetry_action="status")) - return capsys.readouterr().out - - -def test_export_block_default_shows_otlp_disabled(home, capsys): - out = _status(capsys) - assert "Export" in out - assert "OTLP:" in out and "disabled" in out - assert "Content export: off" in out - assert "Secret redaction: on (always)" in out - - -def test_export_block_shows_endpoint_host_never_token(home, capsys, monkeypatch): - from hermes_cli.config import load_config, save_config - monkeypatch.setenv("CORP_OTLP_TOKEN", "supersecret-do-not-print") - c = load_config() - t = c.setdefault("telemetry", {}) - t.setdefault("export", {})["otlp"] = { - "enabled": True, - "endpoint": "https://collector.corp:4318/v1/traces", - "headers_env": {"Authorization": "CORP_OTLP_TOKEN"}, - } - save_config(c) - out = _status(capsys) - # endpoint host present - assert "https://collector.corp:4318/v1/traces" in out - # env var name + set-state present; the VALUE never printed - assert "CORP_OTLP_TOKEN" in out - assert "(set)" in out - assert "supersecret-do-not-print" not in out - - -def test_export_block_reflects_trajectories_gate(home, capsys): - from hermes_cli.config import load_config, save_config - c = load_config() - c.setdefault("telemetry", {})["trajectories"] = {"enabled": True} - save_config(c) - out = _status(capsys) - assert "Content export: on (trajectories enabled)" in out - - -def test_token_env_not_set_shows_not_set(home, capsys): - from hermes_cli.config import load_config, save_config - c = load_config() - t = c.setdefault("telemetry", {}) - t.setdefault("export", {})["otlp"] = { - "enabled": True, - "endpoint": "https://x:4318/v1/traces", - "headers_env": {"Authorization": "TOTALLY_UNSET_ENV_VAR_XYZ"}, - } - save_config(c) - out = _status(capsys) - assert "(NOT set)" in out - - -def test_allow_aggregate_pin_blocks_opt_in(home): - """A managed allow_aggregate:false pin overrides a consent_state opt-in. - - Consent is set in config (as a user or managed-scope pin would); the hard gate - still wins, so may_upload stays false. - """ - from hermes_cli.config import load_config, save_config - from agent.telemetry import policy - c = load_config() - tel = c.setdefault("telemetry", {}) - tel["consent_state"] = "aggregate" - tel["allow_aggregate"] = False - save_config(c) - assert policy.may_upload_aggregate(load_config()) is False diff --git a/tests/telemetry/test_insights_integration.py b/tests/telemetry/test_insights_integration.py deleted file mode 100644 index 814f3eeb09386..0000000000000 --- a/tests/telemetry/test_insights_integration.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Insights ↔ telemetry integration: the observability section in /insights output.""" - -from __future__ import annotations - -import time - -import pytest - -from hermes_state import SessionDB -from agent.insights import InsightsEngine -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -@pytest.fixture() -def db(tmp_path): - session_db = SessionDB(db_path=tmp_path / "ins_tel.db") - yield session_db - session_db.close() - - -def _seed_telemetry(db_path): - em = TelemetryEmitter(events_path=db_path.parent / "tel" / "events.jsonl", db_path=db_path) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway", - platform="telegram", end_reason="completed", - start_ns=now - 90_000_000, end_ns=now)) - em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", - end_reason="failed", start_ns=now - 11_000_000, end_ns=now)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-opus-4", input_tokens=5000, output_tokens=800, - cache_read_tokens=1000, latency_ms=2200)) - em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search", result_class="ok")) - em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate", result_class="error")) - em.flush() - em.close() - - -def test_report_includes_telemetry_when_present(db): - # A session so generate() isn't the empty branch - db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - _seed_telemetry(db.db_path) - - engine = InsightsEngine(db) - report = engine.generate(days=30) - tel = report.get("telemetry") - assert tel, "telemetry section missing" - assert tel["workflows"]["total_runs"] == 2 - assert tel["workflows"]["success_rate"] == 0.5 - assert tel["tool_calls"]["total"] == 2 - assert tel["tool_calls"]["failure_rate"] == 0.5 - assert tel["model_calls"]["by_provider"]["anthropic"] == 1 - - -def test_terminal_output_renders_observability_section(db): - db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - _seed_telemetry(db.db_path) - - engine = InsightsEngine(db) - out = engine.format_terminal(engine.generate(days=30)) - assert "Observability" in out - assert "Workflows:" in out - assert "Failure rate:" in out - assert "Providers:" in out - - -def test_telemetry_section_absent_when_no_tel_rows(db): - # Session present, but no telemetry events seeded. - db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4") - engine = InsightsEngine(db) - report = engine.generate(days=30) - assert report.get("telemetry") == {} - out = engine.format_terminal(report) - assert "Observability" not in out - - -def test_empty_report_has_telemetry_key(db): - # No sessions at all -> empty branch still carries the key (renderer-safe). - engine = InsightsEngine(db) - report = engine.generate(days=30) - assert report.get("empty") is True - assert report.get("telemetry") == {} diff --git a/tests/telemetry/test_otlp_exporter.py b/tests/telemetry/test_otlp_exporter.py deleted file mode 100644 index 49fd990d463ac..0000000000000 --- a/tests/telemetry/test_otlp_exporter.py +++ /dev/null @@ -1,171 +0,0 @@ -"""OTLP exporter tests. - -Skip cleanly when the optional OTel SDK isn't installed. When it is, verify: - * event -> OTel span attribute mapping - * headers_env resolves the value from the named environment variable, not config - * a failing or slow subscriber never breaks the emitter hot path - * is_enabled / is_available gating -""" - -from __future__ import annotations - -import sqlite3 -import time - -import pytest - -import hermes_state -from agent.telemetry import otlp_exporter as OE -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - -otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed") - - -def _in_memory_provider(): - """A TracerProvider with an in-memory span exporter (no network).""" - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import SimpleSpanProcessor - from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - provider = TracerProvider() - mem = InMemorySpanExporter() - provider.add_span_processor(SimpleSpanProcessor(mem)) - return provider, mem - - -def test_event_maps_to_span_with_real_attrs(): - provider, mem = _in_memory_provider() - batch = [ - {"event": "run", "entrypoint": "gateway", "platform": "telegram", - "end_reason": "completed", "model_call_count": 2, "tool_call_count": 3}, - {"event": "model_call", "provider": "anthropic", "model": "claude-opus-4", - "input_tokens": 5000, "output_tokens": 800}, - {"event": "tool_call", "tool_name": "web_search", "result_class": "ok"}, - ] - n = OE.export_batch(provider, batch) - assert n == 3 - spans = mem.get_finished_spans() - names = {s.name for s in spans} - assert names == {"hermes.run", "hermes.model_call", "hermes.tool_call"} - # real values present as span attributes - run = [s for s in spans if s.name == "hermes.run"][0] - assert run.attributes["hermes.entrypoint"] == "gateway" - assert run.attributes["hermes.platform"] == "telegram" - model = [s for s in spans if s.name == "hermes.model_call"][0] - assert model.attributes["hermes.model"] == "claude-opus-4" - assert model.attributes["hermes.provider"] == "anthropic" - tool = [s for s in spans if s.name == "hermes.tool_call"][0] - assert tool.attributes["hermes.tool_name"] == "web_search" - - -def test_headers_resolve_from_env_not_value(monkeypatch): - monkeypatch.setenv("MY_OTLP_TOKEN", "supersecretvalue") - resolved = OE._resolve_headers({"Authorization": "MY_OTLP_TOKEN"}) - assert resolved == {"Authorization": "supersecretvalue"} - # missing env var -> skipped, not crashed - assert OE._resolve_headers({"X": "NOPE_NOT_SET"}) == {} - - -def test_is_enabled_requires_endpoint_and_flag(): - assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}) is True - assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True}}}}) is False - assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": False, "endpoint": "http://x"}}}}) is False - assert OE.is_enabled({}) is False - - -def test_require_sdk_routes_through_lazy_install(monkeypatch): - # _require_sdk(auto_install=True) should call lazy_deps.ensure('export.otlp'). - import tools.lazy_deps as ld - calls = [] - monkeypatch.setattr(ld, "ensure", lambda feature, **kw: calls.append((feature, kw))) - OE._require_sdk(auto_install=True, prompt=False) - assert calls == [("export.otlp", {"prompt": False})] - - -def test_is_available_does_not_install(monkeypatch): - # A pure availability check must NEVER trigger an install. - import tools.lazy_deps as ld - calls = [] - monkeypatch.setattr(ld, "ensure", lambda *a, **k: calls.append(a)) - OE.is_available() - assert calls == [] - - -def test_export_otlp_feature_specs_match_pyproject(): - # The LAZY_DEPS entry must track the [otlp] extra in pyproject.toml. - from tools.lazy_deps import feature_specs - import pathlib, re - specs = set(feature_specs("export.otlp")) - pyproject = pathlib.Path(__file__).resolve().parents[2] / "pyproject.toml" - text = pyproject.read_text(encoding="utf-8") - m = re.search(r"^otlp\s*=\s*\[([^\]]*)\]", text, re.MULTILINE) - assert m, "otlp extra not found in pyproject.toml" - extra = set(re.findall(r'"([^"]+)"', m.group(1))) - assert specs == extra, f"LAZY_DEPS {specs} != pyproject extra {extra}" - - -def test_streamer_subscription_receives_events(tmp_path, monkeypatch): - # Wire an OTLPStreamer-like subscriber via the in-memory provider. - provider, mem = _in_memory_provider() - - db = tmp_path / "state.db" - conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() - em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) - - def subscriber(batch): - OE.export_batch(provider, batch) - - em.subscribe(subscriber) - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed")) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", model="claude-opus-4")) - em.flush() - em.close() - spans = mem.get_finished_spans() - assert {s.name for s in spans} == {"hermes.run", "hermes.model_call"} - - -def test_failing_subscriber_never_breaks_hot_path(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() - em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) - - def bad_subscriber(batch): - time.sleep(0.2) - raise RuntimeError("OTLP collector down") - - em.subscribe(bad_subscriber) - start = time.monotonic() - for i in range(30): - em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=1)) - elapsed = time.monotonic() - start - # emit() returns immediately regardless of the broken subscriber - assert elapsed < 1.0 - em.flush() - # durable writes still happened despite the subscriber raising - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 30 - conn.close() - em.close() - - -def test_export_once_reads_db_and_returns_count(tmp_path, monkeypatch): - db = tmp_path / "state.db" - conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close() - em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db) - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed", - start_ns=time.time_ns(), end_ns=time.time_ns())) - em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok")) - em.flush(); em.close() - - # Patch the provider builder to an in-memory one (no network). The processor - # stand-in only needs force_flush(); provider.shutdown() works on the real one. - provider, mem = _in_memory_provider() - - class _Proc: - def force_flush(self, *a, **k): - return True - - monkeypatch.setattr(OE, "_make_provider", lambda config: (provider, _Proc())) - n = OE.export_once({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}, db_path=db) - assert n == 2 - assert len(mem.get_finished_spans()) == 2 diff --git a/tests/telemetry/test_plugin_autoload.py b/tests/telemetry/test_plugin_autoload.py deleted file mode 100644 index 9cfe5c92a83fb..0000000000000 --- a/tests/telemetry/test_plugin_autoload.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Telemetry plugin auto-load gating: on by default, off when telemetry.local=false.""" - -from __future__ import annotations - -import hermes_cli.plugins as plugins_mod - - -def test_local_enabled_defaults_true(monkeypatch): - monkeypatch.setattr(plugins_mod, "load_config", lambda: {}, raising=False) - # With no telemetry section, the default is on. - monkeypatch.setattr( - "hermes_cli.config.load_config", lambda: {}, raising=False - ) - assert plugins_mod._telemetry_local_enabled() is True - - -def test_local_disabled_when_config_false(monkeypatch): - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": {"local": False}}, - raising=False, - ) - assert plugins_mod._telemetry_local_enabled() is False - - -def test_local_enabled_when_config_true(monkeypatch): - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": {"local": True}}, - raising=False, - ) - assert plugins_mod._telemetry_local_enabled() is True - - -def test_malformed_config_defaults_on(monkeypatch): - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": "not a dict"}, - raising=False, - ) - assert plugins_mod._telemetry_local_enabled() is True - - -def test_plugin_manifest_is_discoverable(): - """The bundled telemetry plugin.yaml exists and declares the lifecycle hooks.""" - from pathlib import Path - import hermes_cli.plugins as p - bundled = p.get_bundled_plugins_dir() - manifest = bundled / "telemetry" / "plugin.yaml" - assert manifest.exists(), f"missing {manifest}" - text = manifest.read_text(encoding="utf-8") - for hook in ("post_api_request", "post_tool_call", "on_session_finalize"): - assert hook in text diff --git a/tests/telemetry/test_plugin_e2e.py b/tests/telemetry/test_plugin_e2e.py deleted file mode 100644 index 9a9ed00913d43..0000000000000 --- a/tests/telemetry/test_plugin_e2e.py +++ /dev/null @@ -1,127 +0,0 @@ -"""End-to-end telemetry wiring test. - -Unlike test_plugin_hooks.py (which calls the plugin's ``_on_*`` callbacks directly), -this drives the REAL dispatch chain that core uses at runtime: - - discover_plugins() -> plugin registers hooks -> invoke_hook(name, **kwargs) - -> registered callback -> emitter -> tel_* tables - -If the bundled plugin stops auto-loading, stops registering a hook, or the hook name -drifts from what core fires, the hand-written hook tests still pass but real runs go -dark. This test is the guard against that — it only touches public entry points -(``discover_plugins`` / ``invoke_hook``), exactly as core does. -""" - -from __future__ import annotations - -import sqlite3 -import time - -import pytest - -import hermes_state - - -@pytest.fixture -def runtime(tmp_path, monkeypatch): - """A clean HERMES_HOME with state.db, a fresh plugin manager, and a fresh emitter.""" - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - db = tmp_path / "state.db" - hermes_state.SessionDB(db_path=db) - - # Reset the global plugin-manager singleton so discovery re-runs in this HERMES_HOME. - import hermes_cli.plugins as plugins_mod - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - - # Reset the emitter singleton so it binds to this state.db (and tear it down after). - from agent.telemetry import emitter as emitter_mod - emitter_mod.reset_emitter_for_tests(None) - # Clear the plugin's per-run accumulators between tests. - import plugins.telemetry as plug - plug._runs.clear() - - yield db, plugins_mod, emitter_mod, plug - - try: - emitter_mod.get_emitter().flush() - except Exception: - pass - emitter_mod.reset_emitter_for_tests(None) - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - - -def _fire_one_turn(invoke_hook): - """Fire the hook sequence of a single completed turn, as core does.""" - invoke_hook("on_session_start", session_id="s1", - model="anthropic/claude-opus-4", platform="cli") - invoke_hook("post_api_request", session_id="s1", platform="cli", - provider="anthropic", base_url=None, model="claude-opus-4", - api_duration=0.9, - usage={"input_tokens": 1000, "output_tokens": 120, - "cache_read_tokens": 0, "cache_write_tokens": 0, - "reasoning_tokens": 0}) - invoke_hook("post_tool_call", session_id="s1", platform="cli", - function_name="web_search", duration_ms=210, result='{"data": "ok"}') - invoke_hook("on_session_finalize", session_id="s1", platform="cli", - reason="shutdown") - - -def test_real_dispatch_writes_tel_rows(runtime): - """The bundled plugin, loaded via discover_plugins, captures a turn end to end.""" - db, plugins_mod, emitter_mod, _plug = runtime - - plugins_mod.discover_plugins(force=True) - - # The plugin must have registered the lifecycle hooks core fires. - mgr = plugins_mod.get_plugin_manager() - registered = {k for k, v in getattr(mgr, "_hooks", {}).items() if v} - for hook in ("on_session_start", "post_api_request", "post_tool_call", - "on_session_finalize"): - assert hook in registered, f"core hook {hook!r} not registered by the plugin" - - _fire_one_turn(plugins_mod.invoke_hook) - - time.sleep(0.5) # let the background writer drain - emitter_mod.get_emitter().flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - assert conn.execute("SELECT COUNT(*) c FROM tel_runs").fetchone()["c"] == 1 - assert conn.execute("SELECT COUNT(*) c FROM tel_model_calls").fetchone()["c"] == 1 - assert conn.execute("SELECT COUNT(*) c FROM tel_tool_calls").fetchone()["c"] == 1 - - # Real values, not buckets. - mc = conn.execute("SELECT provider, model FROM tel_model_calls").fetchone() - assert mc["provider"] == "anthropic" - assert mc["model"] == "claude-opus-4" - tc = conn.execute("SELECT tool_name FROM tel_tool_calls").fetchone() - assert tc["tool_name"] == "web_search" - run = conn.execute("SELECT end_reason, model_call_count, tool_call_count " - "FROM tel_runs").fetchone() - assert run["end_reason"] == "completed" - assert run["model_call_count"] == 1 - assert run["tool_call_count"] == 1 - conn.close() - - -def test_local_disabled_writes_nothing(runtime, monkeypatch): - """telemetry.local=false: the plugin does not auto-load, so no rows are written.""" - db, plugins_mod, emitter_mod, _plug = runtime - monkeypatch.setattr( - "hermes_cli.config.load_config", - lambda: {"telemetry": {"local": False}}, - raising=False, - ) - - plugins_mod.discover_plugins(force=True) - _fire_one_turn(plugins_mod.invoke_hook) - time.sleep(0.3) - try: - emitter_mod.get_emitter().flush() - except Exception: - pass - - conn = sqlite3.connect(db) - assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0 - assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 0 - conn.close() diff --git a/tests/telemetry/test_plugin_hooks.py b/tests/telemetry/test_plugin_hooks.py deleted file mode 100644 index 6b1d1b2efbf7c..0000000000000 --- a/tests/telemetry/test_plugin_hooks.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Telemetry plugin hook tests — feed realistic kwargs, assert tel_* rows + no leak.""" - -from __future__ import annotations - -import sqlite3 - -import pytest - -import hermes_state -from agent.telemetry import emitter as emitter_mod -from agent.telemetry.emitter import TelemetryEmitter - - -@pytest.fixture -def wired(tmp_path, monkeypatch): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db) - emitter_mod.reset_emitter_for_tests(em) - # reset the plugin's per-run accumulators between tests - import plugins.telemetry as plug - plug._runs.clear() - yield db, em, plug - em.flush() - em.close() - emitter_mod.reset_emitter_for_tests(None) - - -def test_full_session_lifecycle_produces_rows(wired): - db, em, plug = wired - - plug._on_session_start(session_id="sess1", platform="telegram") - plug._on_post_api_request( - session_id="sess1", platform="telegram", - provider="anthropic", base_url=None, model="claude-opus-4", - api_duration=2.5, - usage={"input_tokens": 5000, "output_tokens": 800, "cache_read_tokens": 1000, - "cache_write_tokens": 0, "reasoning_tokens": 0}, - ) - plug._on_post_tool_call( - session_id="sess1", platform="telegram", - function_name="web_search", duration_ms=812, result="{\"data\": \"...\"}", - ) - # Production finalize callers pass `reason` (e.g. "shutdown"), not cost. - plug._on_session_finalize( - session_id="sess1", platform="telegram", reason="shutdown", - ) - em.flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - run = conn.execute("SELECT * FROM tel_runs").fetchone() - assert run is not None - assert run["entrypoint"] == "gateway" - assert run["platform"] == "telegram" - assert run["end_reason"] == "completed" - assert run["model_call_count"] == 1 - assert run["tool_call_count"] == 1 - - mc = conn.execute("SELECT * FROM tel_model_calls").fetchone() - assert mc["provider"] == "anthropic" - assert mc["model"] == "claude-opus-4" - assert mc["input_tokens"] == 5000 - - tc = conn.execute("SELECT * FROM tel_tool_calls").fetchone() - assert tc["tool_name"] == "web_search" - assert tc["result_class"] == "ok" - conn.close() - - -def test_tool_error_result_classified_and_counted(wired): - db, em, plug = wired - plug._on_session_start(session_id="s2", platform="cli") - plug._on_post_tool_call( - session_id="s2", function_name="terminal", duration_ms=10, - result="{\"error\": \"command failed\"}", - ) - plug._on_session_finalize(session_id="s2", reason="shutdown") - em.flush() - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - tc = conn.execute("SELECT result_class, tool_name FROM tel_tool_calls").fetchone() - assert tc["result_class"] == "error" - assert tc["tool_name"] == "terminal" - run = conn.execute("SELECT error_count FROM tel_runs").fetchone() - assert run["error_count"] == 1 - conn.close() - - -def test_api_error_recorded(wired): - db, em, plug = wired - plug._on_session_start(session_id="s3", platform="cli") - plug._on_api_request_error(session_id="s3", error_type="provider timeout after 60s") - plug._on_session_finalize(session_id="s3", failed=True) - em.flush() - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - err = conn.execute("SELECT error_class, subsystem FROM tel_error_events").fetchone() - assert err["error_class"] == "provider_timeout" - assert err["subsystem"] == "model_api" - run = conn.execute("SELECT end_reason FROM tel_runs").fetchone() - assert run["end_reason"] == "failed" - conn.close() - - -def test_hooks_never_raise_on_garbage_kwargs(wired): - _, em, plug = wired - # Missing everything — must be swallowed by the _safe wrapper. - plug._on_post_api_request() - plug._on_post_tool_call() - plug._on_session_finalize() - plug._on_api_request_error() - em.flush() - - -def test_no_message_content_in_tool_rows(wired): - """The tool hook receives a result blob; only the classification persists, not content.""" - db, em, plug = wired - plug._on_session_start(session_id="s4", platform="cli") - secret = "{\"data\": \"USER SECRET sk-ABCDEF and /Users/alice/file.txt\"}" - plug._on_post_tool_call(session_id="s4", function_name="web_search", - duration_ms=5, result=secret) - plug._on_session_finalize(session_id="s4") - em.flush() - # The whole tel_tool_calls row must contain none of the result content. - conn = sqlite3.connect(db) - row = conn.execute("SELECT * FROM tel_tool_calls").fetchone() - conn.close() - blob = " ".join(str(x) for x in row) - assert "sk-ABCDEF" not in blob - assert "/Users/alice" not in blob - assert "SECRET" not in blob diff --git a/tests/telemetry/test_policy_consent.py b/tests/telemetry/test_policy_consent.py deleted file mode 100644 index 7205c33b27518..0000000000000 --- a/tests/telemetry/test_policy_consent.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Consent gate tests. - -Consent is a single config field (``telemetry.consent_state``); the aggregate opt-in -is expressed by setting it to ``"aggregate"`` (via ``hermes config set`` or a -managed-scope pin). ``allow_aggregate`` is the hard gate. ``policy.may_upload_aggregate`` -is the gate a future uploader must consult. -""" - -from __future__ import annotations - -from agent.telemetry import policy - - -def _cfg(**telemetry): - return {"telemetry": telemetry} - - -def test_default_posture_never_uploads(): - # No consent recorded → unknown → never uploads. - assert policy.may_upload_aggregate(_cfg(local=True, consent_state="unknown")) is False - - -def test_missing_telemetry_block_never_uploads(): - assert policy.may_upload_aggregate({}) is False - - -def test_opted_in_uploads(): - assert policy.may_upload_aggregate(_cfg(consent_state="aggregate")) is True - - -def test_declined_does_not_upload(): - assert policy.may_upload_aggregate(_cfg(consent_state="local")) is False - - -def test_allow_aggregate_false_overrides_opt_in(): - # An admin pins telemetry.allow_aggregate: false via managed scope. - cfg = _cfg(consent_state="aggregate", allow_aggregate=False) - assert policy.may_upload_aggregate(cfg) is False # the hard gate wins - - -def test_local_off_makes_aggregate_inert(): - # Aggregate metrics derive from the local tables; with local off there is - # nothing to aggregate, so opting in cannot upload. - cfg = _cfg(local=False, consent_state="aggregate", allow_aggregate=True) - assert policy.may_upload_aggregate(cfg) is False - - -def test_install_id_minted_when_empty_and_stable_when_set(): - cfg = _cfg(install_id="") - minted = policy.ensure_install_id(cfg) - assert minted and len(minted) >= 32 # uuid4 - cfg2 = _cfg(install_id="fixed-id") - assert policy.ensure_install_id(cfg2) == "fixed-id" diff --git a/tests/telemetry/test_rollup.py b/tests/telemetry/test_rollup.py deleted file mode 100644 index b271bb916f9e6..0000000000000 --- a/tests/telemetry/test_rollup.py +++ /dev/null @@ -1,88 +0,0 @@ -"""rollup tests: tel_* -> per-run summary events with REAL values (local only).""" - -from __future__ import annotations - -import sqlite3 -import time - -import hermes_state -from agent.telemetry import rollup -from agent.telemetry.emitter import TelemetryEmitter -from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent - - -def _seed(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db) - now = time.time_ns() - em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway", - platform="telegram", end_reason="completed", - start_ns=now - 90_000_000, end_ns=now, - model_call_count=2, tool_call_count=2)) - em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", - model="claude-opus-4", input_tokens=60000, output_tokens=8000)) - em.emit(ModelCallEvent(span_id="m2", run_id="r1", provider="anthropic", - model="claude-opus-4", input_tokens=5000, output_tokens=500)) - em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search", - result_class="ok")) - em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate", - result_class="ok")) - # an in-progress run (no end_ns) must be excluded - em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", start_ns=now)) - em.flush() - em.close() - return db - - -def test_builds_one_event_per_completed_run_with_real_values(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db, - include_heartbeat=False) - wf = [e for e in events if e["event_name"] == "workflow_completed"] - assert len(wf) == 1 # r2 (no end_ns) excluded - e = wf[0] - assert e["entrypoint"] == "gateway" - assert e["platform"] == "telegram" - # REAL model id + provider, not a bucket/class - models = {m["model"] for m in e["models_used"]} - assert models == {"claude-opus-4"} - assert e["models_used"][0]["provider"] == "anthropic" - assert sorted(e["tools_used"]) == ["browser_navigate", "web_search"] - # real token totals, not buckets - assert e["input_tokens"] == 65000 - assert e["output_tokens"] == 8500 - - -def test_real_model_and_tool_names_present(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) - blob = " ".join(str(v) for e in events for v in e.values()) - assert "claude-opus-4" in blob - assert "web_search" in blob - - -def test_heartbeat_included_by_default(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) - assert any(e["event_name"] == "heartbeat" for e in events) - - -def test_summarize_counts_by_event_name(tmp_path): - db = _seed(tmp_path) - events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db) - s = rollup.summarize(events) - assert s["total"] == len(events) - assert s["by_event_name"]["workflow_completed"] == 1 - assert s["by_event_name"]["heartbeat"] == 1 - - -def test_empty_db_yields_only_heartbeat(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.close() - events = rollup.build_aggregate_events(install_id="x", db_path=db) - assert [e["event_name"] for e in events] == ["heartbeat"] diff --git a/tests/telemetry/test_schema_migration.py b/tests/telemetry/test_schema_migration.py deleted file mode 100644 index a5a946da4b040..0000000000000 --- a/tests/telemetry/test_schema_migration.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Schema tests: tel_* tables exist after init; SCHEMA_VERSION bumped.""" - -from __future__ import annotations - -import sqlite3 - -import hermes_state - - -TEL_TABLES = { - "tel_runs", "tel_spans", "tel_model_calls", "tel_tool_calls", - "tel_error_events", -} - - -def test_schema_version_is_17_or_higher(): - assert hermes_state.SCHEMA_VERSION >= 17 - - -def test_tel_tables_present_in_schema_sql(): - for tbl in TEL_TABLES: - assert f"CREATE TABLE IF NOT EXISTS {tbl}" in hermes_state.SCHEMA_SQL - - -def test_tel_tables_created_on_executescript(tmp_path): - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - rows = { - r[0] - for r in conn.execute( - "SELECT name FROM sqlite_master WHERE type='table'" - ).fetchall() - } - conn.close() - assert TEL_TABLES.issubset(rows), f"missing: {TEL_TABLES - rows}" - - -def test_executescript_is_idempotent(tmp_path): - # IF NOT EXISTS means re-running on an existing DB is a no-op, not an error. - db = tmp_path / "state.db" - conn = sqlite3.connect(db) - conn.executescript(hermes_state.SCHEMA_SQL) - conn.executescript(hermes_state.SCHEMA_SQL) # second run must not raise - conn.close() diff --git a/tests/telemetry/test_spans_trace.py b/tests/telemetry/test_spans_trace.py deleted file mode 100644 index c99fb02fb68b8..0000000000000 --- a/tests/telemetry/test_spans_trace.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Trace/span layer: tel_spans is populated as a connected run -> calls tree. - -Drives the real dispatch chain (discover_plugins -> invoke_hook) and asserts the -timing/lineage backbone in tel_spans: - - one root span per run (kind="run", parent_span_id NULL), - - one child span per model/tool call parented to the root, - - a single trace_id across the run, - - call detail rows (tel_model_calls / tel_tool_calls) JOIN to their span by span_id, - - reconstructed durations match the reported latency/duration. - -This is the regression guard for the waterfall a desktop trace viewer renders. -""" - -from __future__ import annotations - -import sqlite3 -import time - -import pytest - -import hermes_state - - -@pytest.fixture -def runtime(tmp_path, monkeypatch): - monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - db = tmp_path / "state.db" - hermes_state.SessionDB(db_path=db) - import hermes_cli.plugins as plugins_mod - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - from agent.telemetry import emitter as emitter_mod - emitter_mod.reset_emitter_for_tests(None) - import plugins.telemetry as plug - plug._runs.clear() - yield db, plugins_mod, emitter_mod - try: - emitter_mod.get_emitter().flush() - except Exception: - pass - emitter_mod.reset_emitter_for_tests(None) - monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False) - - -def _one_turn(invoke_hook): - invoke_hook("on_session_start", session_id="s1", - model="anthropic/claude-opus-4", platform="cli") - invoke_hook("post_api_request", session_id="s1", platform="cli", - provider="anthropic", model="claude-opus-4", api_duration=0.9, - usage={"input_tokens": 1000, "output_tokens": 120}) - invoke_hook("post_tool_call", session_id="s1", platform="cli", - function_name="web_search", duration_ms=210, result='{"data": "ok"}') - invoke_hook("on_session_finalize", session_id="s1", platform="cli", - reason="shutdown") - - -def test_tel_spans_forms_connected_trace(runtime): - db, plugins_mod, emitter_mod = runtime - plugins_mod.discover_plugins(force=True) - _one_turn(plugins_mod.invoke_hook) - time.sleep(0.5) - emitter_mod.get_emitter().flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - spans = conn.execute( - "SELECT span_id, parent_span_id, kind, name, start_ns, end_ns, status, trace_id " - "FROM tel_spans" - ).fetchall() - - # root + model + tool - assert len(spans) == 3 - roots = [s for s in spans if s["parent_span_id"] is None] - children = [s for s in spans if s["parent_span_id"] is not None] - assert len(roots) == 1 - assert roots[0]["kind"] == "run" - assert len(children) == 2 - - # single trace, all children parented to the root - assert len({s["trace_id"] for s in spans}) == 1 - assert all(c["parent_span_id"] == roots[0]["span_id"] for c in children) - - # spans are time-ordered and carry real durations - by_kind = {s["kind"]: s for s in spans} - assert (by_kind["model"]["end_ns"] - by_kind["model"]["start_ns"]) == 900 * 1_000_000 - assert (by_kind["tool"]["end_ns"] - by_kind["tool"]["start_ns"]) == 210 * 1_000_000 - assert by_kind["run"]["end_ns"] >= by_kind["run"]["start_ns"] - - -def test_detail_rows_join_to_spans(runtime): - db, plugins_mod, emitter_mod = runtime - plugins_mod.discover_plugins(force=True) - _one_turn(plugins_mod.invoke_hook) - time.sleep(0.5) - emitter_mod.get_emitter().flush() - - conn = sqlite3.connect(db) - conn.row_factory = sqlite3.Row - mc = conn.execute( - "SELECT m.model, s.kind, s.trace_id FROM tel_model_calls m " - "JOIN tel_spans s ON m.span_id = s.span_id" - ).fetchone() - assert mc is not None and mc["model"] == "claude-opus-4" and mc["kind"] == "model" - - tc = conn.execute( - "SELECT t.tool_name, s.kind FROM tel_tool_calls t " - "JOIN tel_spans s ON t.span_id = s.span_id" - ).fetchone() - assert tc is not None and tc["tool_name"] == "web_search" and tc["kind"] == "tool" - conn.close() diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 3b94efdd4cb14..cd983db6d74ec 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -116,6 +116,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "search.firecrawl": ("firecrawl-py==4.17.0",), "search.parallel": ("parallel-web==0.4.2",), + # ─── Monitoring ───────────────────────────────────────────────────────── + # OTLP gateway monitoring export. Lazily installed on first use of + # monitoring.gateway_health_export / monitoring.export.otlp. Tracks the + # `otlp` extra in pyproject.toml — bump both together. + "export.otlp": ( + "opentelemetry-sdk==1.39.1", + "opentelemetry-exporter-otlp-proto-http==1.39.1", + ), + # ─── TTS providers ───────────────────────────────────────────────────── # Pinned to exact versions to match pyproject.toml's no-ranges policy # (see comment at top of [project.dependencies]). When bumping, update @@ -140,15 +149,6 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { # ─── Image generation backends ───────────────────────────────────────── "image.fal": ("fal-client==0.13.1",), - # ─── Observability ───────────────────────────────────────────────────── - # OTLP telemetry export. Lazily installed on first use of - # `hermes telemetry export --otlp`. Tracks the `otlp` extra in - # pyproject.toml — bump both together. - "export.otlp": ( - "opentelemetry-sdk==1.30.0", - "opentelemetry-exporter-otlp-proto-http==1.30.0", - ), - # ─── Memory providers ────────────────────────────────────────────────── "memory.honcho": ("honcho-ai==2.2.0",), "memory.hindsight": ("hindsight-client==0.6.1",), diff --git a/uv.lock b/uv.lock index 3b035b7e277ad..c91b8d4841b24 100644 --- a/uv.lock +++ b/uv.lock @@ -1415,7 +1415,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/58/5404031044f55afad7aad1aff8be3f22b1bed03e237cfeabbc7e5c8cfde0/greenlet-3.5.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:aca9b4ce85b152b5524ef7d88170efdff80dc0032aa8b75f9aaf7f3479ea95b4", size = 287424, upload-time = "2026-06-26T18:20:31.469Z" }, { url = "https://files.pythonhosted.org/packages/b4/bf/1c65e9b94a54d547068fa5b5a8a06f221f3316b48908e08668d29c77cb50/greenlet-3.5.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f71be4920368fe1fabeeaa53d1e3548337e2b223d9565f8ad5e392a75ba23fc", size = 606523, upload-time = "2026-06-26T19:07:08.859Z" }, { url = "https://files.pythonhosted.org/packages/b8/c7/b66baacc95775ad511287acb0137b95574a9ce5491902372b7564799d790/greenlet-3.5.3-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d77e67f65f98449e3fb83f795b5d0a8437aead2f874ca89c96576caf4be3af6", size = 618315, upload-time = "2026-06-26T19:10:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/b0/a0/68afd1ebad40db87dac0a28ffa120726b98bf9c7c40c481b0f63c105d298/greenlet-3.5.3-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e18619ba655ac05d78d80fc83cac4ba892bd6927b99e3b8237aee861aaacc8bb", size = 626155, upload-time = "2026-06-26T19:24:14.44Z" }, { url = "https://files.pythonhosted.org/packages/78/2b/28ed29463522fdbe4c15b1f63922041626a7478316b34ab4adda3f0a4aba/greenlet-3.5.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8540f1e6205bd13ca0ce685581037219ca54a1b41a0a15d228c6c9b8ad5903d7", size = 617381, upload-time = "2026-06-26T18:32:16.077Z" }, + { url = "https://files.pythonhosted.org/packages/07/7f/e327d912239ec4b3b49999e3967389bcf1ee8722b9ee9194d2752ecd558a/greenlet-3.5.3-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:d27c0c653a60d9535f690226474a5cc1036a8b0d7b57504d1c4f89c44a07a80c", size = 421083, upload-time = "2026-06-26T19:25:35.804Z" }, { url = "https://files.pythonhosted.org/packages/2a/7b/ad04e9d1337fc04965dc9fc616b6a72cb65a24b800a014c011ec812f5489/greenlet-3.5.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ef56fe650f50575bf843acde967b9c567687f3c22340941a899b7bc56e956a8", size = 1577771, upload-time = "2026-06-26T19:09:01.537Z" }, { url = "https://files.pythonhosted.org/packages/d8/33/6c87ab7ba663f70ca21f3022aad1ffe56d3f3e0521e836c2415e13abcc3c/greenlet-3.5.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5121af01cf911e70056c00d4b46d5e9b5d1415550038573d744138bacb59e6b8", size = 1644048, upload-time = "2026-06-26T18:31:42.996Z" }, { url = "https://files.pythonhosted.org/packages/1c/35/f0d8ee998b422cf8693b270f098e55d8d4ec8006b061b333f54f177d28d9/greenlet-3.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:0f41e4a05a3c0cb31b17023eff28dd111e1d16bf7d7d00406cd7df23f31398a7", size = 239137, upload-time = "2026-06-26T18:23:21.664Z" }, @@ -1423,7 +1425,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/6e/4c37d51a2b7f82d2ff11bb6b5f7d766d9a011726624af255e843727627a3/greenlet-3.5.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:719757059f5a53fd0dde23f78cffeafcdd97b21c850ddb7ca684a3c1a1f122e2", size = 288685, upload-time = "2026-06-26T18:22:08.977Z" }, { url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" }, { url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" }, + { url = "https://files.pythonhosted.org/packages/fb/fb/d97dc261209c80744b7c8132693a30d70ec6e7315e632cb0a10b3fec94dd/greenlet-3.5.3-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5795cd1101371140551c645f2d408b8d3c01a5a29cf8a9bce6e759c983682d23", size = 622351, upload-time = "2026-06-26T19:24:16.32Z" }, { url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ac/e5fee13cbbd0e8de312d9a146584b8a51891c68847330ef9dc8b5109d23f/greenlet-3.5.3-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:af4923b3096e26a36d7e9cf24ab88083a20f97d191e3b97f253731ce9b41b28c", size = 425395, upload-time = "2026-06-26T19:25:37.144Z" }, { url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" }, { url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" }, { url = "https://files.pythonhosted.org/packages/14/40/c57489acf8e37d74e2913d4eff63aa0dba17acccc4bdeef874dde2dbbec9/greenlet-3.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:cde8adafa2365676f74a979744629589999093bc86e2484214f58e61df08902c", size = 239882, upload-time = "2026-06-26T18:23:27.518Z" }, @@ -1431,7 +1435,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" }, { url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" }, { url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/57/61/2f5b1adf256d039f5dab8005de8d3d7ad2b0070a3219c0e036b3fbfeb440/greenlet-3.5.3-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ad04dd75458c6300b047c61b8639092433d205a25a14e310d6582a480efcca1", size = 625580, upload-time = "2026-06-26T19:24:18.344Z" }, { url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/ab7fc9e543e44d6879b0a6ef9a4b2188940fd180cc65d6f646883ddf7201/greenlet-3.5.3-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:afaabdd554cd7ae9bbb3ca070b0d7fdfd207dbf1d16865f7233837709d354bda", size = 427933, upload-time = "2026-06-26T19:25:38.219Z" }, { url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" }, { url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" }, { url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" }, @@ -1666,6 +1672,10 @@ modal = [ nemo-relay = [ { name = "nemo-relay" }, ] +otlp = [ + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, +] parallel-web = [ { name = "parallel-web" }, ] @@ -1808,6 +1818,8 @@ requires-dist = [ { name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = ">=0.5,<1.0" }, { name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" }, { name = "openai", specifier = "==2.24.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp'", specifier = "==1.39.1" }, + { name = "opentelemetry-sdk", marker = "extra == 'otlp'", specifier = "==1.39.1" }, { name = "packaging", specifier = "==26.0" }, { name = "parallel-web", marker = "extra == 'parallel-web'", specifier = "==0.4.2" }, { name = "pathspec", specifier = "==1.1.1" }, @@ -1855,7 +1867,7 @@ requires-dist = [ { name = "websockets", specifier = "==15.0.1" }, { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" }, ] -provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] +provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "supermemory", "mem0", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] [[package]] name = "hf-xet"