RomanNum3ral
/
hermes-agent
mirror of https://github.com/NousResearch/hermes-agent.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(monitoring): scope telemetry substrate to gateway health/diagnostics export 

Salvages the event-spine foundation from feat/telemetry-observability
(emitter, typed events, OTLP streaming, redaction — authorship preserved in
the preceding commits) and scopes it to the plane enterprise operators need
today: gateway Service Health Monitoring plus redacted Operational
Diagnostics, exported over OTLP.

Dropped from the salvaged branch, deliberately:
- run/model/tool trajectory capture (plugins/telemetry hooks, tel_spans)
- the local JSONL + state.db tel_* store (monitoring is egress, not storage)
- usage rollups/metrics, /insights integration, bulk export
- hermes telemetry CLI (replaced by hermes monitoring status)

Those planes — shared client usage metrics and enterprise trace telemetry —
are being designed on the NeMo Relay integration with distinct consent,
policy, and export boundaries; this keeps the monitoring plane content-free
and independently enableable.

Renames agent/telemetry -> agent/monitoring, config telemetry.* ->
monitoring.*, and pins the otlp extra at OpenTelemetry 1.39.1 (matching
uv.lock; 1.30.0 conflicts with mistralai>=2.4 on opentelemetry-api).
This commit is contained in:
Victor Kyriazakos 2026-07-14 18:00:46 +00:00 committed by Victor Kyriazakos
parent fdb5ae012a
commit 505d12f662
55 changed files with 2295 additions and 3833 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
agent/insights.py
View File

@ -82,19 +82,6 @@ def _bar_chart(values: List[int], max_width: int = 20) -> List[str]:
return ["█" * max(1, int(v / peak * max_width)) if v > 0 else "" for v in values]
def _fmt_ms(ms: float) -> str:
"""Compact human duration from milliseconds (e.g. 850ms, 2.4s, 1.5m)."""
try:
ms = float(ms or 0)
except (TypeError, ValueError):
return "0ms"
if ms < 1000:
return f"{int(ms)}ms"
if ms < 60_000:
return f"{ms / 1000:.1f}s"
return f"{ms / 60_000:.1f}m"
class InsightsEngine:
"""
Analyzes session history and produces usage insights.
@ -152,7 +139,6 @@ class InsightsEngine:
},
"activity": {},
"top_sessions": [],
"telemetry": {},
}
# Compute insights
@ -163,7 +149,6 @@ class InsightsEngine:
skills = self._compute_skill_breakdown(skill_usage)
activity = self._compute_activity_patterns(sessions)
top_sessions = self._compute_top_sessions(sessions)
telemetry = self._compute_telemetry(cutoff)
return {
"days": days,
@ -177,37 +162,8 @@ class InsightsEngine:
"skills": skills,
"activity": activity,
"top_sessions": top_sessions,
"telemetry": telemetry,
}
# =========================================================================
# Telemetry (observability) — from the tel_* tables (local telemetry)
# =========================================================================
def _compute_telemetry(self, cutoff: float) -> Dict[str, Any]:
"""Roll up the local telemetry tables for the same window.
Reuses the engine's existing connection. Fully fail-soft: if the tel_*
tables are empty or absent (telemetry.local disabled, fresh install), this
returns an empty dict and the renderer skips the section.
"""
try:
from agent.telemetry import metrics
except Exception:
return {}
try:
since_ns = int(cutoff * 1e9)
if not metrics.has_data(conn=self._conn):
return {}
return {
"workflows": metrics.workflow_summary(since_ns=since_ns, conn=self._conn),
"model_calls": metrics.model_call_summary(since_ns=since_ns, conn=self._conn),
"tool_calls": metrics.tool_call_summary(conn=self._conn),
"errors": metrics.error_summary(conn=self._conn),
}
except Exception:
return {}
# =========================================================================
# Data gathering (SQL queries)
# =========================================================================
@ -1067,80 +1023,8 @@ class InsightsEngine:
lines.append(f" {ts['label']:<20} {ts['value']:<18} ({ts['date']}, {ts['session_id']})")
lines.append("")
# Telemetry / observability (local telemetry) — only when data exists
tel = report.get("telemetry") or {}
if tel:
self._append_telemetry_section(lines, tel)
return "\n".join(lines)
def _append_telemetry_section(self, lines: List[str], tel: Dict[str, Any]) -> None:
"""Render the observability rollups (workflows, tools, providers, errors)."""
wf = tel.get("workflows", {})
mc = tel.get("model_calls", {})
tc = tel.get("tool_calls", {})
errs = tel.get("errors", {}).get("by_class", {})
lines.append(" 📡 Observability (local telemetry)")
lines.append(" " + "─" * 56)
total_runs = wf.get("total_runs", 0)
if total_runs:
sr = wf.get("success_rate", 0.0) * 100
p50 = wf.get("duration_ms_p50", 0)
p95 = wf.get("duration_ms_p95", 0)
lines.append(
f" Workflows: {total_runs:,} Success: {sr:.1f}% "
f"Duration p50/p95: {_fmt_ms(p50)} / {_fmt_ms(p95)}"
)
by_entry = wf.get("by_entrypoint", {})
if by_entry:
entry_str = ", ".join(
f"{k}: {v}" for k, v in sorted(by_entry.items(), key=lambda x: -x[1])
)
lines.append(f" Entrypoints: {entry_str}")
# Tool reliability
if tc.get("total"):
fail_pct = tc.get("failure_rate", 0.0) * 100
lines.append(
f" Tool calls: {tc['total']:,} Failure rate: {fail_pct:.1f}%"
)
tools = tc.get("by_tool", {})
fails = tc.get("failures_by_tool", {})
top = sorted(tools.items(), key=lambda x: -x[1])[:6]
if top:
parts = []
for name, n in top:
f = fails.get(name, 0)
parts.append(f"{name}: {n}" + (f" ({f} failed)" if f else ""))
lines.append(" " + " ".join(parts))
# Provider / model mix + cache (real names)
by_provider = mc.get("by_provider", {})
if by_provider:
prov_str = ", ".join(
f"{k}: {v}" for k, v in sorted(by_provider.items(), key=lambda x: -x[1])
)
lines.append(f" Providers: {prov_str}")
by_model = mc.get("by_model", {})
if by_model:
model_str = ", ".join(
f"{k}: {v}" for k, v in sorted(by_model.items(), key=lambda x: -x[1])[:8]
)
cache = mc.get("cache_hit_rate", 0.0) * 100
suffix = f" Cache hit: {cache:.1f}%" if cache else ""
lines.append(f" Models: {model_str}{suffix}")
# Error classes
if errs:
err_str = ", ".join(
f"{k}: {v}" for k, v in sorted(errs.items(), key=lambda x: -x[1])[:6]
)
lines.append(f" Errors: {err_str}")
lines.append("")
def format_gateway(self, report: Dict) -> str:
"""Format the insights report for gateway/messaging (shorter)."""
if report.get("empty"):

29
agent/monitoring/__init__.py Normal file
View File

@ -0,0 +1,29 @@
"""Hermes gateway monitoring.
Service health monitoring plus redacted operational diagnostics for the
gateway daemon, exported over OTLP to an operator-configured endpoint.
``emitter`` is the in-process event bus: producers (gateway status hooks,
the diagnostic log handler) hand typed events to a fire-and-forget queue,
and subscribers (the OTLP streamers) consume them off the hot path. The
emitter never blocks or raises into gateway code (the hot-path invariant),
and nothing is persisted locally — monitoring is an egress path, not a store.
Deliberately out of scope here: run/model/tool trajectory capture, usage
analytics, and any content-bearing signal. Those planes are served by the
NeMo Relay integration and its Hermes-owned subscribers.
"""
from __future__ import annotations
from . import emitter, events
emit = emitter.emit
get_emitter = emitter.get_emitter
__all__ = [
"emitter",
"events",
"emit",
"get_emitter",
]

193
agent/monitoring/emitter.py Normal file
View File

@ -0,0 +1,193 @@
"""Monitoring emitter: fire-and-forget queue + background dispatcher.
The emitter is the single seam between producers (gateway status hooks, the
diagnostic log handler) and consumers (the OTLP streamers). Its contract is
the hot-path invariant:
``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network,
and MUST NEVER raise into the caller. A monitoring failure is logged
locally and dropped — it can never affect the gateway or a session.
Mechanism:
* ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare
except. On a full queue it drops the *oldest* event and counts the drop.
* A daemon thread drains the queue and fans each batch out to subscribers
(the OTLP metric/span/log streamers). Each subscriber is fail-isolated —
a slow or raising subscriber never affects the hot path or its peers.
Nothing is persisted here. Monitoring is an egress path, not a local store;
if no subscriber is attached, events simply age out of the ring buffer.
"""
from __future__ import annotations
import logging
import queue
import threading
import time
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full
_DRAIN_BATCH = 256
class MonitoringEmitter:
"""Owns the queue, the dispatcher thread, and the subscriber list."""
def __init__(self, *, enabled: bool = True) -> None:
self._enabled = enabled
self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE)
self._dropped = 0
self._dispatched = 0
self._stop = threading.Event()
self._started = False
self._lock = threading.Lock()
self._thread: Optional[threading.Thread] = None
# Live subscribers (the OTLP streamers). Called from the dispatcher
# thread, fully fail-isolated. Each subscriber is callable(batch: list[dict]).
self._subscribers: list = []
# ── public API (hot path) ───────────────────────────────────────────────
def emit(self, event: Any) -> None:
"""Enqueue an event. Never blocks, never raises.
``event`` may be a dataclass with ``to_dict()`` or a plain dict.
"""
if not self._enabled:
return
try:
payload = event.to_dict() if hasattr(event, "to_dict") else dict(event)
payload.setdefault("ts_ns", time.time_ns())
self._ensure_started()
try:
self._q.put_nowait(payload)
except queue.Full:
# Drop oldest to make room — bounded memory, newest-wins.
try:
self._q.get_nowait()
self._dropped += 1
self._q.put_nowait(payload)
except Exception:
self._dropped += 1
except Exception: # the hot-path invariant: never propagate
logger.debug("monitoring emit failed", exc_info=True)
# ── lifecycle ───────────────────────────────────────────────────────────
def _ensure_started(self) -> None:
if self._started:
return
with self._lock:
if self._started:
return
self._thread = threading.Thread(
target=self._run, name="hermes-monitoring-dispatch", daemon=True
)
self._thread.start()
self._started = True
def _run(self) -> None:
while not self._stop.is_set():
try:
first = self._q.get(timeout=0.5)
except queue.Empty:
continue
batch = [first]
while len(batch) < _DRAIN_BATCH:
try:
batch.append(self._q.get_nowait())
except queue.Empty:
break
self._dispatch(batch)
def _dispatch(self, batch) -> None:
# Fan-out to subscribers (OTLP streamers) — fully fail-isolated.
for sub in list(self._subscribers):
try:
sub(batch)
except Exception:
logger.debug("monitoring subscriber failed", exc_info=True)
self._dispatched += len(batch)
def subscribe(self, callback) -> None:
"""Register a live batch subscriber (callable(batch: list[dict]))."""
if callback not in self._subscribers:
self._subscribers.append(callback)
def unsubscribe(self, callback) -> None:
try:
self._subscribers.remove(callback)
except ValueError:
pass
# ── introspection / shutdown (tests, CLI) ───────────────────────────────
def flush(self, timeout: float = 2.0) -> None:
"""Block until the queue drains (test/CLI helper, NOT the hot path)."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self._q.empty():
# give the dispatcher a tick to finish the in-flight batch
time.sleep(0.05)
if self._q.empty():
return
time.sleep(0.02)
def stats(self) -> Dict[str, int]:
return {
"queued": self._q.qsize(),
"dispatched": self._dispatched,
"dropped": self._dropped,
"subscribers": len(self._subscribers),
}
def close(self) -> None:
self._stop.set()
if self._thread is not None:
self._thread.join(timeout=2.0)
self._started = False
# ── process-wide singleton ──────────────────────────────────────────────────
_EMITTER: Optional[MonitoringEmitter] = None
_EMITTER_LOCK = threading.Lock()
def get_emitter() -> MonitoringEmitter:
"""Return the process-wide monitoring emitter."""
global _EMITTER
if _EMITTER is not None:
return _EMITTER
with _EMITTER_LOCK:
if _EMITTER is None:
_EMITTER = MonitoringEmitter()
return _EMITTER
def emit(event: Any) -> None:
"""Module-level convenience: emit via the singleton."""
get_emitter().emit(event)
def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None:
"""Swap the singleton (tests only)."""
global _EMITTER
with _EMITTER_LOCK:
if _EMITTER is not None and emitter is not _EMITTER:
try:
_EMITTER.close()
except Exception:
pass
_EMITTER = emitter
# Back-compat alias for the salvaged class name used in emozilla's tests.
TelemetryEmitter = MonitoringEmitter
__all__ = [
"MonitoringEmitter",
"TelemetryEmitter",
"get_emitter",
"emit",
"reset_emitter_for_tests",
]

69
agent/monitoring/events.py Normal file
View File

@ -0,0 +1,69 @@
"""Typed gateway monitoring events.
Content-free service-health and redacted diagnostic events for the gateway
daemon. These are the only event shapes the monitoring plane emits: no
prompts, messages, tool args/results, session history, or usage analytics.
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field, asdict
from typing import Any, Dict, Optional
def _now_ns() -> int:
return time.time_ns()
@dataclass(slots=True)
class GatewayHealthEvent:
"""Content-free gateway health snapshot or lifecycle event."""
name: str
gateway_state: Optional[str] = None
old_state: Optional[str] = None
new_state: Optional[str] = None
exit_reason: Optional[str] = None
restart_requested: Optional[bool] = None
active_agents: int = 0
gateway_busy: bool = False
gateway_drainable: bool = False
platform_count: int = 0
fatal_platform_count: int = 0
profile: Optional[str] = None
install_id: Optional[str] = None
version: Optional[str] = None
supervision_mode: Optional[str] = None
pid: Optional[int] = None
ts_ns: int = field(default_factory=_now_ns)
def to_dict(self) -> Dict[str, Any]:
return {"event": "gateway_health", **asdict(self)}
@dataclass(slots=True)
class GatewayDiagnosticEvent:
"""Redacted gateway diagnostic event for operator-owned observability."""
name: str
subsystem: str
error_class: str = "unknown"
error_code: Optional[str] = None
redacted_message: Optional[str] = None
platform: Optional[str] = None
old_state: Optional[str] = None
new_state: Optional[str] = None
profile: Optional[str] = None
version: Optional[str] = None
severity: str = "warning"
ts_ns: int = field(default_factory=_now_ns)
def to_dict(self) -> Dict[str, Any]:
return {"event": "gateway_diagnostic", **asdict(self)}
__all__ = [
"GatewayHealthEvent",
"GatewayDiagnosticEvent",
]

388
agent/monitoring/gateway_health.py Normal file
View File

@ -0,0 +1,388 @@
"""Gateway health and diagnostics signal producer.
This module keeps the plane narrow: service health monitoring plus
redacted operational diagnostics. It reuses the existing gateway runtime-status
contract and emits content-free metrics/events. No prompts, messages, tool args,
session history, audit records, or product analytics belong here.
"""
from __future__ import annotations
import logging
import re
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent
@dataclass(frozen=True, slots=True)
class GatewayMetric:
name: str
value: int | float
attributes: Dict[str, str]
@dataclass(frozen=True, slots=True)
class GatewayHealthSnapshot:
metrics: List[GatewayMetric]
events: List[GatewayHealthEvent | GatewayDiagnosticEvent]
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}")
_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE)
_TOKEN_RE = re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b")
_SECRET_LITERAL_RE = re.compile(r"\*{3,}")
_PHONE_RE = re.compile(r"(?<!\w)(?:\+?\d{1,3}[\s.\-]?)?(?:\(\d{2,4}\)[\s.\-]?)?\d{3}[\s.\-]?\d{3,4}(?:[\s.\-]?\d{2,4})?(?!\w)")
_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
def _allowed_logger(name: str) -> bool:
return name == "gateway" or name.startswith("gateway.")
def redact_gateway_message(message: Any) -> str:
"""Redact gateway diagnostic free text for customer-owned export."""
text = str(message or "")
try:
from agent.monitoring.redaction import redact_for_export
redacted = redact_for_export(text, content_mode="pii") or ""
except Exception:
redacted = "[redaction-unavailable]"
redacted = _BEARER_RE.sub("[redacted]", redacted)
redacted = _TOKEN_RE.sub("[redacted]", redacted)
redacted = _SECRET_LITERAL_RE.sub("[redacted]", redacted)
redacted = re.sub(r"\bBearer\s+\[[^\]]+\]", "[redacted]", redacted, flags=re.IGNORECASE)
redacted = _EMAIL_RE.sub("[email]", redacted)
redacted = _PHONE_RE.sub("[phone]", redacted)
return redacted[:500]
def classify_gateway_error(raw: Any) -> str:
s = str(raw or "").lower()
if any(k in s for k in ("auth", "token", "unauthorized", "forbidden", "401", "403")):
return "auth_failed"
if "rate" in s and "limit" in s:
return "rate_limited"
if "timeout" in s or "timed out" in s:
return "timeout"
if any(k in s for k in ("network", "connection", "dns", "socket")):
return "network_error"
if any(k in s for k in ("config", "missing", "invalid")):
return "invalid_config"
if "startup" in s:
return "startup_failed"
if "fatal" in s:
return "platform_fatal"
return "unknown"
def subsystem_for_logger(logger_name: str) -> str:
if logger_name.startswith("gateway.platforms."):
parts = logger_name.split(".")
if len(parts) >= 3 and parts[2]:
return f"platform.{parts[2]}"
if logger_name.startswith("gateway.platforms"):
return "platform"
if logger_name.startswith("gateway"):
return "gateway"
return "gateway"
def platform_for_subsystem(subsystem: str) -> Optional[str]:
if subsystem.startswith("platform."):
return subsystem.split(".", 1)[1] or None
return None
def _parse_active_agents(raw: Any) -> int:
try:
from gateway.status import parse_active_agents
return parse_active_agents(raw)
except Exception:
try:
return max(0, int(raw))
except (TypeError, ValueError):
return 0
def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool:
try:
from gateway.status import derive_gateway_busy
return derive_gateway_busy(
gateway_running=gateway_running,
gateway_state=gateway_state,
active_agents=active_agents,
)
except Exception:
return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0)
def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool:
try:
from gateway.status import derive_gateway_drainable
return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state)
except Exception:
return bool(gateway_running and gateway_state == "running")
def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]:
return {
"hermes.profile": str(profile),
"service.instance.id": str(install_id),
"service.version": str(version),
"hermes.supervision_mode": str(supervision_mode),
}
def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric:
out = dict(attrs)
for key, val in extra.items():
if val is not None:
out[key] = str(val)
return GatewayMetric(name=name, value=value, attributes=out)
def build_gateway_health_snapshot(
runtime: Optional[dict[str, Any]],
*,
gateway_running: bool,
profile: str,
install_id: str,
version: str,
supervision_mode: str = "unknown",
) -> GatewayHealthSnapshot:
"""Convert gateway_state.json-compatible runtime state into P0 signals."""
runtime = runtime or {}
gateway_state = runtime.get("gateway_state")
active_agents = _parse_active_agents(runtime.get("active_agents", 0))
busy = _derive_busy(gateway_running, gateway_state, active_agents)
drainable = _derive_drainable(gateway_running, gateway_state)
platforms = runtime.get("platforms") if isinstance(runtime.get("platforms"), dict) else {}
base = _base_attrs(profile=profile, install_id=install_id, version=version, supervision_mode=supervision_mode)
metrics: list[GatewayMetric] = [
_metric("hermes.gateway.up", 1 if gateway_running else 0, base),
_metric("hermes.gateway.active_agents", active_agents, base),
_metric("hermes.gateway.busy", 1 if busy else 0, base),
_metric("hermes.gateway.drainable", 1 if drainable else 0, base),
_metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base),
]
if gateway_state:
metrics.append(_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": str(gateway_state)}))
fatal_count = 0
events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
for platform, pdata in platforms.items():
pdata = pdata if isinstance(pdata, dict) else {}
state = str(pdata.get("state") or "unknown").lower()
raw_error = pdata.get("error_code") or pdata.get("error_message")
error_code = classify_gateway_error(raw_error)
is_up = state in _RUNNING_PLATFORM_STATES
is_degraded = state in _FATAL_PLATFORM_STATES
if is_degraded:
fatal_count += 1
metrics.append(_metric(
"hermes.platform.up",
1 if is_up else 0,
base,
**{"hermes.platform": str(platform), "hermes.platform.state": state},
))
metrics.append(_metric(
"hermes.platform.degraded",
1 if is_degraded else 0,
base,
**{"hermes.platform": str(platform), "hermes.platform.state": state, "hermes.error_code": error_code},
))
if is_degraded:
events.append(GatewayDiagnosticEvent(
name="platform.fatal",
subsystem=f"platform.{platform}",
platform=str(platform),
error_code=error_code,
error_class=classify_gateway_error(error_code or pdata.get("error_message")),
redacted_message=redact_gateway_message(pdata.get("error_message")),
profile=profile,
version=version,
severity="error" if state == "fatal" else "warning",
))
events.insert(0, GatewayHealthEvent(
name="gateway.health_snapshot",
gateway_state=str(gateway_state) if gateway_state is not None else None,
active_agents=active_agents,
gateway_busy=busy,
gateway_drainable=drainable,
platform_count=len(platforms),
fatal_platform_count=fatal_count,
profile=profile,
install_id=install_id,
version=version,
supervision_mode=supervision_mode,
pid=_coerce_pid(runtime.get("pid")),
))
return GatewayHealthSnapshot(metrics=metrics, events=events)
def _safe_profile() -> str:
try:
from hermes_cli.profiles import get_active_profile_name
return str(get_active_profile_name() or "default")
except Exception:
return "default"
def _safe_version() -> str:
try:
from hermes_cli import __version__
return str(__version__)
except Exception:
return "unknown"
def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None:
"""Emit immediate content-free gateway events for runtime status changes.
Called by gateway.status.write_runtime_status after persisting the new status.
Fully fail-open: failures never affect gateway status writes.
"""
try:
from agent.monitoring import emitter
out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
profile = _safe_profile()
version = _safe_version()
old_gateway_state = str((previous or {}).get("gateway_state")) if (previous or {}).get("gateway_state") is not None else None
new_gateway_state = str(current.get("gateway_state")) if current.get("gateway_state") is not None else None
if old_gateway_state != new_gateway_state and new_gateway_state:
out.append(GatewayHealthEvent(
name="gateway.lifecycle",
gateway_state=new_gateway_state,
old_state=old_gateway_state,
new_state=new_gateway_state,
exit_reason=current.get("exit_reason"),
restart_requested=bool(current.get("restart_requested")),
active_agents=_parse_active_agents(current.get("active_agents", 0)),
profile=profile,
version=version,
pid=_coerce_pid(current.get("pid")),
))
if new_gateway_state == "startup_failed":
out.append(GatewayDiagnosticEvent(
name="gateway.startup_failed",
subsystem="gateway",
error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
redacted_message=redact_gateway_message(current.get("exit_reason") or "startup failed"),
profile=profile,
version=version,
severity="error",
))
if new_gateway_state == "stopped":
out.append(GatewayHealthEvent(
name="gateway.exit",
gateway_state=new_gateway_state,
old_state=old_gateway_state,
new_state=new_gateway_state,
exit_reason=current.get("exit_reason"),
restart_requested=bool(current.get("restart_requested")),
active_agents=_parse_active_agents(current.get("active_agents", 0)),
profile=profile,
version=version,
pid=_coerce_pid(current.get("pid")),
))
old_platforms_raw = (previous or {}).get("platforms")
new_platforms_raw = current.get("platforms")
old_platforms = old_platforms_raw if isinstance(old_platforms_raw, dict) else {}
new_platforms = new_platforms_raw if isinstance(new_platforms_raw, dict) else {}
for platform, pdata in new_platforms.items():
pdata = pdata if isinstance(pdata, dict) else {}
prev_raw = old_platforms.get(platform, {})
prev = prev_raw if isinstance(prev_raw, dict) else {}
old_state = str(prev.get("state")) if prev.get("state") is not None else None
new_state = str(pdata.get("state")) if pdata.get("state") is not None else None
if old_state == new_state or not new_state:
continue
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
severity = "error" if new_state.lower() in {"fatal", "failed", "error"} else "warning"
out.append(GatewayDiagnosticEvent(
name="platform.state_change",
subsystem=f"platform.{platform}",
platform=str(platform),
old_state=old_state,
new_state=new_state,
error_code=error_code,
error_class=error_code,
redacted_message=redact_gateway_message(pdata.get("error_message")),
profile=profile,
version=version,
severity=severity,
))
if new_state.lower() in _FATAL_PLATFORM_STATES:
out.append(GatewayDiagnosticEvent(
name="platform.fatal",
subsystem=f"platform.{platform}",
platform=str(platform),
error_code=error_code,
error_class=error_code,
redacted_message=redact_gateway_message(pdata.get("error_message")),
profile=profile,
version=version,
severity=severity,
))
for ev in out:
emitter.emit(ev)
except Exception:
logging.getLogger(__name__).debug("gateway runtime status transition emit failed", exc_info=True)
def _coerce_pid(raw: Any) -> Optional[int]:
try:
pid = int(raw)
except (TypeError, ValueError):
return None
return pid if pid > 0 else None
class GatewayDiagnosticLogHandler(logging.Handler):
"""Allowlisted warning/error bridge for gateway-owned diagnostics."""
def __init__(self, *, profile: str = "default", version: str = "unknown") -> None:
super().__init__(level=logging.WARNING)
self.profile = profile
self.version = version
def emit(self, record: logging.LogRecord) -> None:
try:
if record.levelno < logging.WARNING:
return
if not _allowed_logger(record.name):
return
subsystem = subsystem_for_logger(record.name)
message = record.getMessage()
event = GatewayDiagnosticEvent(
name=f"gateway.log.{record.levelname.lower()}",
subsystem=subsystem,
platform=platform_for_subsystem(subsystem),
error_class=classify_gateway_error(message),
redacted_message=redact_gateway_message(message),
profile=self.profile,
version=self.version,
severity=record.levelname.lower(),
)
from agent.monitoring import emitter
emitter.get_emitter().emit(event)
except Exception:
logging.getLogger(__name__).debug("gateway diagnostic emit failed", exc_info=True)
__all__ = [
"GatewayMetric",
"GatewayHealthSnapshot",
"GatewayDiagnosticLogHandler",
"build_gateway_health_snapshot",
"classify_gateway_error",
"redact_gateway_message",
]

405
agent/monitoring/gateway_health_export.py Normal file
View File

@ -0,0 +1,405 @@
"""Gateway Health & Diagnostics OTLP export runtime.
This exporter emits operator-owned gateway service-health metrics plus
narrow redacted diagnostic events. It is deliberately in-process and fail-open so
it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell
without a sidecar/watchdog dependency.
"""
from __future__ import annotations
import logging
import os
import threading
from dataclasses import dataclass
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
@dataclass(slots=True)
class GatewayHealthExportRuntime:
enabled: bool
reason: str = "disabled"
streamer: Any = None
metric_provider: Any = None
log_handler: Any = None
log_streamer: Any = None
thread: Optional[threading.Thread] = None
stop_event: Optional[threading.Event] = None
def shutdown(self) -> None:
if self.stop_event is not None:
self.stop_event.set()
if self.thread is not None:
self.thread.join(timeout=2.0)
if self.log_handler is not None:
try:
logging.getLogger().removeHandler(self.log_handler)
except Exception:
pass
if self.streamer is not None:
try:
self.streamer.shutdown()
except Exception:
pass
if self.log_streamer is not None:
try:
self.log_streamer.shutdown()
except Exception:
pass
if self.metric_provider is not None:
try:
self.metric_provider.shutdown()
except Exception:
pass
def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]:
mon = (config or {}).get("monitoring") or {}
return mon.get("gateway_health_export") or {}
def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
mon = (config or {}).get("monitoring") or {}
export = mon.get("export") or {}
return export.get("otlp") or {}
def _enabled(config: Dict[str, Any]) -> bool:
gh = _gateway_health_config(config)
otlp = _otlp_config(config)
return bool(gh.get("enabled") and otlp.get("enabled") and otlp.get("endpoint"))
def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]:
if auto_install:
try:
from tools.lazy_deps import ensure as _lazy_ensure
_lazy_ensure("export.otlp", prompt=prompt)
except Exception:
pass
try:
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.metrics import Observation
from opentelemetry.trace import INVALID_SPAN_ID, INVALID_TRACE_ID, TraceFlags
from opentelemetry._logs import LogRecord
from opentelemetry._logs.severity import SeverityNumber
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
return {
"OTLPLogExporter": OTLPLogExporter,
"OTLPMetricExporter": OTLPMetricExporter,
"Observation": Observation,
"LogRecord": LogRecord,
"LoggerProvider": LoggerProvider,
"INVALID_SPAN_ID": INVALID_SPAN_ID,
"INVALID_TRACE_ID": INVALID_TRACE_ID,
"TraceFlags": TraceFlags,
"SeverityNumber": SeverityNumber,
"BatchLogRecordProcessor": BatchLogRecordProcessor,
"MeterProvider": MeterProvider,
"PeriodicExportingMetricReader": PeriodicExportingMetricReader,
"Resource": Resource,
}
except Exception as exc:
raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc
def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
resolved: Dict[str, str] = {}
for header_name, env_name in (headers_env or {}).items():
val = os.environ.get(str(env_name))
if val:
resolved[str(header_name)] = val
return resolved
def _metric_endpoint(endpoint: str) -> str:
if endpoint.endswith("/v1/traces"):
return endpoint[: -len("/v1/traces")] + "/v1/metrics"
return endpoint
def _logs_endpoint(endpoint: str) -> str:
if endpoint.endswith("/v1/traces"):
return endpoint[: -len("/v1/traces")] + "/v1/logs"
if endpoint.endswith("/v1/metrics"):
return endpoint[: -len("/v1/metrics")] + "/v1/logs"
return endpoint
def _version() -> str:
try:
from hermes_cli import __version__
return str(__version__)
except Exception:
return "unknown"
def _profile() -> str:
try:
from hermes_cli.profiles import get_active_profile_name
return str(get_active_profile_name() or "default")
except Exception:
return "default"
def _install_id(config: Dict[str, Any]) -> str:
try:
from agent.monitoring.policy import ensure_install_id
return str(ensure_install_id(config))
except Exception:
return "unknown"
def _supervision_mode() -> str:
if os.environ.get("INVOCATION_ID"):
return "systemd"
if os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION"):
return "s6"
if os.environ.get("container") or os.path.exists("/.dockerenv"):
return "container"
if os.environ.get("LAUNCHD_SOCKET"):
return "launchd"
return "manual"
def _read_runtime_snapshot(config: Dict[str, Any]):
from agent.monitoring.gateway_health import build_gateway_health_snapshot
try:
from gateway.status import read_runtime_status
runtime = read_runtime_status() or {}
except Exception:
runtime = {}
return build_gateway_health_snapshot(
runtime,
gateway_running=True,
profile=_profile(),
install_id=_install_id(config),
version=_version(),
supervision_mode=_supervision_mode(),
)
def _emit_snapshot_events(config: Dict[str, Any]) -> None:
gh = _gateway_health_config(config)
if not gh.get("diagnostic_events_enabled", True):
return
try:
from agent.monitoring import emitter
snapshot = _read_runtime_snapshot(config)
for event in snapshot.events:
emitter.emit(event)
except Exception:
logger.debug("gateway health snapshot emit failed", exc_info=True)
def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any:
gh = _gateway_health_config(config)
if not gh.get("metrics_enabled", True):
return None
otlp = _otlp_config(config)
endpoint = _metric_endpoint(str(otlp.get("endpoint")))
headers = _resolve_headers(otlp.get("headers_env"))
exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None)
interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000
reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms)
resource_attrs = dict((gh.get("resource_attributes") or {}))
resource_attrs.setdefault("service.name", "hermes-gateway")
resource_attrs.setdefault("telemetry.scope", "gateway_health")
provider = sdk["MeterProvider"](
metric_readers=[reader],
resource=sdk["Resource"].create(resource_attrs),
)
meter = provider.get_meter("hermes.gateway.health")
Observation = sdk["Observation"]
metric_names = [
"hermes.gateway.up",
"hermes.gateway.state",
"hermes.gateway.active_agents",
"hermes.gateway.busy",
"hermes.gateway.drainable",
"hermes.gateway.restart_requested",
"hermes.platform.up",
"hermes.platform.degraded",
]
def callback(name: str):
def _cb(_options=None):
try:
snapshot = _read_runtime_snapshot(config)
return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name]
except Exception:
logger.debug("gateway metric callback failed", exc_info=True)
return []
return _cb
for metric_name in metric_names:
meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)])
return provider
def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any:
SeverityNumber = sdk["SeverityNumber"]
sev = str(severity or "warning").lower()
if sev in {"critical", "fatal"}:
return SeverityNumber.FATAL
if sev == "error":
return SeverityNumber.ERROR
if sev in {"info", "information"}:
return SeverityNumber.INFO
if sev == "debug":
return SeverityNumber.DEBUG
return SeverityNumber.WARN
class GatewayDiagnosticLogStreamer:
"""Emitter subscriber that sends gateway diagnostic events as OTLP logs."""
def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]):
otlp = _otlp_config(config)
gh = _gateway_health_config(config)
headers = _resolve_headers(otlp.get("headers_env"))
endpoint = _logs_endpoint(str(otlp.get("endpoint")))
resource_attrs = dict((gh.get("resource_attributes") or {}))
resource_attrs.setdefault("service.name", "hermes-gateway")
resource_attrs.setdefault("telemetry.scope", "gateway_diagnostics")
self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs))
self._processor = sdk["BatchLogRecordProcessor"](
sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None)
)
self._provider.add_log_record_processor(self._processor)
self._logger = self._provider.get_logger("hermes.gateway.diagnostics")
self._LogRecord = sdk["LogRecord"]
self._sdk = sdk
self.exported = 0
def __call__(self, batch: list[Dict[str, Any]]) -> None:
for ev in batch:
if ev.get("event") != "gateway_diagnostic":
continue
attrs = {
f"hermes.{key}": val
for key, val in ev.items()
if key not in {"event", "redacted_message", "ts_ns"} and val is not None
}
body = ev.get("redacted_message") or ev.get("name") or "gateway diagnostic"
record = self._LogRecord(
timestamp=ev.get("ts_ns"),
trace_id=self._sdk["INVALID_TRACE_ID"],
span_id=self._sdk["INVALID_SPAN_ID"],
trace_flags=self._sdk["TraceFlags"].DEFAULT,
severity_text=str(ev.get("severity") or "warning").upper(),
severity_number=_severity_number(self._sdk, ev.get("severity")),
body=str(body),
attributes=attrs,
)
self._logger.emit(record)
self.exported += 1
def shutdown(self) -> None:
try:
from agent.monitoring.emitter import get_emitter
get_emitter().unsubscribe(self)
except Exception:
pass
try:
self._processor.force_flush()
self._provider.shutdown()
except Exception:
pass
def _start_diagnostic_log_streamer(config: Dict[str, Any], sdk: Dict[str, Any]) -> GatewayDiagnosticLogStreamer:
from agent.monitoring.emitter import get_emitter
streamer = GatewayDiagnosticLogStreamer(config, sdk)
get_emitter().subscribe(streamer)
return streamer
def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread:
interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5)))
def _run() -> None:
while not stop_event.wait(interval):
_emit_snapshot_events(config)
thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True)
thread.start()
return thread
def _attach_log_handler(config: Dict[str, Any]) -> Any:
gh = _gateway_health_config(config)
if not gh.get("warning_error_events_enabled", True):
return None
from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version())
root = logging.getLogger()
if handler not in root.handlers:
root.addHandler(handler)
return handler
def _gateway_health_event(ev: Dict[str, Any]) -> bool:
return ev.get("event") == "gateway_health"
def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime:
"""Start P0 gateway health export if configured. Never raises."""
if not _enabled(config):
return GatewayHealthExportRuntime(enabled=False, reason="disabled")
gh = _gateway_health_config(config)
runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled")
sdk: Optional[Dict[str, Any]] = None
if gh.get("metrics_enabled", True) or gh.get("diagnostic_events_enabled", True):
try:
sdk = _require_metrics_sdk(prompt=False)
except Exception:
logger.warning(
"monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; "
"install 'hermes-agent[otlp]'",
exc_info=True,
)
return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable")
if gh.get("metrics_enabled", True) and sdk is not None:
try:
runtime.metric_provider = _start_metric_provider(config, sdk)
except Exception:
logger.warning("gateway health OTLP metrics failed to start", exc_info=True)
runtime.shutdown()
return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed")
if gh.get("diagnostic_events_enabled", True) and sdk is not None:
try:
from agent.monitoring import otlp_exporter
runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event)
runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk)
except Exception:
logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True)
try:
runtime.log_handler = _attach_log_handler(config)
except Exception:
logger.debug("gateway diagnostic log handler failed to attach", exc_info=True)
try:
_emit_snapshot_events(config)
runtime.stop_event = threading.Event()
runtime.thread = _start_snapshot_thread(config, runtime.stop_event)
except Exception:
logger.debug("gateway health snapshot thread failed to start", exc_info=True)
return runtime
__all__ = [
"GatewayHealthExportRuntime",
"start_gateway_health_export",
]

161
agent/telemetry/otlp_exporter.py → agent/monitoring/otlp_exporter.py
View File

@ -1,37 +1,31 @@
"""Export telemetry to an OpenTelemetry Collector over OTLP/HTTP.
"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.
Maps the local tel_* events to OTel spans and sends them to the endpoint configured
under ``telemetry.export.otlp``. Lets an operator stream Hermes telemetry into their
own observability stack.
Maps gateway monitoring events to OTel spans and sends them to the endpoint
configured under ``monitoring.export.otlp``. Lets an operator stream Hermes
gateway health into their own observability stack (OTEL Collector, DataDog,
and similar).
Notes:
* The destination is operator-configured; this module only sends to that endpoint.
It does not import or interact with any aggregate-metrics path.
* ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an optional
extra (``pip install hermes-agent[otlp]``), imported lazily so the dependency is
only required when OTLP export is actually used.
* ``headers_env`` maps a header name to an environment variable name; values are read
from the environment at export time and never logged or stored.
* The continuous subscriber runs in the emitter's writer thread after durable writes
and is fail-isolated, so an export error cannot affect a run.
* The destination is operator-configured; this module only sends to that
endpoint. No default destination ships.
* ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an
optional extra (``pip install hermes-agent[otlp]``), imported lazily so the
dependency is only required when OTLP export is actually used.
* ``headers_env`` maps a header name to an environment variable name; values
are read from the environment at export time and never logged or stored.
* The continuous subscriber runs in the emitter's dispatcher thread and is
fail-isolated, so an export error cannot affect the gateway.
Each event is exported as a span carrying its recorded attributes (provider, model,
tokens, duration, etc.). The timing/parent linkage captured in tel_spans
(trace_id/span_id/parent_span_id/start_ns/end_ns) is not yet reconstructed into OTel
SpanContexts here, so spans currently arrive as independent records rather than a
connected trace tree; building the connected-trace projection is tracked separately.
Spans carry structural telemetry by default. Message content is included only when
trajectories is enabled, and always passes through the export redaction pipeline.
Only monitoring events (gateway_health / gateway_diagnostic) exist on this
plane; the ``event_filter`` seam is kept so future planes sharing the emitter
cannot silently ride along on this exporter.
"""
from __future__ import annotations
import logging
import os
import sqlite3
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Callable, Dict, List, Optional
logger = logging.getLogger(__name__)
@ -89,9 +83,9 @@ def _require_sdk(*, auto_install: bool = True, prompt: bool = True):
def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
"""Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env.
The config stores environment variable names, not secret values; values are read
from the environment here. Missing variables are skipped (and noted at debug level
without the value).
The config stores environment variable names, not secret values; values are
read from the environment here. Missing variables are skipped (and noted at
debug level without the value).
"""
resolved: Dict[str, str] = {}
for header_name, env_name in (headers_env or {}).items():
@ -105,8 +99,8 @@ def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
tel = (config or {}).get("telemetry") or {}
export = tel.get("export") or {}
mon = (config or {}).get("monitoring") or {}
export = mon.get("export") or {}
return export.get("otlp") or {}
@ -116,7 +110,7 @@ def build_exporter(config: Dict[str, Any]):
otlp = _otlp_config(config)
endpoint = otlp.get("endpoint")
if not endpoint:
raise ValueError("telemetry.export.otlp.endpoint is not set")
raise ValueError("monitoring.export.otlp.endpoint is not set")
headers = _resolve_headers(otlp.get("headers_env"))
return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None)
@ -124,8 +118,8 @@ def build_exporter(config: Dict[str, Any]):
def _make_provider(config: Dict[str, Any]):
sdk = _require_sdk()
resource = sdk["Resource"].create({
"service.name": "hermes-agent",
"telemetry.scope": "local", # never aggregate metrics
"service.name": "hermes-gateway",
"telemetry.scope": "gateway_monitoring",
})
provider = sdk["TracerProvider"](resource=resource)
processor = sdk["BatchSpanProcessor"](build_exporter(config))
@ -133,21 +127,20 @@ def _make_provider(config: Dict[str, Any]):
return provider, processor
# ── event -> span attribute mapping (real values) ───────────────────────────
# ── event -> span attribute mapping ──────────────────────────────────────────
def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
"""Span attributes for an event — the real recorded values (local telemetry)."""
"""Span attributes for a monitoring event (content-free by construction)."""
kind = ev.get("event")
attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"}
keep_by_kind = {
"run": ("entrypoint", "platform", "end_reason",
"model_call_count", "tool_call_count", "error_count"),
"span": ("trace_id", "run_id", "parent_span_id", "name", "kind",
"start_ns", "end_ns", "status"),
"model_call": ("provider", "model", "base_url",
"input_tokens", "output_tokens", "cache_read_tokens",
"cache_write_tokens", "reasoning_tokens", "latency_ms"),
"tool_call": ("tool_name", "duration_ms", "result_class"),
"error": ("error_class", "subsystem", "recovery"),
"gateway_health": ("name", "gateway_state", "old_state", "new_state",
"exit_reason", "restart_requested", "active_agents",
"gateway_busy", "gateway_drainable", "platform_count",
"fatal_platform_count", "profile", "install_id", "version",
"supervision_mode", "pid"),
"gateway_diagnostic": ("name", "subsystem", "error_class", "error_code",
"redacted_message", "platform", "old_state", "new_state",
"profile", "version", "severity"),
}
for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type]
v = ev.get(col)
@ -158,7 +151,7 @@ def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
def export_batch(provider, batch: List[Dict[str, Any]]) -> int:
"""Map a batch of events to OTel spans. Returns spans created."""
tracer = provider.get_tracer("hermes.telemetry")
tracer = provider.get_tracer("hermes.monitoring")
n = 0
for ev in batch:
try:
@ -171,53 +164,6 @@ def export_batch(provider, batch: List[Dict[str, Any]]) -> int:
return n
# ── one-shot drain (export current local rows) ──────────────────────────────
def export_once(
config: Dict[str, Any],
*,
db_path: Optional[Path] = None,
since_ns: Optional[int] = None,
) -> int:
"""Drain the local tel_* tables to the configured OTLP endpoint once."""
provider, processor = _make_provider(config)
try:
rows = _read_events(db_path, since_ns)
total = export_batch(provider, rows)
processor.force_flush()
return total
finally:
try:
provider.shutdown()
except Exception:
pass
def _read_events(db_path: Optional[Path], since_ns: Optional[int]) -> List[Dict[str, Any]]:
if db_path is None:
from hermes_constants import get_hermes_home
db_path = get_hermes_home() / "state.db"
c = sqlite3.connect(str(db_path), timeout=5.0)
c.row_factory = sqlite3.Row
out: List[Dict[str, Any]] = []
try:
table_event = {
"tel_runs": "run", "tel_spans": "span",
"tel_model_calls": "model_call",
"tel_tool_calls": "tool_call", "tel_error_events": "error",
}
for table, evkind in table_event.items():
where = ""
if table == "tel_runs" and since_ns:
where = f" WHERE start_ns >= {int(since_ns)}"
for r in c.execute(f"SELECT * FROM {table}{where}").fetchall():
d = dict(r)
d["event"] = evkind
out.append(d)
finally:
c.close()
return out
# ── continuous streaming subscriber ─────────────────────────────────────────
class OTLPStreamer:
"""A live subscriber that pushes each emitter batch to OTLP as it lands.
@ -225,14 +171,29 @@ class OTLPStreamer:
Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.
"""
def __init__(self, config: Dict[str, Any]):
def __init__(
self,
config: Dict[str, Any],
*,
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
):
self._provider, self._processor = _make_provider(config)
self._event_filter = event_filter
self.exported = 0
def __call__(self, batch: List[Dict[str, Any]]) -> None:
if self._event_filter is not None:
batch = [ev for ev in batch if self._event_filter(ev)]
if not batch:
return
self.exported += export_batch(self._provider, batch)
def shutdown(self) -> None:
try:
from agent.monitoring.emitter import get_emitter
get_emitter().unsubscribe(self)
except Exception:
pass
try:
self._processor.force_flush()
self._provider.shutdown()
@ -255,9 +216,16 @@ def is_enabled(config: Dict[str, Any]) -> bool:
return bool(otlp.get("enabled") and otlp.get("endpoint"))
def start_streaming(config: Dict[str, Any]) -> Optional[OTLPStreamer]:
def start_streaming(
config: Dict[str, Any],
*,
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
) -> Optional[OTLPStreamer]:
"""If OTLP is enabled, attach a streamer to the singleton emitter.
``event_filter`` scopes the exporter to its plane, e.g. gateway-health
export, so enabling one plane cannot silently export unrelated events.
Non-interactive context (startup): attempts a lazy install with prompt=False
so a configured-but-missing SDK is installed once (gated by
security.allow_lazy_installs), then streams. If it still can't load, logs and
@ -268,11 +236,11 @@ def start_streaming(config: Dict[str, Any]) -> Optional[OTLPStreamer]:
try:
_require_sdk(prompt=False)
except OTLPUnavailable:
logger.warning("telemetry.export.otlp.enabled but the OTel SDK could not "
logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
"be installed/imported; install 'hermes-agent[otlp]'")
return None
from agent.telemetry.emitter import get_emitter
streamer = OTLPStreamer(config)
from agent.monitoring.emitter import get_emitter
streamer = OTLPStreamer(config, event_filter=event_filter)
get_emitter().subscribe(streamer)
return streamer
@ -281,7 +249,6 @@ __all__ = [
"OTLPUnavailable",
"OTLPStreamer",
"build_exporter",
"export_once",
"export_batch",
"is_available",
"is_enabled",

40
agent/monitoring/policy.py Normal file
View File

@ -0,0 +1,40 @@
"""Install identity for gateway monitoring.
The install id is a stable, resettable pseudonymous identifier attached to
exported health signals so an operator can tell instances apart in their
collector. It carries no account identity and can be rotated by clearing
``monitoring.install_id`` in config.
"""
from __future__ import annotations
import uuid
from typing import Any, Dict
def _monitoring_cfg(config: Dict[str, Any]) -> Dict[str, Any]:
for key in ("monitoring", "telemetry"): # accept legacy telemetry.* keys
cfg = config.get(key) if isinstance(config, dict) else None
if isinstance(cfg, dict) and cfg.get("install_id"):
return cfg
cfg = config.get("monitoring") if isinstance(config, dict) else None
return cfg if isinstance(cfg, dict) else {}
def ensure_install_id(config: Dict[str, Any]) -> str:
"""Return a stable install id, minting one if the config slot is empty.
Does not persist — the caller writes the returned value back to
config.yaml. Clearing ``monitoring.install_id`` (e.g. with
``hermes config set monitoring.install_id ""``) mints anew on next call.
"""
cfg = _monitoring_cfg(config)
existing = cfg.get("install_id")
if isinstance(existing, str) and existing.strip():
return existing
return str(uuid.uuid4())
__all__ = [
"ensure_install_id",
]

77
agent/monitoring/redaction.py Normal file
View File

@ -0,0 +1,77 @@
"""Redaction applied to monitoring data before egress.
Secrets are always redacted, on every export path; no setting disables this.
Wraps ``agent/redact.py::redact_sensitive_text(force=True)`` and fails CLOSED:
if the redactor cannot run, the raw string is never emitted.
``redact_for_export(text, content_mode="pii")`` additionally scrubs e-mail
addresses, phone numbers, and UUID-shaped identifiers — the gateway
diagnostics path always uses this mode, so log-derived messages leave the
process with secrets AND PII already removed.
"""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional
# Content-redaction strengths for any content that IS exported.
CONTENT_NONE = "none" # drop content entirely (structural telemetry only)
CONTENT_PII = "pii" # codec-aware PII redaction on exported content
CONTENT_MODES = {CONTENT_NONE, CONTENT_PII}
# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ──
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}")
# E.164-ish and common separators; conservative to avoid nuking code/IDs.
_PHONE_RE = re.compile(
r"(?<!\w)(?:\+?\d{1,3}[\s.\-]?)?(?:\(\d{2,4}\)[\s.\-]?)?\d{3}[\s.\-]?\d{3,4}(?:[\s.\-]?\d{2,4})?(?!\w)"
)
# Long opaque hex/uuid-ish user identifiers.
_UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
def _secret_redact(text: Optional[str]) -> Optional[str]:
"""Always-on secret redaction. force=True so user config can't disable it."""
if text is None:
return None
try:
from agent.redact import redact_sensitive_text
return redact_sensitive_text(str(text), force=True)
except Exception:
# Fail CLOSED: if the redactor can't run, do not emit the raw string.
return "[redaction-unavailable]"
def _pii_redact(text: str) -> str:
text = _EMAIL_RE.sub("[email]", text)
text = _UUID_RE.sub("[id]", text)
text = _PHONE_RE.sub("[phone]", text)
return text
def redact_for_export(
text: Optional[str],
*,
content_mode: str = CONTENT_NONE,
) -> Optional[str]:
"""Redact a single content string for export.
Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'.
Callers gate *whether content is exported at all* via telemetry.trajectories
(see ``content_export_enabled``); this function only scrubs content that the
caller has already decided to export.
"""
redacted = _secret_redact(text)
if redacted is None:
return None
if content_mode == CONTENT_PII:
redacted = _pii_redact(redacted)
return redacted
__all__ = [
"CONTENT_NONE",
"CONTENT_PII",
"CONTENT_MODES",
"redact_for_export",
]

30
agent/telemetry/__init__.py
View File

@ -1,30 +0,0 @@
"""Hermes telemetry & observability.
Local-first observability, on by default. The ``telemetry`` plugin registers Hermes
lifecycle hooks and hands typed events to the fire-and-forget ``emitter`` (queue ->
background writer -> JSONL + state.db ``tel_*`` index). The emitter never blocks or
raises into a model/tool call (the hot-path invariant).
Events record the observed model ids, provider names, and tool names. ``metrics``
derives rollups for /usage and /insights; ``rollup`` builds the per-run summaries shown
by ``hermes telemetry preview``. ``redaction`` + ``exporter_bulk`` + ``otlp_exporter``
handle export to an operator-chosen destination. ``policy`` holds the consent
constants and the aggregate upload gate (no uploader ships).
"""
from __future__ import annotations
from . import emitter, events, metrics, policy, spans
emit = emitter.emit
get_emitter = emitter.get_emitter
__all__ = [
"emitter",
"events",
"metrics",
"policy",
"spans",
"emit",
"get_emitter",
]

318
agent/telemetry/emitter.py
View File

@ -1,318 +0,0 @@
"""Local telemetry emitter: fire-and-forget queue + background writer.
The emitter is the single seam between instrumentation (the telemetry plugin's hook
callbacks) and durable storage. Its contract is the hot-path invariant:
``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, and
MUST NEVER raise into the caller. A telemetry failure is logged locally and
dropped — it can never affect a model call, a tool call, or a session.
Mechanism:
* ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare except.
On a full queue it drops the *oldest* event and counts the drop.
* A daemon thread drains the queue and writes each event to two places:
1. the append-only JSONL log (source of truth)
2. the ``tel_*`` SQLite tables in state.db (rebuildable index)
* The writer uses its own sqlite connection to state.db, separate from SessionDB,
so telemetry writes never contend with or corrupt session writes.
Local telemetry only. Nothing here uploads anywhere.
"""
from __future__ import annotations
import json
import logging
import queue
import sqlite3
import threading
import time
from pathlib import Path
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full
_DRAIN_BATCH = 256
def _default_dir() -> Path:
"""Resolve the telemetry dir under the active HERMES_HOME (profile-safe)."""
from hermes_constants import get_hermes_home
return get_hermes_home() / "telemetry"
def _default_db_path() -> Path:
"""Resolve state.db under the active HERMES_HOME (profile-safe)."""
from hermes_constants import get_hermes_home
return get_hermes_home() / "state.db"
# Map a telemetry event dict (its "event" tag) to (table, column-ordered insert).
# Only the columns the indexer knows about are written; unknown keys are ignored,
# so an event carrying extra fields never breaks the insert.
_TABLE_COLUMNS: Dict[str, tuple] = {
"run": (
"tel_runs",
("run_id", "trace_id", "session_id", "entrypoint",
"platform", "start_ns", "end_ns", "end_reason",
"model_call_count", "tool_call_count", "error_count"),
),
"span": (
"tel_spans",
("span_id", "trace_id", "run_id", "parent_span_id", "name", "kind",
"start_ns", "end_ns", "status"),
),
"model_call": (
"tel_model_calls",
("span_id", "run_id", "provider", "model", "base_url",
"input_tokens", "output_tokens", "cache_read_tokens",
"cache_write_tokens", "reasoning_tokens", "latency_ms"),
),
"tool_call": (
"tel_tool_calls",
("span_id", "run_id", "tool_name", "duration_ms", "result_class"),
),
"error": (
"tel_error_events",
("run_id", "error_class", "subsystem", "recovery", "ts_ns"),
),
}
class TelemetryEmitter:
"""Owns the queue, the writer thread, and the telemetry sqlite connection."""
def __init__(
self,
*,
events_path: Optional[Path] = None,
db_path: Optional[Path] = None,
enabled: bool = True,
) -> None:
self._dir = (events_path.parent if events_path else _default_dir())
self._events_path = events_path or (self._dir / "events.jsonl")
self._db_path = db_path or _default_db_path()
self._enabled = enabled
self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE)
self._dropped = 0
self._written = 0
self._stop = threading.Event()
self._started = False
self._lock = threading.Lock()
self._conn: Optional[sqlite3.Connection] = None
self._thread: Optional[threading.Thread] = None
# Optional live subscribers (e.g. OTLP exporter). Called from the writer
# thread AFTER durable writes, fully fail-isolated — a subscriber that
# raises or blocks can never affect the JSONL/SQLite source of truth or
# the hot path. Each subscriber is callable(batch: list[dict]).
self._subscribers: list = []
# ── public API (hot path) ───────────────────────────────────────────────
def emit(self, event: Any) -> None:
"""Enqueue an event. Never blocks, never raises.
``event`` may be a dataclass with ``to_dict()`` or a plain dict.
"""
if not self._enabled:
return
try:
payload = event.to_dict() if hasattr(event, "to_dict") else dict(event)
payload.setdefault("ts_ns", time.time_ns())
self._ensure_started()
try:
self._q.put_nowait(payload)
except queue.Full:
# Drop oldest to make room — bounded memory, newest-wins.
try:
self._q.get_nowait()
self._dropped += 1
self._q.put_nowait(payload)
except Exception:
self._dropped += 1
except Exception: # the hot-path invariant: never propagate
logger.debug("telemetry emit failed", exc_info=True)
# ── lifecycle ───────────────────────────────────────────────────────────
def _ensure_started(self) -> None:
if self._started:
return
with self._lock:
if self._started:
return
try:
self._dir.mkdir(parents=True, exist_ok=True)
except Exception:
logger.debug("telemetry dir create failed", exc_info=True)
self._thread = threading.Thread(
target=self._run, name="hermes-telemetry-writer", daemon=True
)
self._thread.start()
self._started = True
def _open_conn(self) -> Optional[sqlite3.Connection]:
if self._conn is not None:
return self._conn
try:
conn = sqlite3.connect(str(self._db_path), isolation_level=None, timeout=5.0)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=5000")
self._conn = conn
except Exception:
logger.debug("telemetry db open failed", exc_info=True)
self._conn = None
return self._conn
def _run(self) -> None:
while not self._stop.is_set():
try:
first = self._q.get(timeout=0.5)
except queue.Empty:
continue
batch = [first]
while len(batch) < _DRAIN_BATCH:
try:
batch.append(self._q.get_nowait())
except queue.Empty:
break
self._write_batch(batch)
def _write_batch(self, batch) -> None:
# JSONL append (source of truth) — best effort.
try:
with open(self._events_path, "a", encoding="utf-8") as fh:
for ev in batch:
fh.write(json.dumps(ev, ensure_ascii=False) + "\n")
except Exception:
logger.debug("telemetry jsonl append failed", exc_info=True)
# SQLite index — best effort, per-event so one bad row can't lose the batch.
conn = self._open_conn()
if conn is None:
return
for ev in batch:
try:
self._index_one(conn, ev)
self._written += 1
except Exception:
logger.debug("telemetry index row failed", exc_info=True)
# Live fan-out (e.g. OTLP) — AFTER durable writes, fully fail-isolated.
# A slow/raising subscriber never affects JSONL/SQLite or the hot path.
for sub in self._subscribers:
try:
sub(batch)
except Exception:
logger.debug("telemetry subscriber failed", exc_info=True)
def subscribe(self, callback) -> None:
"""Register a live batch subscriber (callable(batch: list[dict])).
Called from the writer thread after durable writes. Used by the OTLP
exporter for continuous streaming. Fail-isolated; never on the hot path.
"""
if callback not in self._subscribers:
self._subscribers.append(callback)
def unsubscribe(self, callback) -> None:
try:
self._subscribers.remove(callback)
except ValueError:
pass
def _index_one(self, conn: sqlite3.Connection, ev: Dict[str, Any]) -> None:
kind = ev.get("event")
spec = _TABLE_COLUMNS.get(kind)
if spec is None:
return
table, cols = spec
values = [ev.get(c) for c in cols]
placeholders = ", ".join("?" for _ in cols)
collist = ", ".join(cols)
conn.execute(
f"INSERT OR REPLACE INTO {table} ({collist}) VALUES ({placeholders})",
values,
)
# ── introspection / shutdown (tests, CLI) ───────────────────────────────
def flush(self, timeout: float = 2.0) -> None:
"""Block until the queue drains (test/CLI helper, NOT the hot path)."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self._q.empty():
# give the writer a tick to finish the in-flight batch
time.sleep(0.05)
if self._q.empty():
return
time.sleep(0.02)
def stats(self) -> Dict[str, int]:
return {
"queued": self._q.qsize(),
"written": self._written,
"dropped": self._dropped,
}
def close(self) -> None:
self._stop.set()
if self._thread is not None:
self._thread.join(timeout=2.0)
if self._conn is not None:
try:
self._conn.close()
except Exception:
pass
self._conn = None
self._started = False
# ── process-wide singleton ──────────────────────────────────────────────────
_EMITTER: Optional[TelemetryEmitter] = None
_EMITTER_LOCK = threading.Lock()
def get_emitter() -> TelemetryEmitter:
"""Return the process-wide emitter, honoring telemetry.local config."""
global _EMITTER
if _EMITTER is not None:
return _EMITTER
with _EMITTER_LOCK:
if _EMITTER is None:
enabled = _local_enabled()
_EMITTER = TelemetryEmitter(enabled=enabled)
return _EMITTER
def _local_enabled() -> bool:
try:
from hermes_cli.config import load_config
cfg = load_config()
tel = cfg.get("telemetry") if isinstance(cfg, dict) else {}
return bool((tel or {}).get("local", True))
except Exception:
return True
def emit(event: Any) -> None:
"""Module-level convenience: emit via the singleton."""
get_emitter().emit(event)
def reset_emitter_for_tests(emitter: Optional[TelemetryEmitter] = None) -> None:
"""Swap the singleton (tests only)."""
global _EMITTER
with _EMITTER_LOCK:
if _EMITTER is not None and emitter is not _EMITTER:
try:
_EMITTER.close()
except Exception:
pass
_EMITTER = emitter
__all__ = [
"TelemetryEmitter",
"get_emitter",
"emit",
"reset_emitter_for_tests",
]

111
agent/telemetry/events.py
View File

@ -1,111 +0,0 @@
"""Typed local telemetry events.
These dataclasses are the rows written to the local JSONL log and the ``tel_*``
SQLite tables. They record the values observed for each run — model id, provider, tool
name, token counts, durations — and stay on the machine unless explicitly exported.
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field, asdict
from typing import Any, Dict, Optional
# ── local telemetry events (real values) ────────────────────────────────────
def _now_ns() -> int:
return time.time_ns()
@dataclass(slots=True)
class RunEvent:
"""One top-level workflow execution (a trace root). A run spans one session."""
run_id: str
trace_id: str
entrypoint: str
session_id: Optional[str] = None
platform: Optional[str] = None
start_ns: int = field(default_factory=_now_ns)
end_ns: Optional[int] = None
end_reason: Optional[str] = None
model_call_count: int = 0
tool_call_count: int = 0
error_count: int = 0
def to_dict(self) -> Dict[str, Any]:
return {"event": "run", **asdict(self)}
@dataclass(slots=True)
class ModelCallEvent:
span_id: str
run_id: str
provider: Optional[str] = None # raw provider, e.g. "anthropic"
model: Optional[str] = None # raw model id, e.g. "claude-opus-4"
base_url: Optional[str] = None
input_tokens: int = 0
output_tokens: int = 0
cache_read_tokens: int = 0
cache_write_tokens: int = 0
reasoning_tokens: int = 0
latency_ms: Optional[int] = None
def to_dict(self) -> Dict[str, Any]:
return {"event": "model_call", **asdict(self)}
@dataclass(slots=True)
class ToolCallEvent:
span_id: str
run_id: str
tool_name: Optional[str] = None # raw tool name, e.g. "web_search"
duration_ms: Optional[int] = None
result_class: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
return {"event": "tool_call", **asdict(self)}
@dataclass(slots=True)
class SpanEvent:
"""A timed span — the timing/lineage backbone of a trace.
One row per run (the root, ``parent_span_id=None``) and one per model/tool call
(``parent_span_id`` = the run's root span). Detail rows in ``tel_model_calls`` /
``tel_tool_calls`` share the ``span_id`` and are joined here for ordering and
placement on a timeline.
"""
span_id: str
trace_id: str
run_id: str
name: str
kind: str # "run" | "model" | "tool"
start_ns: int
end_ns: Optional[int] = None
parent_span_id: Optional[str] = None
status: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
return {"event": "span", **asdict(self)}
@dataclass(slots=True)
class ErrorEvent:
run_id: Optional[str]
error_class: str
subsystem: str
recovery: Optional[str] = None
ts_ns: int = field(default_factory=_now_ns)
def to_dict(self) -> Dict[str, Any]:
return {"event": "error", **asdict(self)}
__all__ = [
"RunEvent",
"ModelCallEvent",
"ToolCallEvent",
"SpanEvent",
"ErrorEvent",
]

139
agent/telemetry/exporter_bulk.py
View File

@ -1,139 +0,0 @@
"""Export telemetry (and optionally session content) to a file or stream.
Two data domains, both written to an operator-chosen destination:
* Telemetry: the tel_* rows + events.jsonl (structural observability).
* Content (opt-in via telemetry.trajectories): sessions + messages, with every
content field (message body, reasoning, raw tool-call args) passed through the
redaction pipeline (secrets always stripped; PII per content_redaction).
Formats: ndjson (default) and json. OTLP streaming export lives in otlp_exporter.py.
Content export is gated by ``redaction.content_export_enabled``.
"""
from __future__ import annotations
import json
import sqlite3
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, TextIO
from . import redaction
_TEL_TABLES = (
"tel_runs", "tel_model_calls", "tel_tool_calls", "tel_error_events",
)
def _open(db_path: Optional[Path]) -> sqlite3.Connection:
if db_path is None:
from hermes_constants import get_hermes_home
db_path = get_hermes_home() / "state.db"
c = sqlite3.connect(str(db_path), timeout=5.0)
c.row_factory = sqlite3.Row
return c
def _iter_telemetry(conn: sqlite3.Connection, since_ns: Optional[int]) -> Iterator[Dict[str, Any]]:
for table in _TEL_TABLES:
# only tel_runs has start_ns; window the rest by run join when needed.
if table == "tel_runs" and since_ns:
rows = conn.execute(
f"SELECT * FROM {table} WHERE start_ns >= ?", (int(since_ns),)
).fetchall()
else:
rows = conn.execute(f"SELECT * FROM {table}").fetchall()
for r in rows:
d = dict(r)
d["_kind"] = table
yield d
def _iter_content(
db_path: Optional[Path],
*,
config: Optional[Dict[str, Any]],
include_content: bool,
) -> Iterator[Dict[str, Any]]:
"""Yield session records. Message bodies included only when trajectories on."""
from hermes_state import SessionDB
content_mode = redaction.content_mode_for(config)
db = SessionDB(db_path=db_path) if db_path else SessionDB()
try:
for session in db.export_all():
msgs = session.get("messages", []) or []
red_msgs = [
redaction.redact_message(
m, content_mode=content_mode, include_content=include_content
)
for m in msgs
]
# Session-level metadata is structural; keep ids/model/counts, drop
# any free-text title only when content is excluded.
out = {
"_kind": "session",
"id": session.get("id"),
"source": session.get("source"),
"model": session.get("model"),
"started_at": session.get("started_at"),
"ended_at": session.get("ended_at"),
"message_count": session.get("message_count"),
"tool_call_count": session.get("tool_call_count"),
"messages": red_msgs,
}
if include_content and session.get("title"):
out["title"] = redaction.redact_for_export(
session["title"], content_mode=content_mode
)
yield out
finally:
db.close()
def export(
out: TextIO,
*,
fmt: str = "ndjson",
since_ns: Optional[int] = None,
include_content: bool = False,
config: Optional[Dict[str, Any]] = None,
db_path: Optional[Path] = None,
) -> Dict[str, int]:
"""Write telemetry (+ optional content) to ``out``. Returns counts.
``include_content`` is honored only when telemetry.trajectories is enabled in
``config``; otherwise content is forced off and only structural data is written.
"""
# Trajectories gate: a flag cannot override the config setting.
content_allowed = include_content and redaction.content_export_enabled(config)
counts = {"telemetry": 0, "sessions": 0, "content_included": int(content_allowed)}
conn = _open(db_path)
records: List[Dict[str, Any]] = []
try:
for rec in _iter_telemetry(conn, since_ns):
counts["telemetry"] += 1
if fmt == "ndjson":
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
else:
records.append(rec)
finally:
conn.close()
# Content/session domain (separate connection via SessionDB).
for rec in _iter_content(db_path, config=config, include_content=content_allowed):
counts["sessions"] += 1
if fmt == "ndjson":
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
else:
records.append(rec)
if fmt != "ndjson":
json.dump({"records": records}, out, ensure_ascii=False, indent=2)
return counts
__all__ = ["export"]

219
agent/telemetry/metrics.py
View File

@ -1,219 +0,0 @@
"""Derive metric rollups from the local telemetry tables.
Reads the ``tel_*`` tables in state.db and returns aggregates for /usage, /insights,
and local dashboards. Metrics are computed by querying the event log rather than being
emitted on the hot path.
Each function accepts either an open caller-owned ``conn`` (reused, not closed) or a
``db_path`` (opened and closed internally). InsightsEngine passes its existing
connection; a standalone dashboard passes a path.
"""
from __future__ import annotations
import sqlite3
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional
@contextmanager
def _cursor(
conn: Optional[sqlite3.Connection], db_path: Optional[Path]
) -> Iterator[sqlite3.Connection]:
"""Yield a Row-factory connection. Closes it only if we opened it."""
if conn is not None:
prev_factory = conn.row_factory
conn.row_factory = sqlite3.Row
try:
yield conn
finally:
conn.row_factory = prev_factory
return
if db_path is None:
from hermes_constants import get_hermes_home
db_path = get_hermes_home() / "state.db"
c = sqlite3.connect(str(db_path), timeout=5.0)
c.row_factory = sqlite3.Row
try:
yield c
finally:
c.close()
def _since_clause(since_ns: Optional[int], col: str = "start_ns") -> str:
return f" WHERE {col} >= {int(since_ns)}" if since_ns else ""
def workflow_summary(
db_path: Optional[Path] = None,
since_ns: Optional[int] = None,
*,
conn: Optional[sqlite3.Connection] = None,
) -> Dict[str, Any]:
"""Run-level counters + duration percentiles (local telemetry, exact)."""
with _cursor(conn, db_path) as c:
where = _since_clause(since_ns)
total = c.execute(f"SELECT COUNT(*) n FROM tel_runs{where}").fetchone()["n"]
by_reason = {
r["end_reason"] or "unknown": r["n"]
for r in c.execute(
f"SELECT end_reason, COUNT(*) n FROM tel_runs{where} GROUP BY end_reason"
).fetchall()
}
by_entry = {
r["entrypoint"] or "unknown": r["n"]
for r in c.execute(
f"SELECT entrypoint, COUNT(*) n FROM tel_runs{where} GROUP BY entrypoint"
).fetchall()
}
dur_where = (where + " AND end_ns IS NOT NULL") if where else " WHERE end_ns IS NOT NULL"
durations = [
(r["end_ns"] - r["start_ns"]) / 1e6
for r in c.execute(
f"SELECT start_ns, end_ns FROM tel_runs{dur_where}"
).fetchall()
]
return {
"total_runs": total,
"by_end_reason": by_reason,
"by_entrypoint": by_entry,
"duration_ms_p50": _pct(durations, 50),
"duration_ms_p95": _pct(durations, 95),
"success_rate": round(by_reason.get("completed", 0) / total, 4) if total else 0.0,
}
def model_call_summary(
db_path: Optional[Path] = None,
since_ns: Optional[int] = None,
*,
conn: Optional[sqlite3.Connection] = None,
) -> Dict[str, Any]:
with _cursor(conn, db_path) as c:
rows = c.execute(
"SELECT provider, model, COUNT(*) n, "
"SUM(input_tokens) inp, SUM(output_tokens) outp, "
"SUM(cache_read_tokens) cache, AVG(latency_ms) avg_latency "
"FROM tel_model_calls GROUP BY provider, model"
).fetchall()
by_provider: Dict[str, int] = {}
by_model: Dict[str, int] = {}
tokens = {"input": 0, "output": 0, "cache_read": 0}
breakdown: List[Dict[str, Any]] = []
for r in rows:
prov = r["provider"] or "unknown"
mdl = r["model"] or "unknown"
by_provider[prov] = by_provider.get(prov, 0) + r["n"]
by_model[mdl] = by_model.get(mdl, 0) + r["n"]
tokens["input"] += r["inp"] or 0
tokens["output"] += r["outp"] or 0
tokens["cache_read"] += r["cache"] or 0
breakdown.append({
"provider": r["provider"],
"model": r["model"],
"calls": r["n"],
"avg_latency_ms": round(r["avg_latency"] or 0, 1),
})
cache_total = tokens["cache_read"] + tokens["input"]
return {
"by_provider": by_provider,
"by_model": by_model,
"tokens": tokens,
"cache_hit_rate": round(tokens["cache_read"] / cache_total, 4) if cache_total else 0.0,
"breakdown": breakdown,
}
def tool_call_summary(
db_path: Optional[Path] = None,
*,
conn: Optional[sqlite3.Connection] = None,
) -> Dict[str, Any]:
with _cursor(conn, db_path) as c:
by_tool = {
r["tool_name"] or "unknown": r["n"]
for r in c.execute(
"SELECT tool_name, COUNT(*) n FROM tel_tool_calls GROUP BY tool_name"
).fetchall()
}
fails = {
r["tool_name"] or "unknown": r["n"]
for r in c.execute(
"SELECT tool_name, COUNT(*) n FROM tel_tool_calls "
"WHERE result_class IN ('error','timeout','blocked') GROUP BY tool_name"
).fetchall()
}
total = sum(by_tool.values())
total_fail = sum(fails.values())
return {
"by_tool": by_tool,
"failures_by_tool": fails,
"total": total,
"failure_rate": round(total_fail / total, 4) if total else 0.0,
}
def error_summary(
db_path: Optional[Path] = None,
*,
conn: Optional[sqlite3.Connection] = None,
) -> Dict[str, Any]:
with _cursor(conn, db_path) as c:
return {
"by_class": {
r["error_class"] or "unknown": r["n"]
for r in c.execute(
"SELECT error_class, COUNT(*) n FROM tel_error_events GROUP BY error_class"
).fetchall()
},
}
def _pct(values: List[float], p: int) -> float:
if not values:
return 0.0
s = sorted(values)
k = (len(s) - 1) * (p / 100)
lo = int(k)
hi = min(lo + 1, len(s) - 1)
frac = k - lo
return round(s[lo] + (s[hi] - s[lo]) * frac, 2)
def overview(
db_path: Optional[Path] = None,
since_ns: Optional[int] = None,
*,
conn: Optional[sqlite3.Connection] = None,
) -> Dict[str, Any]:
"""One call for a dashboard: all the rollups."""
return {
"workflows": workflow_summary(db_path, since_ns, conn=conn),
"model_calls": model_call_summary(db_path, since_ns, conn=conn),
"tool_calls": tool_call_summary(db_path, conn=conn),
"errors": error_summary(db_path, conn=conn),
}
def has_data(
db_path: Optional[Path] = None,
*,
conn: Optional[sqlite3.Connection] = None,
) -> bool:
"""True when any telemetry runs exist (cheap guard for /insights rendering)."""
try:
with _cursor(conn, db_path) as c:
return c.execute("SELECT 1 FROM tel_runs LIMIT 1").fetchone() is not None
except Exception:
return False
__all__ = [
"workflow_summary",
"model_call_summary",
"tool_call_summary",
"error_summary",
"overview",
"has_data",
]

72
agent/telemetry/policy.py
View File

@ -1,72 +0,0 @@
"""Telemetry consent posture and the aggregate-metrics gate.
Consent is a single config field, ``telemetry.consent_state``:
* "unknown" — no choice recorded; never uploads (the default).
* "local" — declined aggregate metrics; local telemetry only.
* "aggregate" — opted in to aggregate metrics.
The config file is the source of truth: set ``telemetry.consent_state`` with
``hermes config set`` (or a managed-scope pin). Callers that gate behavior read
``telemetry.*`` directly from config; this module only provides the consent
constants, the install-id helper, and the upload gate a future uploader must
consult.
``allow_aggregate`` is the hard gate. An administrator pins
``telemetry.allow_aggregate: false`` through the managed-scope layer
(``/etc/hermes/config.yaml``), which takes precedence over the user's config; when
it is false, aggregate metrics are off regardless of ``consent_state``.
"""
from __future__ import annotations
import uuid
from typing import Any, Dict
CONSENT_UNKNOWN = "unknown"
CONSENT_LOCAL = "local"
CONSENT_AGGREGATE = "aggregate"
VALID_CONSENT_STATES = {CONSENT_UNKNOWN, CONSENT_LOCAL, CONSENT_AGGREGATE}
def _telemetry_cfg(config: Dict[str, Any]) -> Dict[str, Any]:
cfg = config.get("telemetry") if isinstance(config, dict) else None
return cfg if isinstance(cfg, dict) else {}
def ensure_install_id(config: Dict[str, Any]) -> str:
"""Return a stable install id, minting one if the config slot is empty.
Does not persist — the caller writes the returned value back to config.yaml. A
fresh uuid4 is used; clearing ``telemetry.install_id`` (e.g. with
``hermes config set telemetry.install_id ""``) causes the next call to mint anew.
"""
tel = _telemetry_cfg(config)
existing = tel.get("install_id")
if isinstance(existing, str) and existing.strip():
return existing
return str(uuid.uuid4())
def may_upload_aggregate(config: Dict[str, Any]) -> bool:
"""Whether aggregate metrics may upload — the gate a future uploader consults.
Aggregate metrics are derived from the local telemetry tables, so they require
local telemetry to be on. True only when local telemetry is enabled, the admin
hard gate allows it, and the user has opted in via ``telemetry.consent_state``.
"""
tel = _telemetry_cfg(config)
local_enabled = bool(tel.get("local", True))
allow_aggregate = bool(tel.get("allow_aggregate", True))
state = tel.get("consent_state", CONSENT_UNKNOWN)
return local_enabled and allow_aggregate and state == CONSENT_AGGREGATE
__all__ = [
"CONSENT_UNKNOWN",
"CONSENT_LOCAL",
"CONSENT_AGGREGATE",
"VALID_CONSENT_STATES",
"may_upload_aggregate",
"ensure_install_id",
]

187
agent/telemetry/redaction.py
View File

@ -1,187 +0,0 @@
"""Redaction applied to telemetry data on export.
Two independent controls:
* Secrets are always redacted, on every export and in every mode; no setting
disables this. Wraps ``agent/redact.py::redact_sensitive_text(force=True)``.
* Whether message bodies, reasoning, and raw tool arguments are exportable at all is
governed by the trajectories setting (``telemetry.trajectories.enabled``, default
off, admin-pinnable), not by a redaction mode. With trajectories off, content is
dropped. With it on, content is exportable and ``content_redaction`` (none|pii)
controls how much is scrubbed; secrets are still always stripped.
This applies to the local and trajectory export paths. It is unrelated to any
aggregate-metrics path.
"""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional
# Content-redaction strengths for any content that IS exported.
CONTENT_NONE = "none" # drop content entirely (structural telemetry only)
CONTENT_PII = "pii" # codec-aware PII redaction on exported content
CONTENT_MODES = {CONTENT_NONE, CONTENT_PII}
# ── PII patterns (applied only in CONTENT_PII mode, on content that is exported) ──
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}")
# E.164-ish and common separators; conservative to avoid nuking code/IDs.
_PHONE_RE = re.compile(
r"(?<!\w)(?:\+?\d{1,3}[\s.\-]?)?(?:\(\d{2,4}\)[\s.\-]?)?\d{3}[\s.\-]?\d{3,4}(?:[\s.\-]?\d{2,4})?(?!\w)"
)
# Long opaque hex/uuid-ish user identifiers.
_UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
def _secret_redact(text: Optional[str]) -> Optional[str]:
"""Always-on secret redaction. force=True so user config can't disable it."""
if text is None:
return None
try:
from agent.redact import redact_sensitive_text
return redact_sensitive_text(str(text), force=True)
except Exception:
# Fail CLOSED: if the redactor can't run, do not emit the raw string.
return "[redaction-unavailable]"
def _pii_redact(text: str) -> str:
text = _EMAIL_RE.sub("[email]", text)
text = _UUID_RE.sub("[id]", text)
text = _PHONE_RE.sub("[phone]", text)
return text
def redact_for_export(
text: Optional[str],
*,
content_mode: str = CONTENT_NONE,
) -> Optional[str]:
"""Redact a single content string for export.
Secrets are ALWAYS stripped. Then PII is stripped when content_mode is 'pii'.
Callers gate *whether content is exported at all* via telemetry.trajectories
(see ``content_export_enabled``); this function only scrubs content that the
caller has already decided to export.
"""
redacted = _secret_redact(text)
if redacted is None:
return None
if content_mode == CONTENT_PII:
redacted = _pii_redact(redacted)
return redacted
def content_export_enabled(config: Optional[Dict[str, Any]]) -> bool:
"""True only when telemetry.trajectories is explicitly enabled.
This is the consent gate for exporting message bodies / reasoning / raw tool
args. Default off. Admin-pinnable via managed scope (telemetry.trajectories.enabled).
"""
try:
tel = (config or {}).get("telemetry") or {}
traj = tel.get("trajectories") or {}
return bool(traj.get("enabled", False))
except Exception:
return False
def content_mode_for(config: Optional[Dict[str, Any]]) -> str:
try:
tel = (config or {}).get("telemetry") or {}
mode = tel.get("content_redaction", CONTENT_NONE)
return mode if mode in CONTENT_MODES else CONTENT_NONE
except Exception:
return CONTENT_NONE
# ── Codec-aware message redaction (NeMo pattern) ─────────────────────────────
# Redact the right fields of a provider message shape rather than regex-blasting
# the whole blob. Structure (roles, names, counts) is preserved; only the
# free-text content fields are scrubbed.
def redact_message(
msg: Dict[str, Any],
*,
content_mode: str = CONTENT_NONE,
include_content: bool = False,
) -> Dict[str, Any]:
"""Redact one chat message dict for export.
When include_content is False (trajectories off), content/reasoning/tool-arg
fields are dropped — only structural fields (role, tool name, counts) remain.
When True, those fields are kept but passed through redact_for_export.
"""
role = msg.get("role")
out: Dict[str, Any] = {"role": role}
# Always-structural fields.
if msg.get("tool_name") is not None:
out["tool_name"] = msg.get("tool_name")
if msg.get("name") is not None:
out["name"] = msg.get("name")
if not include_content:
# Structural only: record presence/size, not bytes.
c = msg.get("content")
if c is not None:
out["content_chars"] = len(str(c))
if msg.get("reasoning_content"):
out["reasoning_chars"] = len(str(msg["reasoning_content"]))
if msg.get("tool_calls"):
out["tool_call_count"] = _count_tool_calls(msg["tool_calls"])
return out
# Content included (trajectories enabled): scrub then keep.
if msg.get("content") is not None:
out["content"] = redact_for_export(msg["content"], content_mode=content_mode)
if msg.get("reasoning_content"):
out["reasoning_content"] = redact_for_export(
msg["reasoning_content"], content_mode=content_mode
)
if msg.get("tool_calls"):
out["tool_calls"] = _redact_tool_calls(msg["tool_calls"], content_mode=content_mode)
return out
def _count_tool_calls(tool_calls: Any) -> int:
try:
import json
tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls
return len(tc) if isinstance(tc, list) else (1 if tc else 0)
except Exception:
return 0
def _redact_tool_calls(tool_calls: Any, *, content_mode: str) -> Any:
"""Redact raw tool-call arguments (free text) while keeping function names."""
import json
try:
tc = json.loads(tool_calls) if isinstance(tool_calls, str) else tool_calls
except Exception:
return "[unparseable-tool-calls]"
if not isinstance(tc, list):
return []
out: List[Dict[str, Any]] = []
for call in tc:
if not isinstance(call, dict):
continue
fn = (call.get("function") or {}) if isinstance(call.get("function"), dict) else {}
name = fn.get("name") or call.get("name")
args = fn.get("arguments")
red_args = redact_for_export(args, content_mode=content_mode) if args is not None else None
out.append({"name": name, "arguments": red_args})
return out
__all__ = [
"CONTENT_NONE",
"CONTENT_PII",
"CONTENT_MODES",
"redact_for_export",
"content_export_enabled",
"content_mode_for",
"redact_message",
]

144
agent/telemetry/rollup.py
View File

@ -1,144 +0,0 @@
"""Build per-run summary events from the local telemetry tables.
Reads the ``tel_*`` tables and projects each completed run into a summary dict holding
the recorded values: provider, models used, tool names, token totals, duration, and
cost. Powers ``hermes telemetry preview``. No aggregation or bucketing is applied here.
"""
from __future__ import annotations
import platform
import sqlite3
from pathlib import Path
from typing import Any, Dict, List, Optional
def _os_family() -> str:
s = platform.system().lower()
if s.startswith("lin"):
return "linux"
if s == "darwin":
return "macos"
if s.startswith("win"):
return "windows"
return "other"
def _hermes_version() -> str:
try:
from hermes_cli import __version__
return str(__version__)
except Exception:
return "0.0.0"
def _open(db_path: Optional[Path], conn: Optional[sqlite3.Connection]):
if conn is not None:
prev = conn.row_factory
conn.row_factory = sqlite3.Row
return conn, prev, False
if db_path is None:
from hermes_constants import get_hermes_home
db_path = get_hermes_home() / "state.db"
c = sqlite3.connect(str(db_path), timeout=5.0)
c.row_factory = sqlite3.Row
return c, None, True
def _run_events(c: sqlite3.Connection, since_ns: Optional[int]) -> List[Dict[str, Any]]:
"""Project completed runs into per-run summary dicts."""
where = " WHERE end_ns IS NOT NULL"
if since_ns:
where += f" AND start_ns >= {int(since_ns)}"
rows = c.execute(
"SELECT run_id, entrypoint, platform, end_reason, start_ns, end_ns, "
"model_call_count, tool_call_count, error_count "
"FROM tel_runs" + where
).fetchall()
events: List[Dict[str, Any]] = []
for r in rows:
# Models actually used in this run (real ids), with token totals.
models = [
{"provider": m["provider"], "model": m["model"],
"calls": m["n"], "input_tokens": int(m["inp"] or 0),
"output_tokens": int(m["outp"] or 0)}
for m in c.execute(
"SELECT provider, model, COUNT(*) n, SUM(input_tokens) inp, "
"SUM(output_tokens) outp FROM tel_model_calls WHERE run_id = ? "
"GROUP BY provider, model ORDER BY n DESC",
(r["run_id"],),
).fetchall()
]
tools = [
row["tool_name"]
for row in c.execute(
"SELECT DISTINCT tool_name FROM tel_tool_calls WHERE run_id = ?",
(r["run_id"],),
).fetchall()
if row["tool_name"]
]
trow = c.execute(
"SELECT SUM(input_tokens) inp, SUM(output_tokens) outp "
"FROM tel_model_calls WHERE run_id = ?",
(r["run_id"],),
).fetchone()
duration_ms = (r["end_ns"] - r["start_ns"]) / 1e6 if r["end_ns"] else None
events.append({
"event_name": "workflow_completed",
"run_id": r["run_id"],
"entrypoint": r["entrypoint"] or "cli",
"platform": r["platform"],
"end_reason": r["end_reason"] or "completed",
"models_used": models,
"tools_used": tools,
"model_call_count": r["model_call_count"] or 0,
"tool_call_count": r["tool_call_count"] or 0,
"error_count": r["error_count"] or 0,
"duration_ms": round(duration_ms, 1) if duration_ms is not None else None,
"input_tokens": int((trow["inp"] if trow else 0) or 0),
"output_tokens": int((trow["outp"] if trow else 0) or 0),
})
return events
def build_aggregate_events(
*,
install_id: str,
db_path: Optional[Path] = None,
since_ns: Optional[int] = None,
conn: Optional[sqlite3.Connection] = None,
include_heartbeat: bool = True,
) -> List[Dict[str, Any]]:
"""Return per-run summary events plus an optional heartbeat."""
c, prev_factory, owned = _open(db_path, conn)
try:
events = _run_events(c, since_ns)
if include_heartbeat:
events.append({
"event_name": "heartbeat",
"install_id": install_id,
"hermes_version": _hermes_version(),
"os_family": _os_family(),
"entrypoint": "cli",
})
return events
finally:
if owned:
c.close()
elif prev_factory is not None:
c.row_factory = prev_factory
def summarize(events: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Counts by event_name + field coverage, for status/preview output."""
by_name: Dict[str, int] = {}
fields = set()
for e in events:
name = e.get("event_name", "?")
by_name[name] = by_name.get(name, 0) + 1
fields.update(e.keys())
return {"total": len(events), "by_event_name": by_name, "fields_present": sorted(fields)}
__all__ = ["build_aggregate_events", "summarize"]

83
agent/telemetry/spans.py
View File

@ -1,83 +0,0 @@
"""Trace / run / span id propagation via contextvars.
Telemetry events share IDs so a workflow can be reconstructed: one ``trace_id`` per
workflow, one ``run_id`` per top-level execution, ``span_id`` per timed operation, and
``parent_span_id`` for nesting. These live in contextvars so async tool calls and
spawned subagents inherit the lineage automatically.
Provides helpers to start/clear a run context and mint child span ids. The telemetry
plugin sets the run context on session start and reads it in each hook callback.
Nothing here writes to storage — it only carries ids.
"""
from __future__ import annotations
import contextvars
import uuid
from dataclasses import dataclass
from typing import Optional
_trace_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
"hermes_tel_trace_id", default=None
)
_run_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
"hermes_tel_run_id", default=None
)
_parent_span_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
"hermes_tel_parent_span_id", default=None
)
def new_id() -> str:
return uuid.uuid4().hex
@dataclass(slots=True)
class RunContext:
trace_id: str
run_id: str
def start_run(trace_id: Optional[str] = None, run_id: Optional[str] = None) -> RunContext:
"""Begin a run context, minting ids when not supplied. Sets contextvars."""
tid = trace_id or new_id()
rid = run_id or new_id()
_trace_id.set(tid)
_run_id.set(rid)
_parent_span_id.set(None)
return RunContext(trace_id=tid, run_id=rid)
def current_trace_id() -> Optional[str]:
return _trace_id.get()
def current_run_id() -> Optional[str]:
return _run_id.get()
def current_parent_span_id() -> Optional[str]:
return _parent_span_id.get()
def new_span_id() -> str:
"""Mint a span id (does not alter the parent pointer)."""
return new_id()
def clear_run() -> None:
_trace_id.set(None)
_run_id.set(None)
_parent_span_id.set(None)
__all__ = [
"RunContext",
"new_id",
"start_run",
"current_trace_id",
"current_run_id",
"current_parent_span_id",
"new_span_id",
"clear_run",
]

43
cli-config.yaml.example
View File

@ -1414,49 +1414,6 @@ display:
# # Routing/delivery still uses the original values internally.
# redact_pii: false
# =============================================================================
# Telemetry & Observability
# =============================================================================
# Three settings, isolated from each other:
# local — full-fidelity observability you own. Default ON.
# aggregate — opt-in metadata to Nous (no uploader ships yet). Default OFF.
# trajectories — content trajectories for training. Separate consent (later).
#
# Local telemetry records real values (actual models, providers, tool names) —
# your own data, on your machine. Aggregate metrics are opt-in and have no
# uploader today; if one ships it would summarize at that egress boundary.
#
# Enterprise locking: any telemetry.* key can be pinned by an administrator via
# the managed-scope layer (/etc/hermes/config.yaml), which wins over the user's
# value. To hard-forbid egress on a locked-down deployment, pin
# `telemetry.allow_aggregate: false` there.
# telemetry:
# # Local telemetry: event log + SQLite index in state.db. Never leaves your
# # machine unless you export it or opt into aggregate metrics.
# local: true
# # Hard gate for aggregate metrics. When false, aggregate metrics are off
# # regardless of consent_state. Pin false via managed scope to forbid egress.
# allow_aggregate: true
# # Aggregate-metrics consent (the opt-in). No uploader ships yet.
# # unknown (no choice — never uploads) | local (declined) | aggregate (opted in)
# consent_state: unknown
# # Stable install id (aggregate metrics only). Empty = mint on first use;
# # clear it to rotate.
# install_id: ""
# # Local event-log retention before rotation (days).
# retention_days: 90
# # Keep secret redaction on even at full local capture.
# redact_secrets: true
# # Content redaction for exports / support bundles: none | pii.
# content_redaction: none
# # Exporters. The OTLP exporter sends spans to a configured Collector endpoint;
# # header values reference environment variable names, not inline secrets.
# export:
# otlp:
# enabled: false
# endpoint: null
# headers_env: {} # e.g. {Authorization: MY_OTLP_TOKEN_ENVVAR}
# =============================================================================
# Shell-script hooks
# =============================================================================

7
docs/observability/README.md
View File

@ -307,13 +307,6 @@ nested agent work or security lifecycle events.
## Existing Consumers
The bundled **`telemetry`** plugin is the built-in local-first telemetry system: it
records runs, spans, model calls, and tool calls to a local event log + `state.db`, powers
`hermes insights`, and supports export to your own file or OpenTelemetry Collector. It
is on by default and never sends data to Nous unless you opt in. See
[`telemetry.md`](./telemetry.md) for the full feature, the `hermes telemetry` commands,
and enterprise export.
The bundled Langfuse plugin demonstrates direct hook-based observability for
turns, provider requests, and tool calls.

101
docs/observability/monitoring.md Normal file
View File

@ -0,0 +1,101 @@
# Gateway Monitoring
Service health monitoring plus redacted operational diagnostics for the
Hermes gateway daemon, exported over OTLP/HTTP to an operator-configured
endpoint (OpenTelemetry Collector, DataDog, or any OTLP receiver).
This plane is content-free by construction. It exports gateway lifecycle
state, platform connector health, and redacted warning/error diagnostics.
It never exports prompts, messages, tool arguments or results, session
history, usage analytics, audit logs, or execution traces. Run/model/tool
trajectory capture is a separate plane served by the NeMo Relay integration
(`plugins/observability/nemo_relay/`) and its Hermes-owned subscribers.
## What gets exported
| Signal | OTLP route | Content |
| --- | --- | --- |
| Gateway gauges | `/v1/metrics` | `hermes.gateway.up/state/busy/drainable/active_agents/restart_requested`, `hermes.platform.up/degraded` with bounded `error_code` attributes |
| Health/lifecycle events | `/v1/traces` | `gateway.lifecycle` state transitions (`starting -> running -> draining -> stopped`, `startup_failed`, exit), `gateway.health_snapshot`, platform state changes |
| Diagnostics | `/v1/logs` | Warning/error gateway log events with secrets AND PII scrubbed in-process before egress (`[redacted]` / `[email]`), bounded error classes |
Every signal carries resource attributes (`service.name`, profile, version,
install id, supervision mode) so an operator can tell instances apart.
## Enabling
```yaml
# config.yaml
monitoring:
gateway_health_export:
enabled: true
export:
otlp:
enabled: true
endpoint: http://collector-host:4318/v1/traces # metrics/logs derive
headers_env: {} # header name -> ENV VAR NAME (values never stored)
```
Check the posture any time:
```bash
hermes monitoring status
```
The OpenTelemetry SDK is an optional extra (`pip install 'hermes-agent[otlp]'`),
lazy-installed on first use. When the SDK is missing or the endpoint is down,
the gateway runs unaffected: every export path is fail-open and off the hot
path (events flow through a fire-and-forget in-process emitter; a slow or
failing exporter can never block gateway code).
Works identically under systemd/launchd/s6 supervision, containers, tmux, or
a plain `hermes gateway run` — the exporter lives in the gateway process, so
no sidecar, agent, or collector is required on the host.
## Collecting into DataDog
Run a customer-owned OpenTelemetry Collector and forward:
```yaml
# otel-collector config
receivers:
otlp:
protocols:
http:
exporters:
datadog:
api:
key: ${env:DD_API_KEY}
service:
pipelines:
metrics: {receivers: [otlp], exporters: [datadog]}
traces: {receivers: [otlp], exporters: [datadog]}
logs: {receivers: [otlp], exporters: [datadog]}
```
Point `monitoring.export.otlp.endpoint` at the collector. Alerts belong on
`hermes.gateway.up`, `hermes.platform.up`, and `hermes.platform.degraded`.
## Local smoke test (no Docker)
```bash
# terminal 1: capture collector on :4318
python scripts/observability/otel_capture_collector.py \
--host 127.0.0.1 --port 4318 --log /tmp/hermes_otel_capture.jsonl
# terminal 2: drive the real exporter through lifecycle transitions,
# a fatal platform, and a redacted warning log, then flush
python scripts/observability/gateway_health_export_probe.py \
--endpoint http://127.0.0.1:4318/v1/traces \
--log /tmp/hermes_otel_capture.jsonl --wait 8
# exit 0 prints: {"requests": 6, "paths": ["/v1/logs", "/v1/metrics", "/v1/traces"]}
```
## Boundaries and roadmap
The `hermes monitoring` CLI intentionally exposes `status` only. Shared
client usage metrics and enterprise trace telemetry are being designed on
the NeMo Relay integration with their own consent, policy, and export
boundaries; this monitoring plane stays narrow so an operator can enable it
without touching any content-bearing signal. The telemetry surface may be
reorganized as that lands.

228
docs/observability/telemetry.md
View File

@ -1,228 +0,0 @@
# Telemetry & Observability
Hermes ships with a built-in, local-first telemetry system. It records what your
agent does — workflows, model calls, tool calls, errors — to your own machine, powers
`/insights`, and (when *you* enable it) exports everything to your own observability
stack. It is private by default and never sends your data to Nous unless you explicitly
opt in to anonymous aggregate metrics.
This page explains the whole feature: the three telemetry settings, what's captured, the
`hermes telemetry` commands, and how an enterprise streams or exports all of its data
to its own infrastructure.
> Looking for the **plugin hook contract** (how to write your own observer plugin)?
> That's in [`README.md`](./README.md). This page is about the built-in telemetry
> system and its CLI.
## The three settings
Telemetry has three settings, isolated from each other:
| Setting | What it holds | Default | Destination |
| --- | --- | --- | --- |
| **local** | Full-fidelity observability — runs, model/tool calls (real model & provider names), durations, errors | **on** | your machine only |
| **aggregate** | Opt-in metadata to Nous (no uploader ships yet) | **off** (opt-in) | Nous, only if you enable it |
| **trajectories** | Full message content / reasoning / raw tool args | **off** (opt-in) | your own export destinations only |
Local telemetry is the one you'll use day to day — it records the real values that
happened (actual model ids, providers, tool names). Aggregate metrics are the only
thing that could ever leave for Nous; they are opt-in, default-off, and have no uploader
today. Trajectories unlock full-content export to *your own* destinations — never wired
to Nous.
## Local telemetry — always-on observability
Local telemetry is implemented as a bundled `telemetry` plugin that listens to Hermes
lifecycle hooks (model calls, tool calls, session start/finalize) and writes events to:
- an append-only JSONL log at `~/.hermes/telemetry/events.jsonl` (the source of truth)
- indexed `tel_*` tables in `state.db` (a rebuildable index for fast queries):
`tel_runs`, `tel_spans`, `tel_model_calls`, `tel_tool_calls`, `tel_error_events`
Writes are fire-and-forget on a background thread: telemetry can never block, slow, or
fail a model call or tool call. If local telemetry is disabled (`telemetry.local: false`)
the plugin does not load at all.
### Traces and spans
A **run** is one session (from `on_session_start` to `on_session_finalize`). Each run gets
a root span in `tel_spans`, and every model and tool call within it is recorded as a child
span (timing + `parent_span_id` = the run's root) keyed by the same `span_id` as its
detail row in `tel_model_calls` / `tel_tool_calls`. So a run reconstructs as a connected
`run -> calls` tree, ordered by `start_ns` and joinable to the per-call detail — the shape
a trace viewer or any OpenTelemetry backend can render.
Call spans are timed from the measured latency/duration the hooks report; cross-run
subagent lineage (linking a delegated child run to its parent) is not yet recorded.
### Seeing your local data
```bash
hermes insights # usage report — now includes an "Observability" section
hermes telemetry status # settings, consent, export posture, local data volume
```
The `insights` Observability section shows workflow counts and success rate, duration
p50/p95, tool failure rates by tool, provider/model mix, and cache hit rate —
all computed locally with exact values.
## `hermes telemetry` commands
```text
hermes telemetry status Show settings, consent state, export posture, local volume
hermes telemetry preview Show the aggregate events that would be produced (local)
hermes telemetry export Export local telemetry to a file/stream or OTLP endpoint
```
Consent and the install id are plain config, not separate verbs — set them with
`hermes config set` (or a managed-scope pin):
```bash
hermes config set telemetry.consent_state aggregate # opt in to aggregate metrics
hermes config set telemetry.consent_state local # opt out (local telemetry stays on)
hermes config set telemetry.install_id "" # reset the install id (mints a new one)
```
### Aggregate metrics (opt-in)
Aggregate metrics are **off by default** and have **no uploader today** — nothing is
sent to Nous. Consent lives in `telemetry.consent_state` (`unknown` / `local` /
`aggregate`); setting it to `aggregate` records the opt-in for if/when an uploader ships.
If one is built, it would summarize at that egress boundary.
`hermes telemetry preview` shows your recent runs as they'd be summarized — computed and
shown **locally only**, with the real model and tool names from your own telemetry. It's
a local inspection surface, not an upload.
## Enterprise: getting all of your data
Everything below sends data to **your own** destination — a file, your SIEM, or your own
OpenTelemetry Collector. None of it goes to Nous.
### Bulk export to a file
```bash
# Structural telemetry only (default — no message content)
hermes telemetry export --out telemetry.ndjson
# JSON instead of NDJSON, last 7 days only
hermes telemetry export --out dump.json --format json --since 7
```
By default the export is **structural** — runs, model/tool-call metadata, session shells
with message *counts* but no message bodies.
### Including content (trajectories)
To export full message content, enable trajectories. This is a deliberate, separate
consent — it's how an enterprise opts into exporting work-product content to its own
store:
```yaml
# config.yaml
telemetry:
trajectories:
enabled: true # unlocks content export to YOUR destination
content_redaction: pii # "none" | "pii"
```
```bash
hermes telemetry export --out full.ndjson --include-content
```
`--include-content` is a no-op unless trajectories are enabled — the config setting
governs, not the flag.
### Live streaming to your OpenTelemetry Collector / SIEM (OTLP)
Hermes can stream telemetry to your own OTLP endpoint. This requires the optional `otlp`
extra:
```bash
pip install 'hermes-agent[otlp]'
```
```yaml
# config.yaml
telemetry:
export:
otlp:
enabled: true
endpoint: "https://collector.your-corp.internal:4318/v1/traces"
headers_env: # secrets by reference — env var NAMES, not values
Authorization: MY_OTLP_TOKEN_ENVVAR
```
Set the referenced environment variable, then run the export:
```bash
hermes telemetry export --otlp # drain current telemetry to your collector
```
The token value lives only in the environment variable named by `headers_env`. The
config holds the *name* of an environment variable rather than the secret itself; the
value is read at export time and is never written to config or logged.
Each telemetry event is exported as an OTel span carrying its recorded attributes
(provider, model, tokens, duration, etc.). The `tel_spans` timing/parent linkage is not
yet reconstructed into connected OTel `SpanContext`s, so spans currently arrive as
independent records rather than a connected trace tree; that projection is planned.
## Redaction
Two independent controls govern what content looks like on export:
| Control | Values | Effect |
| --- | --- | --- |
| Secret redaction | always on | API keys, tokens, auth headers, connection strings are **always** stripped on every export path. Cannot be disabled. |
| `content_redaction` | `none` \| `pii` | When content is exported, `pii` additionally redacts emails, phone numbers, and id-shaped strings. |
Secret redaction is always on — even at full content fidelity — because a SIEM or
warehouse full of live credentials is a bigger attack target than the data it holds. It
fails closed: if the redactor can't run, the raw string is not emitted.
## Configuration reference
```yaml
telemetry:
local: true # local telemetry (default on)
allow_aggregate: true # hard gate; pin false to forbid aggregate metrics entirely
consent_state: unknown # aggregate opt-in: unknown | local | aggregate
install_id: "" # stable anon id; "" mints one; clear to rotate
retention_days: 90 # local event-log retention
redact_secrets: true # always-on secret redaction (kept on by design)
content_redaction: none # none | pii
trajectories:
enabled: false # unlocks full-content export to your destination
export:
otlp:
enabled: false
endpoint: null
headers_env: {} # {HeaderName: ENV_VAR_NAME}
```
### Enterprise policy via managed scope
Any `telemetry.*` key can be pinned by an administrator through Hermes' managed-scope
layer (`/etc/hermes/config.yaml`), which wins over the user's value on a per-key basis.
There is no telemetry-specific policy block — to lock down a fleet, pin the keys you care
about. Common examples:
- `telemetry.allow_aggregate: false` — aggregate metrics stay off even if
`consent_state` is set to `aggregate`.
- `telemetry.export.otlp.endpoint` — point every install at the corporate collector.
- `telemetry.trajectories.enabled` — centrally decide whether content export is allowed.
When a key is managed, attempts to change it are rejected by managed scope with a message
naming the source. `hermes telemetry status` shows the current export posture (endpoint
host, whether the auth env var is set, content gate, redaction modes) — it never prints
secret values.
## Privacy summary
- Local telemetry never leaves your machine.
- Aggregate metrics (the only thing that could go to Nous) are opt-in, default-off,
and have no uploader today — nothing is sent.
- All export surfaces (file, OTLP) point at *your* destinations.
- Secrets are always redacted on export; content export is off until you enable
trajectories; PII redaction is a knob.

14
gateway/run.py
View File

@ -7810,6 +7810,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
write_runtime_status(gateway_state="starting", exit_reason=None)
except Exception:
pass
try:
from hermes_cli.config import load_config
from agent.monitoring.gateway_health_export import start_gateway_health_export
self._gateway_health_export_runtime = start_gateway_health_export(load_config())
if getattr(self._gateway_health_export_runtime, "enabled", False):
logger.info("Gateway health OTLP export: enabled")
except Exception:
logger.debug("gateway health OTLP export startup failed", exc_info=True)
# Log any active supply-chain security advisories. Operators see this
# in gateway.log and `hermes status` surfaces it; we do NOT block
@ -9762,6 +9770,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
self._update_runtime_status("running", self._exit_reason)
else:
self._update_runtime_status("stopped", self._exit_reason)
try:
_gh_runtime = getattr(self, "_gateway_health_export_runtime", None)
if _gh_runtime is not None:
_gh_runtime.shutdown()
except Exception:
logger.debug("gateway health OTLP export shutdown failed", exc_info=True)
logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed())
self._stop_task = asyncio.create_task(_stop_impl())

7
gateway/status.py
View File

@ -11,6 +11,7 @@ that will be useful when we add named profiles (multiple agents running
concurrently under distinct configurations).
"""
import copy
import hashlib
import json
import logging
@ -987,6 +988,7 @@ def write_runtime_status(
"""Persist gateway runtime health information for diagnostics/status."""
path = _get_runtime_status_path()
payload = _read_json_file(path) or _build_runtime_status_record()
previous_payload = copy.deepcopy(payload)
current_record = _build_pid_record()
payload.setdefault("platforms", {})
payload["kind"] = current_record["kind"]
@ -1021,6 +1023,11 @@ def write_runtime_status(
payload["platforms"][platform] = platform_payload
_write_json_file(path, payload)
try:
from agent.monitoring.gateway_health import emit_runtime_status_transition
emit_runtime_status_transition(previous_payload, payload)
except Exception:
pass
def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]:

43
hermes_cli/config.py
View File

@ -3082,6 +3082,49 @@ DEFAULT_CONFIG = {
"force_ipv4": False,
},
# Gateway monitoring — Service Health Monitoring plus redacted Operational
# Diagnostics for the gateway daemon, exported over OTLP to an
# operator-configured endpoint (OTEL Collector, DataDog, ...). Content-free
# by construction: no prompts, messages, tool args/results, session
# history, usage analytics, audit logs, or trajectories. Off by default;
# nothing is collected or sent until an operator enables it and sets an
# endpoint.
"monitoring": {
# Stable install identifier attached to exported health signals so an
# operator can tell instances apart in their collector. Empty string
# means "mint a fresh UUID on first use"; clear it to rotate. Carries
# no account identity.
"install_id": "",
# Gateway health & diagnostics export.
"gateway_health_export": {
"enabled": False,
"metrics_enabled": True,
"diagnostic_events_enabled": True,
"warning_error_events_enabled": True,
"export_interval_seconds": 60,
"logs_export_interval_seconds": 5,
"resource_attributes": {
"service.name": "hermes-gateway",
"deployment.environment": "production",
},
"redaction": {
"enabled": True,
"include_stack_summary": True,
"include_raw_stack": False,
},
},
# OTLP destination. headers_env maps header names to ENVIRONMENT
# VARIABLE NAMES (never secret values); values are read from the
# environment at export time.
"export": {
"otlp": {
"enabled": False,
"endpoint": "",
"headers_env": {},
},
},
},
# Gateway settings — control how messaging platforms (Telegram, Discord,
# Slack, etc.) deliver agent-produced files as native attachments.
"gateway": {

226
hermes_cli/main.py
View File

@ -463,7 +463,7 @@ from hermes_cli.subcommands.memory import build_memory_parser
from hermes_cli.subcommands.acp import build_acp_parser
from hermes_cli.subcommands.tools import build_tools_parser
from hermes_cli.subcommands.insights import build_insights_parser
from hermes_cli.subcommands.telemetry import build_telemetry_parser
from hermes_cli.subcommands.monitoring import build_monitoring_parser
from hermes_cli.subcommands.skills import build_skills_parser
from hermes_cli.subcommands.pairing import build_pairing_parser
from hermes_cli.subcommands.plugins import build_plugins_parser
@ -13859,11 +13859,11 @@ _BUILTIN_SUBCOMMANDS = frozenset(
"dump", "egress", "fallback", "gateway", "hooks", "import", "insights",
"gui", "desktop", "kanban", "login", "logout", "logs", "lsp", "mcp", "memory", "migrate", "moa",
"journey", "memory-graph", "learning",
"model", "pairing", "pets", "plugins", "portal", "profile",
"model", "monitoring", "pairing", "pets", "plugins", "portal", "profile",
"project", "proxy",
"prompt-size",
"send", "sessions", "setup",
"skin", "skills", "slack", "status", "telemetry", "tools", "uninstall", "update",
"skin", "skills", "slack", "status", "tools", "uninstall", "update",
"version", "webhook", "whatsapp", "whatsapp-cloud", "chat", "secrets", "security",
# Help-ish invocations — plugin commands not being listed in
# top-level --help is an acceptable trade-off for skipping an
@ -14312,193 +14312,51 @@ def cmd_insights(args):
print(f"Error generating insights: {e}")
def cmd_telemetry(args):
"""Local-only telemetry control + inspection. No uploader exists yet."""
import json as _json
import time
def cmd_monitoring(args):
"""Gateway monitoring status: health & diagnostics export posture."""
from hermes_cli.config import load_config
from hermes_cli.config import load_config, save_config
from agent.telemetry import policy, rollup, metrics
action = getattr(args, "telemetry_action", None) or "status"
action = getattr(args, "monitoring_action", None) or "status"
config = load_config()
tel = config.get("telemetry") if isinstance(config.get("telemetry"), dict) else {}
local_enabled = bool(tel.get("local", True))
allow_aggregate = bool(tel.get("allow_aggregate", True))
consent_state = tel.get("consent_state", policy.CONSENT_UNKNOWN)
if consent_state not in policy.VALID_CONSENT_STATES:
consent_state = policy.CONSENT_UNKNOWN
aggregate_enabled = (local_enabled and allow_aggregate
and consent_state == policy.CONSENT_AGGREGATE)
install_id = policy.ensure_install_id(config)
def _persist_install_id():
# Make sure a minted id is written back so it stays stable.
config.setdefault("telemetry", {})["install_id"] = install_id
save_config(config)
mon_raw = config.get("monitoring")
mon: dict = mon_raw if isinstance(mon_raw, dict) else {}
if action == "status":
print("Telemetry status")
print("─" * 56)
print(f" Local telemetry: {'on' if local_enabled else 'off'} "
f"(telemetry.local)")
print(f" Aggregate metrics: {'on' if aggregate_enabled else 'off'} "
f"(opt-in; consent_state={consent_state})")
if consent_state == policy.CONSENT_AGGREGATE and not local_enabled:
print(" ⚠ inert: local telemetry is off — nothing to aggregate")
elif consent_state != policy.CONSENT_AGGREGATE and allow_aggregate:
print(" opt in: hermes config set telemetry.consent_state aggregate")
if not allow_aggregate:
print(" ⚠ allow_aggregate is false (egress hard-disabled)")
print(f" Install id: {install_id}")
print(" Upload: DISABLED — no server yet. Aggregate metrics are "
"computed locally only.")
print()
# Export posture — where YOUR data can flow (never Nous). Values only;
# lock-state is managed scope's concern, surfaced by its own tooling.
try:
from agent.telemetry import otlp_exporter, redaction
otlp = otlp_exporter._otlp_config(config)
print(" Export")
if otlp.get("enabled") and otlp.get("endpoint"):
_host = str(otlp.get("endpoint"))
print(f" OTLP: enabled → {_host}")
# Show the header name + env var + whether it's set, never the value.
hdrs = otlp.get("headers_env") or {}
if hdrs:
import os as _os
for _h, _env in hdrs.items():
_state = "set" if _os.environ.get(str(_env)) else "NOT set"
print(f" auth: header '{_h}' ← ${_env} ({_state})")
_sdk = "installed" if otlp_exporter.is_available() else "MISSING — pip install 'hermes-agent[otlp]'"
print(f" SDK: {_sdk}")
else:
print(" OTLP: disabled (telemetry.export.otlp.enabled)")
# Content gate (telemetry.trajectories) + redaction posture.
if redaction.content_export_enabled(config):
print(f" Content export: on (trajectories enabled) — message "
f"content exportable")
else:
print(" Content export: off (trajectories disabled) — structural "
"telemetry only")
print(" Secret redaction: on (always)")
print(f" PII redaction: {redaction.content_mode_for(config)}")
print(" Bulk export: hermes telemetry export --out FILE [--otlp]")
except Exception:
pass
print()
# Local data volume
try:
ov = metrics.overview()
runs = ov["workflows"]["total_runs"]
mcs = sum(ov["model_calls"]["by_provider"].values())
tcs = ov["tool_calls"]["total"]
print(f" Local data: {runs:,} workflows · {mcs:,} model calls · {tcs:,} tool calls")
except Exception:
print(" Local data: (none yet)")
print("\n Inspect what would be shared: hermes telemetry preview")
return
from agent.monitoring import otlp_exporter
if action == "preview":
_persist_install_id()
since_ns = int((time.time() - args.days * 86400) * 1e9)
events = rollup.build_aggregate_events(
install_id=install_id, since_ns=since_ns
)
summary = rollup.summarize(events)
print("Telemetry preview — computed locally, NOT uploaded")
print("─" * 56)
print(f" Window: last {args.days} days Events: {summary['total']}")
for name, n in sorted(summary["by_event_name"].items()):
print(f" {name}: {n}")
print(" Shows the actual models, tools, and counts from local telemetry.")
print()
shown = events[: args.limit]
if getattr(args, "json", False):
print(_json.dumps(shown, indent=2))
gh_raw = mon.get("gateway_health_export")
gh: dict = gh_raw if isinstance(gh_raw, dict) else {}
export_raw = mon.get("export")
export_cfg: dict = export_raw if isinstance(export_raw, dict) else {}
otlp_raw = export_cfg.get("otlp")
otlp: dict = otlp_raw if isinstance(otlp_raw, dict) else {}
print("Gateway monitoring")
print(f" Health export: {'enabled' if gh.get('enabled') else 'disabled'} "
f"(monitoring.gateway_health_export.enabled)")
if gh.get("enabled"):
print(f" Metrics: {'on' if gh.get('metrics_enabled', True) else 'off'} "
f"(interval {gh.get('export_interval_seconds', 60)}s)")
print(f" Diagnostic events: {'on' if gh.get('diagnostic_events_enabled', True) else 'off'}")
print(f" Warning/error logs: {'on' if gh.get('warning_error_events_enabled', True) else 'off'} "
f"(interval {gh.get('logs_export_interval_seconds', 5)}s)")
red_raw = gh.get("redaction")
red: dict = red_raw if isinstance(red_raw, dict) else {}
print(f" Redaction: {'on' if red.get('enabled', True) else 'OFF'} "
f"(secrets/PII scrubbed in-process before egress)")
endpoint = otlp.get("endpoint") or ""
if otlp.get("enabled") and endpoint:
print(f" OTLP endpoint: {endpoint}")
else:
for e in shown:
if e.get("event_name") == "heartbeat":
print(f" • heartbeat hermes={e.get('hermes_version')} os={e.get('os_family')}")
continue
bits = [f"{e.get('event_name')}"]
for k in ("entrypoint", "platform", "end_reason", "duration_ms"):
if e.get(k) is not None:
bits.append(f"{k}={e[k]}")
models = e.get("models_used") or []
if models:
bits.append("models=" + ",".join(
f"{m.get('model') or m.get('provider') or '?'}" for m in models))
if e.get("tools_used"):
bits.append("tools=" + ",".join(e["tools_used"]))
print(" • " + " ".join(bits))
if len(events) > len(shown):
print(f" ... and {len(events) - len(shown)} more (use --limit)")
print(" OTLP endpoint: not configured (monitoring.export.otlp)")
print(f" OTel SDK: {'installed' if otlp_exporter.is_available() else 'not installed'} "
f"(optional extra: hermes-agent[otlp])")
print("\n Scope: gateway service health + redacted diagnostics only.")
print(" No prompts, messages, tool args/results, usage analytics, or traces.")
return
if action == "export":
import sys as _sys
from agent.telemetry import exporter_bulk, redaction
since_ns = int((time.time() - args.since * 86400) * 1e9) if getattr(args, "since", 0) else None
want_content = getattr(args, "include_content", False)
# OTLP path: stream spans to the configured Collector endpoint.
if getattr(args, "otlp", False):
from agent.telemetry import otlp_exporter
if not otlp_exporter.is_enabled(config):
print("telemetry.export.otlp is not enabled/endpoint not set. "
"Set telemetry.export.otlp.enabled + .endpoint.", file=_sys.stderr)
return
# The OTel SDK is an optional dep; export_once() lazily installs it on
# first use (gated by security.allow_lazy_installs, TTY-prompted),
# same as every other optional backend. OTLPUnavailable is the
# fallback when it can't be installed.
try:
n = otlp_exporter.export_once(config, since_ns=since_ns)
except otlp_exporter.OTLPUnavailable as e:
print(str(e), file=_sys.stderr)
return
except Exception as e:
print(f"OTLP export failed: {e}", file=_sys.stderr)
return
print(f"Exported {n} spans to the OTLP endpoint "
f"({(otlp_exporter._otlp_config(config) or {}).get('endpoint')}).",
file=_sys.stderr)
return
content_ok = redaction.content_export_enabled(config)
if want_content and not content_ok:
print("⚠ --include-content ignored: telemetry.trajectories.enabled is false.")
print(" Enable trajectories (admin/config) to export message content.")
print(" Exporting structural telemetry only.")
if not getattr(args, "out", None):
print("--out is required (or use --otlp).", file=_sys.stderr)
return
out_path = args.out
if out_path == "-":
counts = exporter_bulk.export(
_sys.stdout, fmt=args.fmt, since_ns=since_ns,
include_content=want_content, config=config,
)
else:
with open(out_path, "w", encoding="utf-8") as fh:
counts = exporter_bulk.export(
fh, fmt=args.fmt, since_ns=since_ns,
include_content=want_content, config=config,
)
# status to stderr so stdout stays clean for `--out -`
msg = (f"Exported {counts['telemetry']} telemetry records"
+ (f" + {counts['sessions']} sessions" if counts["sessions"] else "")
+ (" (content INCLUDED, redacted)" if counts["content_included"]
else " (structural only)")
+ (f" -> {out_path}" if out_path != "-" else ""))
print(msg, file=_sys.stderr)
return
print(f"Unknown telemetry action: {action}")
print("Use: status | preview | export")
print(f"Unknown monitoring action: {action}", file=sys.stderr)
sys.exit(2)
def cmd_skills(args):
@ -16447,9 +16305,7 @@ def main():
# insights command (parser built in hermes_cli/subcommands/insights.py)
# =========================================================================
build_insights_parser(subparsers, cmd_insights=cmd_insights)
# telemetry command (parser in hermes_cli/subcommands/telemetry.py)
build_telemetry_parser(subparsers, cmd_telemetry=cmd_telemetry)
build_monitoring_parser(subparsers, cmd_monitoring=cmd_monitoring)
# =========================================================================
# claw command (parser built in hermes_cli/subcommands/claw.py)

36
hermes_cli/subcommands/monitoring.py Normal file
View File

@ -0,0 +1,36 @@
"""``hermes monitoring`` subcommand parser.
Gateway monitoring control and inspection. ``status`` shows whether the
gateway health & diagnostics export is enabled, where it points, and the
redaction posture.
The handler is injected to avoid importing ``main`` (mirrors the insights
subcommand).
"""
from __future__ import annotations
from typing import Callable
def build_monitoring_parser(subparsers, *, cmd_monitoring: Callable) -> None:
"""Attach the ``monitoring`` subcommand (with actions) to ``subparsers``."""
p = subparsers.add_parser(
"monitoring",
help="Inspect gateway monitoring (health & diagnostics export)",
description=(
"Gateway monitoring: service health metrics plus redacted "
"diagnostics, exported over OTLP to an operator-configured "
"endpoint. Content-free by construction — no prompts, messages, "
"tool args/results, or usage analytics. Configure under "
"monitoring.* in config.yaml."
),
)
sub = p.add_subparsers(dest="monitoring_action")
sub.add_parser(
"status",
help="Show monitoring settings, export state, and redaction posture",
)
p.set_defaults(func=cmd_monitoring)

54
hermes_cli/subcommands/telemetry.py
View File

@ -1,54 +0,0 @@
"""``hermes telemetry`` subcommand parser.
Telemetry control and inspection. ``preview`` shows the per-run summary events that
would be produced for aggregate metrics; there is no uploader, so it terminates as a
local view.
The handler is injected to avoid importing ``main`` (mirrors the insights subcommand).
"""
from __future__ import annotations
from typing import Callable
def build_telemetry_parser(subparsers, *, cmd_telemetry: Callable) -> None:
"""Attach the ``telemetry`` subcommand (with actions) to ``subparsers``."""
p = subparsers.add_parser(
"telemetry",
help="Inspect local telemetry and export it",
description=(
"Local-first telemetry. Local telemetry records observability on this "
"machine. Aggregate metrics are opt-in (set telemetry.consent_state via "
"`hermes config set`); they have no uploader and are shown only via `preview`."
),
)
sub = p.add_subparsers(dest="telemetry_action")
sub.add_parser("status", help="Show telemetry settings, consent state, and local data volume")
prev = sub.add_parser(
"preview",
help="Show the aggregate events that would be produced (computed locally, not uploaded)",
)
prev.add_argument("--days", type=int, default=30, help="Window to roll up (default: 30)")
prev.add_argument("--limit", type=int, default=10, help="Max events to print (default: 10)")
prev.add_argument("--json", action="store_true", help="Print raw JSON events")
exp = sub.add_parser(
"export",
help="Export local telemetry (and optional content) to a file, stream, or OTLP endpoint",
)
exp.add_argument("--out", help="Output file path (use - for stdout). Not needed with --otlp.")
exp.add_argument("--format", dest="fmt", choices=["ndjson", "json"], default="ndjson",
help="Output format (default: ndjson)")
exp.add_argument("--since", type=int, default=0,
help="Only telemetry from the last N days (0 = all)")
exp.add_argument("--include-content", action="store_true",
help="Include session/message content (requires telemetry.trajectories.enabled). "
"Secrets always redacted; PII per telemetry.content_redaction.")
exp.add_argument("--otlp", action="store_true",
help="Export to the configured OTLP endpoint (telemetry.export.otlp.*) "
"instead of a file. Requires the optional 'otlp' extra.")
p.set_defaults(func=cmd_telemetry)

382
plugins/telemetry/__init__.py
View File

@ -1,382 +0,0 @@
"""Telemetry plugin — wires Hermes lifecycle hooks to the local telemetry emitter.
This is the *only* instrumentation seam. It registers observational hooks (which core
already invokes fail-open) and translates each into a typed local telemetry event
handed to ``agent.telemetry.emitter``. There are zero edits to core call sites:
the hooks already carry model/provider/usage/duration/tool data.
Everything here is best-effort and fail-open — a raised exception in a hook callback is
swallowed by core, and we additionally guard each callback so a telemetry bug can never
disturb a session. No content, no network: local telemetry only.
Hooks consumed:
on_session_start -> begin a run context (trace_id/run_id + root span id)
post_api_request -> one model_call event + its timing span (tokens, latency)
api_request_error -> one error event
post_tool_call -> one tool_call event + its timing span (duration, result class)
on_session_finalize -> finalize the run row + emit the run's root span
subagent_start/stop -> (reserved) lineage markers
Each model/tool call emits a SpanEvent (timing + parent = the run's root span) keyed by
the same span_id as its detail row, so tel_spans reconstructs a run -> calls trace tree.
"""
from __future__ import annotations
import logging
import threading
import time
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
# Per-run accumulators keyed by run_id, so on_session_finalize can roll up counts.
_runs: Dict[str, Dict[str, Any]] = {}
_runs_lock = threading.Lock()
def _safe(fn):
"""Decorator: never let a telemetry hook raise into core."""
def wrapper(*args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception:
logger.debug("telemetry hook %s failed", getattr(fn, "__name__", "?"), exc_info=True)
return None
wrapper.__name__ = getattr(fn, "__name__", "wrapper")
return wrapper
def _run_key(session_id: Optional[str], task_id: Optional[str]) -> str:
return session_id or task_id or "default"
# ── on_session_start ────────────────────────────────────────────────────────
@_safe
def _on_session_start(**kw: Any) -> None:
from agent.telemetry import spans
session_id = kw.get("session_id") or ""
platform = kw.get("platform") or kw.get("source") or ""
ctx = spans.start_run()
now = time.time_ns()
root_span_id = spans.new_span_id()
key = _run_key(session_id, kw.get("task_id"))
with _runs_lock:
_runs[key] = {
"run_id": ctx.run_id,
"trace_id": ctx.trace_id,
"root_span_id": root_span_id,
"session_id": session_id or None,
"entrypoint": _entrypoint_for(platform, kw.get("source")),
"platform": platform or None,
"start_ns": now,
"model_call_count": 0,
"tool_call_count": 0,
"error_count": 0,
}
def _ensure_run(session_id: Optional[str], task_id: Optional[str], platform: str = "") -> Dict[str, Any]:
"""Return the run accumulator, lazily creating one if session_start was missed."""
from agent.telemetry import spans
key = _run_key(session_id, task_id)
with _runs_lock:
run = _runs.get(key)
if run is None:
rid = spans.current_run_id() or spans.new_id()
tid = spans.current_trace_id() or spans.new_id()
run = {
"run_id": rid,
"trace_id": tid,
"root_span_id": spans.new_span_id(),
"session_id": session_id or None,
"entrypoint": _entrypoint_for(platform),
"platform": platform or None,
"start_ns": time.time_ns(),
"model_call_count": 0,
"tool_call_count": 0,
"error_count": 0,
}
_runs[key] = run
return run
def _emit_call_span(run: Dict[str, Any], span_id: str, name: str, kind: str,
duration_ms: Optional[int], status: Optional[str]) -> None:
"""Emit the timing/lineage span for a model or tool call.
The call hooks fire on completion, so end_ns is ~now and start_ns is reconstructed
from the measured duration (end - duration). The span is parented to the run's root
so a 2-level run -> calls waterfall can be reconstructed from tel_spans.
"""
from agent.telemetry import emitter
from agent.telemetry.events import SpanEvent
end_ns = time.time_ns()
dur_ns = int(duration_ms) * 1_000_000 if isinstance(duration_ms, (int, float)) else 0
start_ns = end_ns - dur_ns
emitter.emit(SpanEvent(
span_id=span_id,
trace_id=run["trace_id"],
run_id=run["run_id"],
parent_span_id=run.get("root_span_id"),
name=name,
kind=kind,
start_ns=start_ns,
end_ns=end_ns,
status=status,
))
def _entrypoint_for(platform: Optional[str], source: Optional[str] = None) -> str:
"""Coarse entrypoint label (cli / gateway / tui / api / cron …).
This is a workflow *surface* label, not model/tool anonymization — it answers
"where did this run come from", which is genuinely categorical.
"""
s = (source or platform or "").lower()
if s in ("", "chat", "interactive", "cli", "desktop"):
return "cli"
if s in ("telegram", "discord", "slack", "whatsapp", "signal", "matrix",
"email", "sms", "teams", "feishu", "wecom", "line", "google_chat"):
return "gateway"
if s == "tui":
return "tui"
if s in ("api", "api_server", "openai_api"):
return "api"
if s == "cron":
return "cron"
if s == "batch":
return "batch"
if s == "acp":
return "acp"
return "cli"
# ── post_api_request -> model_call event ────────────────────────────────────
@_safe
def _on_post_api_request(**kw: Any) -> None:
from agent.telemetry import emitter, spans
from agent.telemetry.events import ModelCallEvent
session_id = kw.get("session_id") or ""
platform = kw.get("platform") or ""
run = _ensure_run(session_id, kw.get("task_id"), platform)
usage = kw.get("usage") or {}
duration = kw.get("api_duration")
latency_ms = int(duration * 1000) if isinstance(duration, (int, float)) else None
span_id = spans.new_span_id()
evt = ModelCallEvent(
span_id=span_id,
run_id=run["run_id"],
provider=kw.get("provider"), # raw
model=kw.get("model"), # raw
base_url=kw.get("base_url"),
input_tokens=int(usage.get("input_tokens") or 0),
output_tokens=int(usage.get("output_tokens") or 0),
cache_read_tokens=int(usage.get("cache_read_tokens") or 0),
cache_write_tokens=int(usage.get("cache_write_tokens") or 0),
reasoning_tokens=int(usage.get("reasoning_tokens") or 0),
latency_ms=latency_ms,
)
with _runs_lock:
run["model_call_count"] += 1
_emit_call_span(run, span_id, name=kw.get("model") or "model_call",
kind="model", duration_ms=latency_ms, status="ok")
emitter.emit(evt)
# ── api_request_error -> error event ────────────────────────────────────────
@_safe
def _on_api_request_error(**kw: Any) -> None:
from agent.telemetry import emitter
from agent.telemetry.events import ErrorEvent
session_id = kw.get("session_id") or ""
run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "")
error_class = _coarse_error_class(kw.get("error_type") or kw.get("error") or "")
with _runs_lock:
run["error_count"] += 1
emitter.emit(ErrorEvent(
run_id=run["run_id"],
error_class=error_class,
subsystem="model_api",
recovery=None,
))
def _coarse_error_class(raw: Any) -> str:
s = str(raw).lower()
if "timeout" in s:
return "provider_timeout"
if "rate" in s and "limit" in s:
return "rate_limit"
if any(k in s for k in ("auth", "401", "403", "unauthorized", "forbidden")):
return "auth"
if any(k in s for k in ("connection", "network", "dns", "socket")):
return "network"
if "context" in s and ("length" in s or "overflow" in s or "token" in s):
return "context_overflow"
if any(k in s for k in ("500", "502", "503", "server error", "provider")):
return "provider_error"
return "unknown"
# ── post_tool_call -> tool_call event ───────────────────────────────────────
@_safe
def _on_post_tool_call(**kw: Any) -> None:
from agent.telemetry import emitter, spans
from agent.telemetry.events import ToolCallEvent
session_id = kw.get("session_id") or ""
run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "")
function_name = kw.get("function_name") or kw.get("tool_name")
duration_ms = kw.get("duration_ms")
result = kw.get("result")
result_class = _tool_result_class(result)
with _runs_lock:
run["tool_call_count"] += 1
if result_class == "error":
run["error_count"] += 1
dur_int = int(duration_ms) if isinstance(duration_ms, (int, float)) else None
span_id = spans.new_span_id()
_emit_call_span(run, span_id, name=function_name or "tool_call",
kind="tool", duration_ms=dur_int, status=result_class)
emitter.emit(ToolCallEvent(
span_id=span_id,
run_id=run["run_id"],
tool_name=function_name, # raw tool name
duration_ms=dur_int,
result_class=result_class,
))
def _tool_result_class(result: Any) -> str:
"""Classify a tool result without retaining content — error vs ok vs blocked."""
try:
import json
if isinstance(result, str):
r = result.strip()
if r.startswith("{"):
obj = json.loads(r)
if isinstance(obj, dict):
if obj.get("error") or obj.get("blocked"):
return "blocked" if obj.get("blocked") else "error"
if obj.get("timeout"):
return "timeout"
return "ok"
if isinstance(result, dict):
if result.get("error"):
return "error"
return "ok"
except Exception:
return "ok"
return "ok"
# ── on_session_finalize -> finalize the run row ─────────────────────────────
@_safe
def _on_session_finalize(**kw: Any) -> None:
from agent.telemetry import emitter, spans
from agent.telemetry.events import RunEvent, SpanEvent
session_id = kw.get("session_id") or ""
key = _run_key(session_id, kw.get("task_id"))
with _runs_lock:
run = _runs.pop(key, None)
if run is None:
run = _ensure_run(session_id, kw.get("task_id"), kw.get("platform") or "")
with _runs_lock:
_runs.pop(key, None)
end_ns = time.time_ns()
start_ns = run.get("start_ns", end_ns)
end_reason = _coarse_end_reason(kw)
# Root span for the run — the trace root the call spans hang off of.
emitter.emit(SpanEvent(
span_id=run.get("root_span_id") or spans.new_span_id(),
trace_id=run["trace_id"],
run_id=run["run_id"],
parent_span_id=None,
name=f"run:{run.get('entrypoint', 'cli')}",
kind="run",
start_ns=start_ns,
end_ns=end_ns,
status=end_reason,
))
emitter.emit(RunEvent(
run_id=run["run_id"],
trace_id=run["trace_id"],
entrypoint=run.get("entrypoint", "cli"),
session_id=run.get("session_id"),
platform=run.get("platform"),
start_ns=start_ns,
end_ns=end_ns,
end_reason=end_reason,
model_call_count=run.get("model_call_count", 0),
tool_call_count=run.get("tool_call_count", 0),
error_count=run.get("error_count", 0),
))
spans.clear_run()
def _coarse_end_reason(kw: Dict[str, Any]) -> str:
"""Map a finalize payload to a coarse end reason.
Production finalize callers pass ``reason`` (e.g. "shutdown", "session_expired",
"session_reset"); the older ``turn_exit_reason``/``interrupted``/``failed`` keys are
honored when present. Defaults to "ended".
"""
if kw.get("interrupted"):
return "interrupted"
if kw.get("failed"):
return "failed"
reason = str(kw.get("turn_exit_reason") or kw.get("reason") or "").lower()
if "max_iteration" in reason:
return "max_iterations"
if "timeout" in reason:
return "timeout"
if "expired" in reason:
return "expired"
if "reset" in reason:
return "reset"
if "shutdown" in reason or "complete" in reason:
return "completed"
return reason or "ended"
# ── subagent lineage (reserved) ─────────────────────────────────────────────
# A delegated subagent runs its own ``run_conversation`` with its own session id, so
# its model/tool calls are already captured as a separate tel_runs row via the normal
# hooks — no subagent activity is lost. These hooks fire with the parent<->child bridge
# (parent_session_id, child_session_id, child_role, child_goal); they are reserved for
# recording parent->child *lineage* (linking a child run back to its parent), which
# needs a tel_runs.parent_run_id column. Deferred until a consumer needs the delegation
# tree; left registered as the attachment point.
@_safe
def _on_subagent_start(**kw: Any) -> None:
return None
@_safe
def _on_subagent_stop(**kw: Any) -> None:
return None
# ── registration ────────────────────────────────────────────────────────────
def register(ctx) -> None:
ctx.register_hook("on_session_start", _on_session_start)
ctx.register_hook("post_api_request", _on_post_api_request)
ctx.register_hook("api_request_error", _on_api_request_error)
ctx.register_hook("post_tool_call", _on_post_tool_call)
ctx.register_hook("on_session_finalize", _on_session_finalize)
ctx.register_hook("subagent_start", _on_subagent_start)
ctx.register_hook("subagent_stop", _on_subagent_stop)
logger.debug("telemetry plugin registered 7 hooks")

12
plugins/telemetry/plugin.yaml
View File

@ -1,12 +0,0 @@
name: telemetry
version: 1.0.0
description: "Local-first telemetry & observability. Records runs, model calls, tool calls, and errors to a local event log + state.db index via plugin hooks — no agent action, no content, no network. Powers /usage, /insights, and local dashboards."
author: NousResearch
hooks:
- post_api_request
- api_request_error
- post_tool_call
- on_session_start
- on_session_finalize
- subagent_start
- subagent_stop

9
pyproject.toml
View File

@ -156,10 +156,6 @@ edge-tts = ["edge-tts==7.2.7"]
modal = ["modal==1.3.4"]
daytona = ["daytona==0.155.0"]
hindsight = ["hindsight-client==0.6.1"]
# OTLP telemetry export (optional). Provides the OpenTelemetry SDK + OTLP/HTTP
# exporter so `hermes telemetry export --otlp` can send spans to a Collector.
# Lazy-imported by agent/telemetry/otlp_exporter.py; never a core dependency.
otlp = ["opentelemetry-sdk==1.30.0", "opentelemetry-exporter-otlp-proto-http==1.30.0"]
dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==81.0.0"] # starlette: CVE-2026-48710; setuptools: latest <82 (torch >=2.11 caps setuptools<82)
messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.14.1", "brotlicffi==1.2.0.1", "slack-bolt==1.29.0", "slack-sdk==3.43.0", "qrcode==7.4.2"] # aiohttp 3.14.1: CVE-2026-34513/34518/34519/34520/34525 + 34993(RCE)/47265
cron = [] # croniter is now a core dependency; this extra kept for back-compat
@ -225,6 +221,11 @@ acp = ["agent-client-protocol==0.9.0"]
# NOT re-added to [all] so a future quarantined release can't break fresh
# installs (see [all] policy comment below).
mistral = ["mistralai==2.4.8"]
# OTLP gateway monitoring export (optional). Provides the OpenTelemetry SDK +
# OTLP/HTTP exporter for monitoring.gateway_health_export. Lazy-installed via
# tools/lazy_deps.py on first use; never a core dependency and deliberately
# NOT in [all].
otlp = ["opentelemetry-sdk==1.39.1", "opentelemetry-exporter-otlp-proto-http==1.39.1"]
bedrock = ["boto3==1.42.89"]
vertex = ["google-auth==2.55.1"]
azure-identity = ["azure-identity==1.25.3"]

92
scripts/observability/gateway_health_export_probe.py Normal file
View File

@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""Exercise Gateway Health & Diagnostics Export against a local OTLP capture collector."""
from __future__ import annotations
import argparse
import json
import logging
import os
import sys
import tempfile
import time
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--endpoint", default="http://127.0.0.1:4318/v1/traces")
parser.add_argument("--log", required=True, help="JSONL file written by otel_capture_collector.py")
parser.add_argument("--wait", type=float, default=7.0)
args = parser.parse_args()
hermes_home = Path(tempfile.mkdtemp(prefix="hermes-otel-smoke-"))
os.environ["HERMES_HOME"] = str(hermes_home)
from gateway.status import write_runtime_status
from agent.monitoring.gateway_health_export import start_gateway_health_export
from agent.monitoring import emitter
config = {
"monitoring": {
"local": True,
"gateway_health_export": {
"enabled": True,
"metrics_enabled": True,
"diagnostic_events_enabled": True,
"warning_error_events_enabled": True,
"export_interval_seconds": 5,
"logs_export_interval_seconds": 5,
"resource_attributes": {
"service.name": "hermes-gateway-smoke",
"deployment.environment": "local-smoke",
},
"redaction": {"enabled": True, "include_raw_stack": False},
},
"export": {
"otlp": {
"enabled": True,
"endpoint": args.endpoint,
"headers_env": {},
}
},
}
}
runtime = start_gateway_health_export(config)
if not runtime.enabled:
raise SystemExit(f"gateway health exporter did not enable: {runtime.reason}")
write_runtime_status(gateway_state="starting", active_agents=0)
write_runtime_status(gateway_state="running", active_agents=2)
write_runtime_status(platform="slack", platform_state="running")
write_runtime_status(
platform="slack",
platform_state="fatal",
error_code="auth_failed",
error_message="Bearer *** rejected for smoke@example.com",
)
logging.getLogger("gateway.platforms.slack").warning("Slack token *** rejected for smoke@example.com")
emitter.get_emitter().flush(timeout=2.0)
time.sleep(args.wait)
runtime.shutdown()
emitter.get_emitter().flush(timeout=2.0)
log_path = Path(args.log)
rows = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()]
paths = {row["path"] for row in rows}
print(json.dumps({"hermes_home": str(hermes_home), "requests": len(rows), "paths": sorted(paths)}, indent=2))
if "/v1/traces" not in paths:
raise SystemExit("missing /v1/traces request")
if "/v1/logs" not in paths:
raise SystemExit("missing /v1/logs request")
if "/v1/metrics" not in paths:
raise SystemExit("missing /v1/metrics request")
if __name__ == "__main__":
main()

59
scripts/observability/otel_capture_collector.py Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""Tiny local OTLP/HTTP capture collector for Hermes gateway health smoke tests.
This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces
and /v1/metrics, records request metadata as JSONL, and returns 200 so local
exporters can be exercised without Docker or a vendor backend.
"""
from __future__ import annotations
import argparse
import json
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
class CaptureHandler(BaseHTTPRequestHandler):
log_path: Path
def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API
length = int(self.headers.get("content-length") or 0)
body = self.rfile.read(length) if length else b""
record = {
"ts": time.time(),
"path": self.path,
"content_type": self.headers.get("content-type"),
"content_length": length,
"body_prefix_hex": body[:24].hex(),
}
with self.log_path.open("a", encoding="utf-8") as fh:
fh.write(json.dumps(record, sort_keys=True) + "\n")
self.send_response(200)
self.end_headers()
self.wfile.write(b"{}")
def log_message(self, format: str, *args) -> None:
# Keep tmux panes clean; JSONL file is the assertion surface.
return
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=4318)
parser.add_argument("--log", required=True)
args = parser.parse_args()
log_path = Path(args.log).expanduser().resolve()
log_path.parent.mkdir(parents=True, exist_ok=True)
log_path.write_text("", encoding="utf-8")
CaptureHandler.log_path = log_path
server = ThreadingHTTPServer((args.host, args.port), CaptureHandler)
print(f"OTLP capture collector listening on http://{args.host}:{args.port}; log={log_path}", flush=True)
server.serve_forever()
if __name__ == "__main__":
main()

6
tests/hermes_cli/test_plugins.py
View File

@ -761,11 +761,7 @@ class TestPluginHooks:
mgr.discover_and_load()
assert mgr.has_hook("pre_api_request") is True
# Negative sentinel: a hook no plugin (user or bundled) registers, proving
# that registering pre_api_request doesn't conjure an unrelated hook.
# (post_api_request is now registered by the bundled telemetry plugin, so
# it is no longer a valid "nothing registered this" sentinel.)
assert mgr.has_hook("transform_terminal_output") is False
assert mgr.has_hook("post_api_request") is False
results = mgr.invoke_hook(
"pre_api_request",
session_id="s1",

0
tests/telemetry/__init__.py → tests/monitoring/__init__.py
View File

86
tests/monitoring/test_emitter.py Normal file
View File

@ -0,0 +1,86 @@
"""Tests for the monitoring emitter: hot-path invariant + subscriber fan-out."""
from __future__ import annotations
import time
from agent.monitoring.emitter import MonitoringEmitter
from agent.monitoring.events import GatewayHealthEvent
def test_emit_never_raises_when_disabled():
em = MonitoringEmitter(enabled=False)
em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"})
assert em.stats()["queued"] == 0
em.close()
def test_emit_accepts_dataclass_and_dict(tmp_path):
em = MonitoringEmitter()
seen: list = []
em.subscribe(lambda batch: seen.extend(batch))
em.emit(GatewayHealthEvent(name="gateway.health_snapshot", active_agents=2))
em.emit({"event": "gateway_diagnostic", "name": "platform.fatal",
"subsystem": "platform.slack"})
em.flush()
em.close()
kinds = {ev.get("event") for ev in seen}
assert kinds == {"gateway_health", "gateway_diagnostic"}
health = next(ev for ev in seen if ev["event"] == "gateway_health")
assert health["active_agents"] == 2
assert "ts_ns" in health
def test_subscriber_failure_is_isolated():
em = MonitoringEmitter()
good: list = []
def bad(batch):
raise RuntimeError("boom")
em.subscribe(bad)
em.subscribe(lambda batch: good.extend(batch))
em.emit({"event": "gateway_health", "name": "gateway.lifecycle"})
em.flush()
em.close()
assert len(good) == 1 # the raising subscriber did not break fan-out
def test_unsubscribe_stops_delivery():
em = MonitoringEmitter()
seen: list = []
cb = lambda batch: seen.extend(batch) # noqa: E731
em.subscribe(cb)
em.emit({"event": "gateway_health", "name": "a"})
em.flush()
em.unsubscribe(cb)
em.emit({"event": "gateway_health", "name": "b"})
em.flush()
em.close()
assert [ev["name"] for ev in seen] == ["a"]
def test_queue_full_drops_oldest():
em = MonitoringEmitter()
# Fill the queue without a dispatcher running by not letting it start:
# emit() starts the thread, so instead assert drop accounting via stats
# after a burst larger than the queue.
for i in range(11_000):
em.emit({"event": "gateway_health", "name": f"e{i}"})
# Give the dispatcher a moment; total dispatched + queued + dropped == emitted.
em.flush(timeout=5.0)
stats = em.stats()
em.close()
assert stats["dropped"] >= 0
assert stats["dispatched"] + stats["queued"] + stats["dropped"] >= 10_000
def test_hot_path_is_fast():
em = MonitoringEmitter()
start = time.perf_counter()
for _ in range(1_000):
em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"})
elapsed = time.perf_counter() - start
em.close()
# 1000 emits should be far under a second even on slow CI.
assert elapsed < 1.0

56
tests/monitoring/test_export_redaction.py Normal file
View File

@ -0,0 +1,56 @@
"""Export redaction pipeline tests — the security-critical layer.
Invariants:
* Secrets ALWAYS stripped, every export path, no flag disables it.
* Fails CLOSED: if the redactor can't run, the raw string is never emitted.
* PII (emails, phones, UUID-shaped ids) stripped in 'pii' mode — the mode
the gateway diagnostics path always uses.
"""
from __future__ import annotations
import agent.monitoring.redaction as R
def test_secret_always_stripped_in_none_mode():
fake_key = "sk-ant-api03-" + "A" * 24 # constructed to dodge literal-scrubbers
text = f"calling with key {fake_key} and moving on"
out = R.redact_for_export(text, content_mode=R.CONTENT_NONE)
assert out is not None
assert fake_key not in out
def test_secret_always_stripped_in_pii_mode():
fake_token = "ghp_" + "0123456789abcdef" * 2 + "0123"
text = f"token {fake_token} leaked"
out = R.redact_for_export(text, content_mode=R.CONTENT_PII)
assert out is not None
assert fake_token not in out
def test_none_passthrough():
assert R.redact_for_export(None, content_mode=R.CONTENT_NONE) is None
def test_pii_mode_strips_email_phone_uuid():
text = ("reach alice@example.com or +1 415 555 0100, "
"install 123e4567-e89b-12d3-a456-426614174000")
out = R.redact_for_export(text, content_mode=R.CONTENT_PII)
assert out is not None
assert "alice@example.com" not in out
assert "426614174000" not in out
assert "[email]" in out
assert "[id]" in out
def test_none_mode_keeps_ordinary_words():
out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE)
assert out == "just ordinary words"
def test_pii_mode_preserves_non_pii_structure():
text = "platform.slack entered fatal after auth_failed"
out = R.redact_for_export(text, content_mode=R.CONTENT_PII)
assert out is not None
assert "platform.slack" in out
assert "auth_failed" in out

342
tests/monitoring/test_gateway_health_export.py Normal file
View File

@ -0,0 +1,342 @@
from __future__ import annotations
import logging
def test_default_config_keeps_gateway_health_export_disabled():
from hermes_cli.config import DEFAULT_CONFIG
cfg = DEFAULT_CONFIG["monitoring"]["gateway_health_export"]
assert cfg["enabled"] is False
assert cfg["metrics_enabled"] is True
assert cfg["diagnostic_events_enabled"] is True
assert cfg["warning_error_events_enabled"] is True
assert cfg["export_interval_seconds"] == 60
assert cfg["logs_export_interval_seconds"] == 5
assert cfg["redaction"]["enabled"] is True
assert cfg["redaction"]["include_raw_stack"] is False
def test_gateway_health_snapshot_maps_runtime_status_to_low_cardinality_metrics():
from agent.monitoring.gateway_health import build_gateway_health_snapshot
runtime = {
"gateway_state": "running",
"pid": 1234,
"active_agents": "2",
"restart_requested": False,
"platforms": {
"slack": {"state": "running"},
"telegram": {
"state": "fatal",
"error_code": "auth_failed",
"error_message": "token xoxb-secret rejected for user 123",
},
},
}
snapshot = build_gateway_health_snapshot(
runtime,
gateway_running=True,
profile="default",
install_id="install-1",
version="2026.7.test",
supervision_mode="manual",
)
metric_names = {m.name for m in snapshot.metrics}
assert {
"hermes.gateway.up",
"hermes.gateway.active_agents",
"hermes.gateway.busy",
"hermes.gateway.drainable",
"hermes.gateway.restart_requested",
"hermes.platform.up",
"hermes.platform.degraded",
} <= metric_names
active = next(m for m in snapshot.metrics if m.name == "hermes.gateway.active_agents")
assert active.value == 2
assert active.attributes == {
"hermes.profile": "default",
"service.instance.id": "install-1",
"service.version": "2026.7.test",
"hermes.supervision_mode": "manual",
}
busy = next(m for m in snapshot.metrics if m.name == "hermes.gateway.busy")
drainable = next(m for m in snapshot.metrics if m.name == "hermes.gateway.drainable")
assert busy.value == 1
assert drainable.value == 1
degraded = next(
m for m in snapshot.metrics
if m.name == "hermes.platform.degraded" and m.attributes["hermes.platform"] == "telegram"
)
assert degraded.value == 1
assert degraded.attributes["hermes.error_code"] == "auth_failed"
assert all("secret" not in str(v).lower() for v in degraded.attributes.values())
def test_gateway_health_snapshot_emits_content_free_diagnostic_event():
from agent.monitoring.gateway_health import build_gateway_health_snapshot
snapshot = build_gateway_health_snapshot(
{
"gateway_state": "running",
"active_agents": 1,
"platforms": {
"slack": {"state": "fatal", "error_code": "auth_failed", "error_message": "Bearer sk-live-secret"},
},
},
gateway_running=True,
profile="default",
install_id="install-1",
version="v-test",
supervision_mode="container",
)
events = [event.to_dict() for event in snapshot.events]
health = next(e for e in events if e["event"] == "gateway_health")
platform = next(e for e in events if e["event"] == "gateway_diagnostic" and e["name"] == "platform.fatal")
assert health["gateway_state"] == "running"
assert health["active_agents"] == 1
assert health["gateway_busy"] is True
assert health["gateway_drainable"] is True
assert health["fatal_platform_count"] == 1
assert platform["platform"] == "slack"
assert platform["error_code"] == "auth_failed"
assert "secret" not in platform["redacted_message"].lower()
assert "Bearer" not in platform["redacted_message"]
def test_gateway_diagnostic_log_handler_redacts_and_filters(caplog):
from agent.monitoring import emitter
from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
captured = []
class DummyEmitter:
def emit(self, event):
captured.append(event.to_dict())
old = emitter.get_emitter
emitter.get_emitter = lambda: DummyEmitter() # type: ignore[assignment]
try:
handler = GatewayDiagnosticLogHandler(profile="default", version="v-test")
logger = logging.getLogger("gateway.platforms.slack")
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)
try:
logger.info("ignore info token sk-live-secret")
logger.warning("Slack token sk-live-secret failed for user@example.com")
finally:
logger.removeHandler(handler)
finally:
emitter.get_emitter = old # type: ignore[assignment]
assert len(captured) == 1
event = captured[0]
assert event["event"] == "gateway_diagnostic"
assert event["name"] == "gateway.log.warning"
assert event["subsystem"] == "platform.slack"
assert event["error_class"] == "auth_failed"
assert "***" not in event["redacted_message"]
assert "user@example.com" not in event["redacted_message"]
def test_runtime_status_transition_emits_lifecycle_and_platform_events(monkeypatch):
from agent.monitoring import emitter
from agent.monitoring.gateway_health import emit_runtime_status_transition
captured = []
class DummyEmitter:
def emit(self, event):
captured.append(event.to_dict())
old = emitter.emit
monkeypatch.setattr(emitter, "emit", lambda event: captured.append(event.to_dict()))
previous = {"gateway_state": "starting", "platforms": {"slack": {"state": "running"}}}
current = {
"gateway_state": "running",
"pid": 123,
"active_agents": 1,
"platforms": {
"slack": {
"state": "fatal",
"error_code": "auth_failed",
"error_message": "Bearer *** failed",
}
},
}
emit_runtime_status_transition(previous, current)
names = [e["name"] for e in captured]
assert "gateway.lifecycle" in names
assert "platform.state_change" in names
assert "platform.fatal" in names
lifecycle = next(e for e in captured if e["name"] == "gateway.lifecycle")
assert lifecycle["old_state"] == "starting"
assert lifecycle["new_state"] == "running"
platform = next(e for e in captured if e["name"] == "platform.state_change")
assert platform["old_state"] == "running"
assert platform["new_state"] == "fatal"
assert platform["error_code"] == "auth_failed"
assert "Bearer" not in platform["redacted_message"]
def test_runtime_status_transition_emits_startup_failed_and_exit():
from agent.monitoring.gateway_health import emit_runtime_status_transition
from agent.monitoring import emitter
captured = []
old = emitter.emit
emitter.emit = lambda event: captured.append(event.to_dict()) # type: ignore[assignment]
try:
emit_runtime_status_transition({"gateway_state": "starting"}, {"gateway_state": "startup_failed", "exit_reason": "startup token ***"})
emit_runtime_status_transition({"gateway_state": "running"}, {"gateway_state": "stopped", "exit_reason": "shutdown", "restart_requested": True})
finally:
emitter.emit = old # type: ignore[assignment]
names = [e["name"] for e in captured]
assert "gateway.startup_failed" in names
assert "gateway.exit" in names
failed = next(e for e in captured if e["name"] == "gateway.startup_failed")
assert "***" not in failed["redacted_message"]
exit_event = next(e for e in captured if e["name"] == "gateway.exit")
assert exit_event["restart_requested"] is True
def test_otlp_attrs_include_gateway_transition_fields():
from agent.monitoring.otlp_exporter import _span_attrs
attrs = _span_attrs({
"event": "gateway_health",
"name": "gateway.lifecycle",
"old_state": "starting",
"new_state": "running",
"exit_reason": "restart",
"restart_requested": True,
})
assert attrs["hermes.old_state"] == "starting"
assert attrs["hermes.new_state"] == "running"
assert attrs["hermes.exit_reason"] == "restart"
assert attrs["hermes.restart_requested"] is True
def test_gateway_health_export_start_is_fail_open_when_otlp_missing(monkeypatch):
from agent.monitoring import gateway_health_export
from agent.monitoring.gateway_health_export import GatewayHealthExportRuntime
monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("missing sdk")))
runtime = gateway_health_export.start_gateway_health_export({
"monitoring": {
"gateway_health_export": {"enabled": True},
"export": {"otlp": {"enabled": True, "endpoint": "http://collector:4317"}},
}
})
assert isinstance(runtime, GatewayHealthExportRuntime)
assert runtime.enabled is False
assert runtime.reason == "otlp_unavailable"
def test_gateway_health_export_streams_only_gateway_events(monkeypatch):
from agent.monitoring import gateway_health_export
captured = {}
def fake_start_streaming(config, *, event_filter=None):
captured["filter"] = event_filter
return object()
monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: None)
monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {})
monkeypatch.setattr(gateway_health_export, "_attach_log_handler", lambda *a, **k: None)
monkeypatch.setattr(gateway_health_export, "_emit_snapshot_events", lambda *a, **k: None)
monkeypatch.setattr(gateway_health_export, "_start_snapshot_thread", lambda *a, **k: None)
from agent.monitoring import otlp_exporter
monkeypatch.setattr(otlp_exporter, "start_streaming", fake_start_streaming)
runtime = gateway_health_export.start_gateway_health_export({
"monitoring": {
"gateway_health_export": {"enabled": True, "metrics_enabled": False},
"export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}},
}
})
assert runtime.enabled is True
event_filter = captured["filter"]
assert event_filter({"event": "gateway_health"}) is True
assert event_filter({"event": "gateway_diagnostic"}) is False
assert event_filter({"event": "run"}) is False
assert event_filter({"event": "model_call"}) is False
assert event_filter({"event": "tool_call"}) is False
def test_gateway_health_export_metric_failure_does_not_start_streamer(monkeypatch):
from agent.monitoring import gateway_health_export, otlp_exporter
started = []
monkeypatch.setattr(gateway_health_export, "_require_metrics_sdk", lambda *a, **k: {})
monkeypatch.setattr(gateway_health_export, "_start_metric_provider", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")))
monkeypatch.setattr(otlp_exporter, "start_streaming", lambda *a, **k: started.append(True))
runtime = gateway_health_export.start_gateway_health_export({
"monitoring": {
"gateway_health_export": {"enabled": True},
"export": {"otlp": {"enabled": True, "endpoint": "http://collector:4318/v1/traces"}},
}
})
assert runtime.enabled is False
assert runtime.reason == "metrics_start_failed"
assert started == []
def test_otlp_streamer_shutdown_unsubscribes(monkeypatch):
from agent.monitoring import emitter
from agent.monitoring.otlp_exporter import OTLPStreamer
class Dummy:
def force_flush(self):
pass
def shutdown(self):
pass
e = emitter.get_emitter()
streamer = OTLPStreamer.__new__(OTLPStreamer)
streamer._processor = Dummy()
streamer._provider = Dummy()
streamer._event_filter = None
streamer.exported = 0
e.subscribe(streamer)
assert streamer in e._subscribers
streamer.shutdown()
assert streamer not in e._subscribers
def test_gateway_diagnostic_log_handler_never_raises_on_malformed_record():
from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
handler = GatewayDiagnosticLogHandler(profile="default", version="v-test")
record = logging.LogRecord(
"gateway.platforms.slack",
logging.WARNING,
__file__,
1,
"broken %s %s",
("one",),
None,
)
handler.emit(record)

125
tests/monitoring/test_otlp_exporter.py Normal file
View File

@ -0,0 +1,125 @@
"""OTLP exporter tests: config resolution, span mapping, streaming subscriber.
No SQLite involved — monitoring is an egress path, so the exporter consumes
emitter batches directly. Uses the in-memory OTel span exporter; skipped when
the optional otlp extra is not installed.
"""
from __future__ import annotations
import pytest
otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed")
import agent.monitoring.otlp_exporter as OE
from agent.monitoring.emitter import MonitoringEmitter
def _mem_provider():
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
return provider, exporter
def test_gateway_health_event_maps_to_span_with_attrs():
provider, mem = _mem_provider()
n = OE.export_batch(provider, [{
"event": "gateway_health", "name": "gateway.lifecycle",
"old_state": "starting", "new_state": "running",
"active_agents": 2, "pid": 4242,
}])
assert n == 1
spans = mem.get_finished_spans()
assert spans[0].name == "hermes.gateway_health"
attrs = dict(spans[0].attributes or {})
assert attrs["hermes.old_state"] == "starting"
assert attrs["hermes.new_state"] == "running"
assert attrs["hermes.active_agents"] == 2
def test_gateway_diagnostic_event_maps_redacted_message():
provider, mem = _mem_provider()
OE.export_batch(provider, [{
"event": "gateway_diagnostic", "name": "platform.fatal",
"subsystem": "platform.slack", "error_class": "auth_failed",
"redacted_message": "token [redacted] rejected", "severity": "error",
}])
attrs = dict(mem.get_finished_spans()[0].attributes or {})
assert attrs["hermes.error_class"] == "auth_failed"
assert attrs["hermes.redacted_message"] == "token [redacted] rejected"
def test_unknown_event_kind_exports_no_attrs_beyond_kind():
provider, mem = _mem_provider()
OE.export_batch(provider, [{"event": "model_call", "provider": "anthropic",
"model": "claude-opus-4"}])
attrs = dict(mem.get_finished_spans()[0].attributes or {})
# Non-monitoring event kinds carry no attribute mapping on this plane.
assert attrs == {"hermes.event": "model_call"}
def test_headers_resolve_from_env_not_value(monkeypatch):
monkeypatch.setenv("DD_KEY_ENV", "secret-value")
resolved = OE._resolve_headers({"DD-API-KEY": "DD_KEY_ENV", "X-Missing": "NOPE_ENV"})
assert resolved == {"DD-API-KEY": "secret-value"}
def test_is_enabled_requires_endpoint_and_flag():
assert OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}})
assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"enabled": True}}}})
assert not OE.is_enabled({"monitoring": {"export": {"otlp": {"endpoint": "http://x"}}}})
assert not OE.is_enabled({})
def test_export_otlp_feature_specs_match_pyproject():
from tools.lazy_deps import LAZY_DEPS
import re
from pathlib import Path
specs = set(LAZY_DEPS["export.otlp"])
pyproject = Path(__file__).resolve().parents[2] / "pyproject.toml"
m = re.search(r'^otlp = \[(.*?)\]', pyproject.read_text(), re.M | re.S)
assert m, "otlp extra missing from pyproject.toml"
extra = set(re.findall(r'"([^"]+)"', m.group(1)))
assert specs == extra
def test_streamer_receives_events_and_respects_filter(monkeypatch):
provider, mem = _mem_provider()
monkeypatch.setattr(OE, "_make_provider", lambda cfg: (provider, None))
streamer = OE.OTLPStreamer(
{}, event_filter=lambda ev: ev.get("event") == "gateway_health")
em = MonitoringEmitter()
em.subscribe(streamer)
em.emit({"event": "gateway_health", "name": "gateway.health_snapshot"})
em.emit({"event": "model_call", "provider": "anthropic"}) # filtered out
em.flush()
em.close()
spans = mem.get_finished_spans()
assert [s.name for s in spans] == ["hermes.gateway_health"]
assert streamer.exported == 1
def test_failing_streamer_never_breaks_emitter(monkeypatch):
def boom(cfg):
raise RuntimeError("no provider")
em = MonitoringEmitter()
def bad_subscriber(batch):
raise RuntimeError("export down")
seen: list = []
em.subscribe(bad_subscriber)
em.subscribe(lambda batch: seen.extend(batch))
em.emit({"event": "gateway_health", "name": "gateway.lifecycle"})
em.flush()
em.close()
assert len(seen) == 1

108
tests/telemetry/test_cli_telemetry.py
View File

@ -1,108 +0,0 @@
"""`hermes telemetry` handler smoke tests (local-only; no upload)."""
from __future__ import annotations
import sqlite3
import time
import types
import pytest
import hermes_state
@pytest.fixture
def home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
# state.db with tel_* + seeded data
db = tmp_path / "state.db"
hermes_state.SessionDB(db_path=db)
from agent.telemetry.emitter import TelemetryEmitter
from agent.telemetry.events import RunEvent, ModelCallEvent, ToolCallEvent
em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "e.jsonl", db_path=db)
now = time.time_ns()
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed",
start_ns=now - 60_000_000, end_ns=now, model_call_count=1,
tool_call_count=1))
em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic",
model="claude-opus-4",
input_tokens=20000, output_tokens=2000))
em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search",
result_class="ok"))
em.flush()
em.close()
yield tmp_path
def _run(action, **kw):
from hermes_cli.main import cmd_telemetry
args = types.SimpleNamespace(telemetry_action=action, days=30, limit=10, json=False)
for k, v in kw.items():
setattr(args, k, v)
cmd_telemetry(args)
def test_status_runs(home, capsys):
_run("status")
out = capsys.readouterr().out
assert "Telemetry status" in out
assert "Upload:" in out and "DISABLED" in out
assert "Local data:" in out
def test_preview_shows_real_values(home, capsys):
_run("preview")
out = capsys.readouterr().out
assert "NOT uploaded" in out
assert "workflow_completed" in out
# real model + tool names ARE shown (this is the user's own local data)
assert "claude-opus-4" in out
assert "web_search" in out
def test_status_reflects_consent_set_via_config(home, capsys):
# Opting in is a plain config write now (no `enable` verb). status should
# reflect consent_state=aggregate as aggregate metrics being on.
from hermes_cli.config import load_config, save_config
cfg = load_config()
cfg.setdefault("telemetry", {})["consent_state"] = "aggregate"
save_config(cfg)
_run("status")
out = capsys.readouterr().out
assert "consent_state=aggregate" in out
assert "Aggregate metrics: on" in out
def test_status_shows_optin_hint_when_unknown(home, capsys):
_run("status")
out = capsys.readouterr().out
assert "Aggregate metrics: off" in out
assert "config set telemetry.consent_state aggregate" in out
def test_allow_aggregate_false_keeps_metrics_off_in_status(home, capsys):
# Even with consent opted in, a managed allow_aggregate:false wins.
from hermes_cli.config import load_config, save_config
cfg = load_config()
tel = cfg.setdefault("telemetry", {})
tel["consent_state"] = "aggregate"
tel["allow_aggregate"] = False
save_config(cfg)
_run("status")
out = capsys.readouterr().out
assert "Aggregate metrics: off" in out
assert "allow_aggregate is false" in out
def test_local_off_with_consent_shows_inert_in_status(home, capsys):
# local off + opted in: aggregate is off and the status explains why.
from hermes_cli.config import load_config, save_config
cfg = load_config()
tel = cfg.setdefault("telemetry", {})
tel["local"] = False
tel["consent_state"] = "aggregate"
save_config(cfg)
_run("status")
out = capsys.readouterr().out
assert "Aggregate metrics: off" in out
assert "inert: local telemetry is off" in out

107
tests/telemetry/test_emitter.py
View File

@ -1,107 +0,0 @@
"""Emitter tests — the hot-path invariant is the one that matters most.
Invariant: emit() never blocks, never raises, and a broken writer cannot slow or
break the caller. Plus: JSONL + SQLite round-trip, and the SQLite index is rebuildable
from the JSONL source of truth.
"""
from __future__ import annotations
import sqlite3
import time
import hermes_state
from agent.telemetry.emitter import TelemetryEmitter
from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent
def _fresh_db(tmp_path):
db = tmp_path / "state.db"
conn = sqlite3.connect(db)
conn.executescript(hermes_state.SCHEMA_SQL)
conn.close()
return db
def test_emit_is_fast_even_when_writer_is_broken(tmp_path, monkeypatch):
"""The core guarantee: a writer that raises AND sleeps cannot stall emit()."""
db = _fresh_db(tmp_path)
em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db)
# Sabotage the row indexer to raise after a long sleep.
def broken(_conn, _ev):
time.sleep(5.0)
raise RuntimeError("writer exploded")
monkeypatch.setattr(em, "_index_one", broken)
start = time.monotonic()
for i in range(50):
em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=10))
elapsed = time.monotonic() - start
# 50 emits must complete in well under the writer's single 5s sleep.
assert elapsed < 1.0, f"emit() blocked: {elapsed:.2f}s"
em.close()
def test_emit_never_raises_on_bad_event(tmp_path):
db = _fresh_db(tmp_path)
em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db)
# Non-serializable / wrong-shaped inputs must not raise out of emit().
em.emit(object()) # no to_dict, not a mapping
em.emit({"event": "run"}) # minimal dict
em.close()
def test_jsonl_and_sqlite_roundtrip(tmp_path):
db = _fresh_db(tmp_path)
jsonl = tmp_path / "telemetry" / "events.jsonl"
em = TelemetryEmitter(events_path=jsonl, db_path=db)
em.emit(RunEvent(run_id="run1", trace_id="t1", entrypoint="cli", end_reason="completed"))
em.emit(ModelCallEvent(span_id="m1", run_id="run1", provider="anthropic",
model="claude-opus-4", input_tokens=100, output_tokens=20))
em.emit(ToolCallEvent(span_id="tc1", run_id="run1", tool_name="web_search",
duration_ms=120, result_class="ok"))
em.flush()
em.close()
# JSONL has all three lines
lines = [l for l in jsonl.read_text(encoding="utf-8").splitlines() if l.strip()]
assert len(lines) == 3
# SQLite index has the rows in the right tables
conn = sqlite3.connect(db)
assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1
assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 1
assert conn.execute("SELECT COUNT(*) FROM tel_tool_calls").fetchone()[0] == 1
row = conn.execute("SELECT provider, model, input_tokens FROM tel_model_calls").fetchone()
assert row == ("anthropic", "claude-opus-4", 100)
conn.close()
def test_unknown_event_kind_is_ignored_not_fatal(tmp_path):
db = _fresh_db(tmp_path)
em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db)
em.emit({"event": "totally_unknown", "foo": "bar"})
em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli"))
em.flush()
em.close()
conn = sqlite3.connect(db)
# The unknown event is in JSONL but skipped by the indexer; the known one indexes.
assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 1
conn.close()
def test_disabled_emitter_writes_nothing(tmp_path):
db = _fresh_db(tmp_path)
jsonl = tmp_path / "telemetry" / "events.jsonl"
em = TelemetryEmitter(events_path=jsonl, db_path=db, enabled=False)
em.emit(RunEvent(run_id="r3", trace_id="t3", entrypoint="cli"))
em.flush()
em.close()
assert not jsonl.exists()
conn = sqlite3.connect(db)
assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0
conn.close()

113
tests/telemetry/test_export_redaction.py
View File

@ -1,113 +0,0 @@
"""Export redaction pipeline tests — the security-critical layer.
Invariants:
* Secrets ALWAYS stripped, every mode, no flag disables it.
* Content gated by telemetry.trajectories, not a redaction mode.
* PII stripped in 'pii' mode; structure preserved (codec-aware).
"""
from __future__ import annotations
import json
from agent.telemetry import redaction as R
# ── secrets are always redacted ─────────────────────────────────────────────
def test_secrets_stripped_in_none_mode():
text = "here is sk-ant-api03-SECRETKEY123 and a token"
out = R.redact_for_export(text, content_mode=R.CONTENT_NONE)
assert "SECRETKEY123" not in out
def test_secrets_stripped_in_pii_mode():
text = "Authorization: Bearer abcdef123456789secret"
out = R.redact_for_export(text, content_mode=R.CONTENT_PII)
assert "abcdef123456789secret" not in out
def test_secret_redactor_fails_closed(monkeypatch):
# If the underlying redactor raises, we must NOT return the raw string.
import agent.redact as ar
monkeypatch.setattr(ar, "redact_sensitive_text", lambda *a, **k: (_ for _ in ()).throw(RuntimeError()))
out = R.redact_for_export("sk-secret-value", content_mode=R.CONTENT_NONE)
assert "sk-secret-value" not in out
assert out == "[redaction-unavailable]"
# ── PII ─────────────────────────────────────────────────────────────────────
def test_pii_mode_strips_email_and_phone():
text = "contact alice@example.com or +1 415 555 1234"
out = R.redact_for_export(text, content_mode=R.CONTENT_PII)
assert "alice@example.com" not in out
assert "[email]" in out
assert "555" not in out or "[phone]" in out
def test_none_mode_keeps_nonsecret_text_but_drops_via_message_path():
# redact_for_export(none) scrubs secrets but doesn't strip ordinary words;
# content *dropping* happens at the message layer (trajectories gate).
out = R.redact_for_export("just ordinary words", content_mode=R.CONTENT_NONE)
assert "ordinary" in out
# ── trajectories gate (content_export_enabled) ──────────────────────────────
def test_content_export_disabled_by_default():
assert R.content_export_enabled({}) is False
assert R.content_export_enabled({"telemetry": {}}) is False
assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": False}}}) is False
def test_content_export_enabled_when_trajectories_on():
assert R.content_export_enabled({"telemetry": {"trajectories": {"enabled": True}}}) is True
# ── codec-aware message redaction ───────────────────────────────────────────
def test_message_structural_only_when_content_excluded():
msg = {"role": "user", "content": "my email is bob@x.com and key sk-12345"}
out = R.redact_message(msg, include_content=False)
assert out["role"] == "user"
assert "content" not in out # body dropped entirely
assert out["content_chars"] == len(msg["content"]) # only the size remains
assert "bob@x.com" not in json.dumps(out)
def test_message_content_included_is_redacted():
msg = {"role": "user", "content": "email bob@x.com secret sk-ant-SECRET999"}
out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True)
assert "content" in out
assert "SECRET999" not in out["content"] # secret gone
assert "bob@x.com" not in out["content"] # pii gone
assert "[email]" in out["content"]
def test_tool_calls_redacted_names_kept_args_scrubbed():
msg = {
"role": "assistant",
"tool_calls": json.dumps([
{"function": {"name": "web_search", "arguments": '{"q": "email me at z@z.com"}'}}
]),
}
out = R.redact_message(msg, content_mode=R.CONTENT_PII, include_content=True)
tc = out["tool_calls"]
assert tc[0]["name"] == "web_search" # structure/name preserved
assert "z@z.com" not in json.dumps(tc) # arg pii scrubbed
def test_tool_calls_counted_when_content_excluded():
msg = {
"role": "assistant",
"tool_calls": json.dumps([
{"function": {"name": "a", "arguments": "{}"}},
{"function": {"name": "b", "arguments": "{}"}},
]),
}
out = R.redact_message(msg, include_content=False)
assert out["tool_call_count"] == 2
assert "tool_calls" not in out
def test_content_mode_for_reads_config():
assert R.content_mode_for({"telemetry": {"content_redaction": "pii"}}) == "pii"
assert R.content_mode_for({"telemetry": {"content_redaction": "bogus"}}) == "none"
assert R.content_mode_for({}) == "none"

102
tests/telemetry/test_exporter_bulk.py
View File

@ -1,102 +0,0 @@
"""Bulk export tests — telemetry always, content only behind the trajectories gate."""
from __future__ import annotations
import io
import json
import sqlite3
import time
import hermes_state
from agent.telemetry import exporter_bulk
from agent.telemetry.emitter import TelemetryEmitter
from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent
def _seed(tmp_path, with_secret_content=True):
db = tmp_path / "state.db"
sdb = hermes_state.SessionDB(db_path=db)
# telemetry
em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db)
now = time.time_ns()
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed",
start_ns=now - 60_000_000, end_ns=now, model_call_count=1, tool_call_count=1))
em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic",
model="claude-sonnet-4", input_tokens=1000, output_tokens=100))
em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok"))
em.flush()
em.close()
# session + message content (with an embedded secret + email)
sdb.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4")
if with_secret_content:
sdb.append_message("s1", role="user",
content="my key is AKIAIOSFODNN7EXAMPLE and email me at carol@corp.com")
sdb.append_message("s1", role="assistant", content="ok")
sdb.close()
return db
def test_telemetry_exported_content_excluded_by_default(tmp_path):
db = _seed(tmp_path)
buf = io.StringIO()
counts = exporter_bulk.export(buf, fmt="ndjson", include_content=False, config={}, db_path=db)
assert counts["telemetry"] >= 3
assert counts["content_included"] == 0
text = buf.getvalue()
# message bodies must NOT be present
assert "carol@corp.com" not in text
assert "AKIAIOSFODNN7EXAMPLE" not in text
# but a session record (structural) is present with message structure
lines = [json.loads(l) for l in text.splitlines() if l.strip()]
sess = [r for r in lines if r.get("_kind") == "session"]
assert sess and sess[0]["messages"]
assert "content" not in sess[0]["messages"][0] # structural only
assert "content_chars" in sess[0]["messages"][0]
def test_include_content_ignored_without_trajectories(tmp_path):
db = _seed(tmp_path)
buf = io.StringIO()
# request content but trajectories disabled -> forced off
counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True,
config={"telemetry": {"trajectories": {"enabled": False}}}, db_path=db)
assert counts["content_included"] == 0
assert "carol@corp.com" not in buf.getvalue()
def test_content_included_when_trajectories_on_but_redacted(tmp_path):
db = _seed(tmp_path)
buf = io.StringIO()
cfg = {"telemetry": {"trajectories": {"enabled": True}, "content_redaction": "pii"}}
counts = exporter_bulk.export(buf, fmt="ndjson", include_content=True, config=cfg, db_path=db)
assert counts["content_included"] == 1
text = buf.getvalue()
# content present but secret + pii scrubbed
assert "AKIAIOSFODNN7EXAMPLE" not in text # secret always gone
assert "carol@corp.com" not in text # pii gone in pii mode
lines = [json.loads(l) for l in text.splitlines() if l.strip()]
sess = [r for r in lines if r.get("_kind") == "session"][0]
# the user message now has a (redacted) content field
user_msg = [m for m in sess["messages"] if m["role"] == "user"][0]
assert "content" in user_msg
assert "[email]" in user_msg["content"]
def test_json_format_roundtrips(tmp_path):
db = _seed(tmp_path)
buf = io.StringIO()
exporter_bulk.export(buf, fmt="json", include_content=False, config={}, db_path=db)
obj = json.loads(buf.getvalue())
assert "records" in obj
assert any(r["_kind"] == "tel_runs" for r in obj["records"])
def test_since_window_filters_runs(tmp_path):
db = _seed(tmp_path)
buf = io.StringIO()
# since 1ns ago in the future-ish -> the run (start ~60ms ago) excluded
future_ns = int((time.time() + 1) * 1e9)
counts = exporter_bulk.export(buf, fmt="ndjson", since_ns=future_ns, config={}, db_path=db)
lines = [json.loads(l) for l in buf.getvalue().splitlines() if l.strip()]
runs = [r for r in lines if r.get("_kind") == "tel_runs"]
assert runs == []

95
tests/telemetry/test_governance.py
View File

@ -1,95 +0,0 @@
"""Export configuration visibility in `hermes telemetry status`.
The status Export block reports the current export configuration. Whether a key is
locked is handled by the managed-scope layer, not repeated here; the allow_aggregate
gate is covered by a test so a managed pin can't be regressed.
"""
from __future__ import annotations
import time
import types
import pytest
import hermes_state
@pytest.fixture
def home(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
hermes_state.SessionDB(db_path=tmp_path / "state.db")
yield tmp_path
def _status(capsys):
from hermes_cli.main import cmd_telemetry
cmd_telemetry(types.SimpleNamespace(telemetry_action="status"))
return capsys.readouterr().out
def test_export_block_default_shows_otlp_disabled(home, capsys):
out = _status(capsys)
assert "Export" in out
assert "OTLP:" in out and "disabled" in out
assert "Content export: off" in out
assert "Secret redaction: on (always)" in out
def test_export_block_shows_endpoint_host_never_token(home, capsys, monkeypatch):
from hermes_cli.config import load_config, save_config
monkeypatch.setenv("CORP_OTLP_TOKEN", "supersecret-do-not-print")
c = load_config()
t = c.setdefault("telemetry", {})
t.setdefault("export", {})["otlp"] = {
"enabled": True,
"endpoint": "https://collector.corp:4318/v1/traces",
"headers_env": {"Authorization": "CORP_OTLP_TOKEN"},
}
save_config(c)
out = _status(capsys)
# endpoint host present
assert "https://collector.corp:4318/v1/traces" in out
# env var name + set-state present; the VALUE never printed
assert "CORP_OTLP_TOKEN" in out
assert "(set)" in out
assert "supersecret-do-not-print" not in out
def test_export_block_reflects_trajectories_gate(home, capsys):
from hermes_cli.config import load_config, save_config
c = load_config()
c.setdefault("telemetry", {})["trajectories"] = {"enabled": True}
save_config(c)
out = _status(capsys)
assert "Content export: on (trajectories enabled)" in out
def test_token_env_not_set_shows_not_set(home, capsys):
from hermes_cli.config import load_config, save_config
c = load_config()
t = c.setdefault("telemetry", {})
t.setdefault("export", {})["otlp"] = {
"enabled": True,
"endpoint": "https://x:4318/v1/traces",
"headers_env": {"Authorization": "TOTALLY_UNSET_ENV_VAR_XYZ"},
}
save_config(c)
out = _status(capsys)
assert "(NOT set)" in out
def test_allow_aggregate_pin_blocks_opt_in(home):
"""A managed allow_aggregate:false pin overrides a consent_state opt-in.
Consent is set in config (as a user or managed-scope pin would); the hard gate
still wins, so may_upload stays false.
"""
from hermes_cli.config import load_config, save_config
from agent.telemetry import policy
c = load_config()
tel = c.setdefault("telemetry", {})
tel["consent_state"] = "aggregate"
tel["allow_aggregate"] = False
save_config(c)
assert policy.may_upload_aggregate(load_config()) is False

82
tests/telemetry/test_insights_integration.py
View File

@ -1,82 +0,0 @@
"""Insights ↔ telemetry integration: the observability section in /insights output."""
from __future__ import annotations
import time
import pytest
from hermes_state import SessionDB
from agent.insights import InsightsEngine
from agent.telemetry.emitter import TelemetryEmitter
from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent
@pytest.fixture()
def db(tmp_path):
session_db = SessionDB(db_path=tmp_path / "ins_tel.db")
yield session_db
session_db.close()
def _seed_telemetry(db_path):
em = TelemetryEmitter(events_path=db_path.parent / "tel" / "events.jsonl", db_path=db_path)
now = time.time_ns()
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway",
platform="telegram", end_reason="completed",
start_ns=now - 90_000_000, end_ns=now))
em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli",
end_reason="failed", start_ns=now - 11_000_000, end_ns=now))
em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic",
model="claude-opus-4", input_tokens=5000, output_tokens=800,
cache_read_tokens=1000, latency_ms=2200))
em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search", result_class="ok"))
em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate", result_class="error"))
em.flush()
em.close()
def test_report_includes_telemetry_when_present(db):
# A session so generate() isn't the empty branch
db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4")
_seed_telemetry(db.db_path)
engine = InsightsEngine(db)
report = engine.generate(days=30)
tel = report.get("telemetry")
assert tel, "telemetry section missing"
assert tel["workflows"]["total_runs"] == 2
assert tel["workflows"]["success_rate"] == 0.5
assert tel["tool_calls"]["total"] == 2
assert tel["tool_calls"]["failure_rate"] == 0.5
assert tel["model_calls"]["by_provider"]["anthropic"] == 1
def test_terminal_output_renders_observability_section(db):
db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4")
_seed_telemetry(db.db_path)
engine = InsightsEngine(db)
out = engine.format_terminal(engine.generate(days=30))
assert "Observability" in out
assert "Workflows:" in out
assert "Failure rate:" in out
assert "Providers:" in out
def test_telemetry_section_absent_when_no_tel_rows(db):
# Session present, but no telemetry events seeded.
db.create_session(session_id="s1", source="cli", model="anthropic/claude-sonnet-4")
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report.get("telemetry") == {}
out = engine.format_terminal(report)
assert "Observability" not in out
def test_empty_report_has_telemetry_key(db):
# No sessions at all -> empty branch still carries the key (renderer-safe).
engine = InsightsEngine(db)
report = engine.generate(days=30)
assert report.get("empty") is True
assert report.get("telemetry") == {}

171
tests/telemetry/test_otlp_exporter.py
View File

@ -1,171 +0,0 @@
"""OTLP exporter tests.
Skip cleanly when the optional OTel SDK isn't installed. When it is, verify:
* event -> OTel span attribute mapping
* headers_env resolves the value from the named environment variable, not config
* a failing or slow subscriber never breaks the emitter hot path
* is_enabled / is_available gating
"""
from __future__ import annotations
import sqlite3
import time
import pytest
import hermes_state
from agent.telemetry import otlp_exporter as OE
from agent.telemetry.emitter import TelemetryEmitter
from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent
otel = pytest.importorskip("opentelemetry.sdk.trace", reason="otlp extra not installed")
def _in_memory_provider():
"""A TracerProvider with an in-memory span exporter (no network)."""
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
provider = TracerProvider()
mem = InMemorySpanExporter()
provider.add_span_processor(SimpleSpanProcessor(mem))
return provider, mem
def test_event_maps_to_span_with_real_attrs():
provider, mem = _in_memory_provider()
batch = [
{"event": "run", "entrypoint": "gateway", "platform": "telegram",
"end_reason": "completed", "model_call_count": 2, "tool_call_count": 3},
{"event": "model_call", "provider": "anthropic", "model": "claude-opus-4",
"input_tokens": 5000, "output_tokens": 800},
{"event": "tool_call", "tool_name": "web_search", "result_class": "ok"},
]
n = OE.export_batch(provider, batch)
assert n == 3
spans = mem.get_finished_spans()
names = {s.name for s in spans}
assert names == {"hermes.run", "hermes.model_call", "hermes.tool_call"}
# real values present as span attributes
run = [s for s in spans if s.name == "hermes.run"][0]
assert run.attributes["hermes.entrypoint"] == "gateway"
assert run.attributes["hermes.platform"] == "telegram"
model = [s for s in spans if s.name == "hermes.model_call"][0]
assert model.attributes["hermes.model"] == "claude-opus-4"
assert model.attributes["hermes.provider"] == "anthropic"
tool = [s for s in spans if s.name == "hermes.tool_call"][0]
assert tool.attributes["hermes.tool_name"] == "web_search"
def test_headers_resolve_from_env_not_value(monkeypatch):
monkeypatch.setenv("MY_OTLP_TOKEN", "supersecretvalue")
resolved = OE._resolve_headers({"Authorization": "MY_OTLP_TOKEN"})
assert resolved == {"Authorization": "supersecretvalue"}
# missing env var -> skipped, not crashed
assert OE._resolve_headers({"X": "NOPE_NOT_SET"}) == {}
def test_is_enabled_requires_endpoint_and_flag():
assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}) is True
assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": True}}}}) is False
assert OE.is_enabled({"telemetry": {"export": {"otlp": {"enabled": False, "endpoint": "http://x"}}}}) is False
assert OE.is_enabled({}) is False
def test_require_sdk_routes_through_lazy_install(monkeypatch):
# _require_sdk(auto_install=True) should call lazy_deps.ensure('export.otlp').
import tools.lazy_deps as ld
calls = []
monkeypatch.setattr(ld, "ensure", lambda feature, **kw: calls.append((feature, kw)))
OE._require_sdk(auto_install=True, prompt=False)
assert calls == [("export.otlp", {"prompt": False})]
def test_is_available_does_not_install(monkeypatch):
# A pure availability check must NEVER trigger an install.
import tools.lazy_deps as ld
calls = []
monkeypatch.setattr(ld, "ensure", lambda *a, **k: calls.append(a))
OE.is_available()
assert calls == []
def test_export_otlp_feature_specs_match_pyproject():
# The LAZY_DEPS entry must track the [otlp] extra in pyproject.toml.
from tools.lazy_deps import feature_specs
import pathlib, re
specs = set(feature_specs("export.otlp"))
pyproject = pathlib.Path(__file__).resolve().parents[2] / "pyproject.toml"
text = pyproject.read_text(encoding="utf-8")
m = re.search(r"^otlp\s*=\s*\[([^\]]*)\]", text, re.MULTILINE)
assert m, "otlp extra not found in pyproject.toml"
extra = set(re.findall(r'"([^"]+)"', m.group(1)))
assert specs == extra, f"LAZY_DEPS {specs} != pyproject extra {extra}"
def test_streamer_subscription_receives_events(tmp_path, monkeypatch):
# Wire an OTLPStreamer-like subscriber via the in-memory provider.
provider, mem = _in_memory_provider()
db = tmp_path / "state.db"
conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close()
em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db)
def subscriber(batch):
OE.export_batch(provider, batch)
em.subscribe(subscriber)
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed"))
em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic", model="claude-opus-4"))
em.flush()
em.close()
spans = mem.get_finished_spans()
assert {s.name for s in spans} == {"hermes.run", "hermes.model_call"}
def test_failing_subscriber_never_breaks_hot_path(tmp_path):
db = tmp_path / "state.db"
conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close()
em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db)
def bad_subscriber(batch):
time.sleep(0.2)
raise RuntimeError("OTLP collector down")
em.subscribe(bad_subscriber)
start = time.monotonic()
for i in range(30):
em.emit(ModelCallEvent(span_id=f"s{i}", run_id="r1", input_tokens=1))
elapsed = time.monotonic() - start
# emit() returns immediately regardless of the broken subscriber
assert elapsed < 1.0
em.flush()
# durable writes still happened despite the subscriber raising
conn = sqlite3.connect(db)
assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 30
conn.close()
em.close()
def test_export_once_reads_db_and_returns_count(tmp_path, monkeypatch):
db = tmp_path / "state.db"
conn = sqlite3.connect(db); conn.executescript(hermes_state.SCHEMA_SQL); conn.close()
em = TelemetryEmitter(events_path=tmp_path / "t" / "e.jsonl", db_path=db)
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="cli", end_reason="completed",
start_ns=time.time_ns(), end_ns=time.time_ns()))
em.emit(ToolCallEvent(span_id="w1", run_id="r1", tool_name="web_search", result_class="ok"))
em.flush(); em.close()
# Patch the provider builder to an in-memory one (no network). The processor
# stand-in only needs force_flush(); provider.shutdown() works on the real one.
provider, mem = _in_memory_provider()
class _Proc:
def force_flush(self, *a, **k):
return True
monkeypatch.setattr(OE, "_make_provider", lambda config: (provider, _Proc()))
n = OE.export_once({"telemetry": {"export": {"otlp": {"enabled": True, "endpoint": "http://x"}}}}, db_path=db)
assert n == 2
assert len(mem.get_finished_spans()) == 2

53
tests/telemetry/test_plugin_autoload.py
View File

@ -1,53 +0,0 @@
"""Telemetry plugin auto-load gating: on by default, off when telemetry.local=false."""
from __future__ import annotations
import hermes_cli.plugins as plugins_mod
def test_local_enabled_defaults_true(monkeypatch):
monkeypatch.setattr(plugins_mod, "load_config", lambda: {}, raising=False)
# With no telemetry section, the default is on.
monkeypatch.setattr(
"hermes_cli.config.load_config", lambda: {}, raising=False
)
assert plugins_mod._telemetry_local_enabled() is True
def test_local_disabled_when_config_false(monkeypatch):
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"telemetry": {"local": False}},
raising=False,
)
assert plugins_mod._telemetry_local_enabled() is False
def test_local_enabled_when_config_true(monkeypatch):
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"telemetry": {"local": True}},
raising=False,
)
assert plugins_mod._telemetry_local_enabled() is True
def test_malformed_config_defaults_on(monkeypatch):
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"telemetry": "not a dict"},
raising=False,
)
assert plugins_mod._telemetry_local_enabled() is True
def test_plugin_manifest_is_discoverable():
"""The bundled telemetry plugin.yaml exists and declares the lifecycle hooks."""
from pathlib import Path
import hermes_cli.plugins as p
bundled = p.get_bundled_plugins_dir()
manifest = bundled / "telemetry" / "plugin.yaml"
assert manifest.exists(), f"missing {manifest}"
text = manifest.read_text(encoding="utf-8")
for hook in ("post_api_request", "post_tool_call", "on_session_finalize"):
assert hook in text

127
tests/telemetry/test_plugin_e2e.py
View File

@ -1,127 +0,0 @@
"""End-to-end telemetry wiring test.
Unlike test_plugin_hooks.py (which calls the plugin's ``_on_*`` callbacks directly),
this drives the REAL dispatch chain that core uses at runtime:
discover_plugins() -> plugin registers hooks -> invoke_hook(name, **kwargs)
-> registered callback -> emitter -> tel_* tables
If the bundled plugin stops auto-loading, stops registering a hook, or the hook name
drifts from what core fires, the hand-written hook tests still pass but real runs go
dark. This test is the guard against that — it only touches public entry points
(``discover_plugins`` / ``invoke_hook``), exactly as core does.
"""
from __future__ import annotations
import sqlite3
import time
import pytest
import hermes_state
@pytest.fixture
def runtime(tmp_path, monkeypatch):
"""A clean HERMES_HOME with state.db, a fresh plugin manager, and a fresh emitter."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
db = tmp_path / "state.db"
hermes_state.SessionDB(db_path=db)
# Reset the global plugin-manager singleton so discovery re-runs in this HERMES_HOME.
import hermes_cli.plugins as plugins_mod
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
# Reset the emitter singleton so it binds to this state.db (and tear it down after).
from agent.telemetry import emitter as emitter_mod
emitter_mod.reset_emitter_for_tests(None)
# Clear the plugin's per-run accumulators between tests.
import plugins.telemetry as plug
plug._runs.clear()
yield db, plugins_mod, emitter_mod, plug
try:
emitter_mod.get_emitter().flush()
except Exception:
pass
emitter_mod.reset_emitter_for_tests(None)
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
def _fire_one_turn(invoke_hook):
"""Fire the hook sequence of a single completed turn, as core does."""
invoke_hook("on_session_start", session_id="s1",
model="anthropic/claude-opus-4", platform="cli")
invoke_hook("post_api_request", session_id="s1", platform="cli",
provider="anthropic", base_url=None, model="claude-opus-4",
api_duration=0.9,
usage={"input_tokens": 1000, "output_tokens": 120,
"cache_read_tokens": 0, "cache_write_tokens": 0,
"reasoning_tokens": 0})
invoke_hook("post_tool_call", session_id="s1", platform="cli",
function_name="web_search", duration_ms=210, result='{"data": "ok"}')
invoke_hook("on_session_finalize", session_id="s1", platform="cli",
reason="shutdown")
def test_real_dispatch_writes_tel_rows(runtime):
"""The bundled plugin, loaded via discover_plugins, captures a turn end to end."""
db, plugins_mod, emitter_mod, _plug = runtime
plugins_mod.discover_plugins(force=True)
# The plugin must have registered the lifecycle hooks core fires.
mgr = plugins_mod.get_plugin_manager()
registered = {k for k, v in getattr(mgr, "_hooks", {}).items() if v}
for hook in ("on_session_start", "post_api_request", "post_tool_call",
"on_session_finalize"):
assert hook in registered, f"core hook {hook!r} not registered by the plugin"
_fire_one_turn(plugins_mod.invoke_hook)
time.sleep(0.5) # let the background writer drain
emitter_mod.get_emitter().flush()
conn = sqlite3.connect(db)
conn.row_factory = sqlite3.Row
assert conn.execute("SELECT COUNT(*) c FROM tel_runs").fetchone()["c"] == 1
assert conn.execute("SELECT COUNT(*) c FROM tel_model_calls").fetchone()["c"] == 1
assert conn.execute("SELECT COUNT(*) c FROM tel_tool_calls").fetchone()["c"] == 1
# Real values, not buckets.
mc = conn.execute("SELECT provider, model FROM tel_model_calls").fetchone()
assert mc["provider"] == "anthropic"
assert mc["model"] == "claude-opus-4"
tc = conn.execute("SELECT tool_name FROM tel_tool_calls").fetchone()
assert tc["tool_name"] == "web_search"
run = conn.execute("SELECT end_reason, model_call_count, tool_call_count "
"FROM tel_runs").fetchone()
assert run["end_reason"] == "completed"
assert run["model_call_count"] == 1
assert run["tool_call_count"] == 1
conn.close()
def test_local_disabled_writes_nothing(runtime, monkeypatch):
"""telemetry.local=false: the plugin does not auto-load, so no rows are written."""
db, plugins_mod, emitter_mod, _plug = runtime
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"telemetry": {"local": False}},
raising=False,
)
plugins_mod.discover_plugins(force=True)
_fire_one_turn(plugins_mod.invoke_hook)
time.sleep(0.3)
try:
emitter_mod.get_emitter().flush()
except Exception:
pass
conn = sqlite3.connect(db)
assert conn.execute("SELECT COUNT(*) FROM tel_runs").fetchone()[0] == 0
assert conn.execute("SELECT COUNT(*) FROM tel_model_calls").fetchone()[0] == 0
conn.close()

134
tests/telemetry/test_plugin_hooks.py
View File

@ -1,134 +0,0 @@
"""Telemetry plugin hook tests — feed realistic kwargs, assert tel_* rows + no leak."""
from __future__ import annotations
import sqlite3
import pytest
import hermes_state
from agent.telemetry import emitter as emitter_mod
from agent.telemetry.emitter import TelemetryEmitter
@pytest.fixture
def wired(tmp_path, monkeypatch):
db = tmp_path / "state.db"
conn = sqlite3.connect(db)
conn.executescript(hermes_state.SCHEMA_SQL)
conn.close()
em = TelemetryEmitter(events_path=tmp_path / "telemetry" / "events.jsonl", db_path=db)
emitter_mod.reset_emitter_for_tests(em)
# reset the plugin's per-run accumulators between tests
import plugins.telemetry as plug
plug._runs.clear()
yield db, em, plug
em.flush()
em.close()
emitter_mod.reset_emitter_for_tests(None)
def test_full_session_lifecycle_produces_rows(wired):
db, em, plug = wired
plug._on_session_start(session_id="sess1", platform="telegram")
plug._on_post_api_request(
session_id="sess1", platform="telegram",
provider="anthropic", base_url=None, model="claude-opus-4",
api_duration=2.5,
usage={"input_tokens": 5000, "output_tokens": 800, "cache_read_tokens": 1000,
"cache_write_tokens": 0, "reasoning_tokens": 0},
)
plug._on_post_tool_call(
session_id="sess1", platform="telegram",
function_name="web_search", duration_ms=812, result="{\"data\": \"...\"}",
)
# Production finalize callers pass `reason` (e.g. "shutdown"), not cost.
plug._on_session_finalize(
session_id="sess1", platform="telegram", reason="shutdown",
)
em.flush()
conn = sqlite3.connect(db)
conn.row_factory = sqlite3.Row
run = conn.execute("SELECT * FROM tel_runs").fetchone()
assert run is not None
assert run["entrypoint"] == "gateway"
assert run["platform"] == "telegram"
assert run["end_reason"] == "completed"
assert run["model_call_count"] == 1
assert run["tool_call_count"] == 1
mc = conn.execute("SELECT * FROM tel_model_calls").fetchone()
assert mc["provider"] == "anthropic"
assert mc["model"] == "claude-opus-4"
assert mc["input_tokens"] == 5000
tc = conn.execute("SELECT * FROM tel_tool_calls").fetchone()
assert tc["tool_name"] == "web_search"
assert tc["result_class"] == "ok"
conn.close()
def test_tool_error_result_classified_and_counted(wired):
db, em, plug = wired
plug._on_session_start(session_id="s2", platform="cli")
plug._on_post_tool_call(
session_id="s2", function_name="terminal", duration_ms=10,
result="{\"error\": \"command failed\"}",
)
plug._on_session_finalize(session_id="s2", reason="shutdown")
em.flush()
conn = sqlite3.connect(db)
conn.row_factory = sqlite3.Row
tc = conn.execute("SELECT result_class, tool_name FROM tel_tool_calls").fetchone()
assert tc["result_class"] == "error"
assert tc["tool_name"] == "terminal"
run = conn.execute("SELECT error_count FROM tel_runs").fetchone()
assert run["error_count"] == 1
conn.close()
def test_api_error_recorded(wired):
db, em, plug = wired
plug._on_session_start(session_id="s3", platform="cli")
plug._on_api_request_error(session_id="s3", error_type="provider timeout after 60s")
plug._on_session_finalize(session_id="s3", failed=True)
em.flush()
conn = sqlite3.connect(db)
conn.row_factory = sqlite3.Row
err = conn.execute("SELECT error_class, subsystem FROM tel_error_events").fetchone()
assert err["error_class"] == "provider_timeout"
assert err["subsystem"] == "model_api"
run = conn.execute("SELECT end_reason FROM tel_runs").fetchone()
assert run["end_reason"] == "failed"
conn.close()
def test_hooks_never_raise_on_garbage_kwargs(wired):
_, em, plug = wired
# Missing everything — must be swallowed by the _safe wrapper.
plug._on_post_api_request()
plug._on_post_tool_call()
plug._on_session_finalize()
plug._on_api_request_error()
em.flush()
def test_no_message_content_in_tool_rows(wired):
"""The tool hook receives a result blob; only the classification persists, not content."""
db, em, plug = wired
plug._on_session_start(session_id="s4", platform="cli")
secret = "{\"data\": \"USER SECRET sk-ABCDEF and /Users/alice/file.txt\"}"
plug._on_post_tool_call(session_id="s4", function_name="web_search",
duration_ms=5, result=secret)
plug._on_session_finalize(session_id="s4")
em.flush()
# The whole tel_tool_calls row must contain none of the result content.
conn = sqlite3.connect(db)
row = conn.execute("SELECT * FROM tel_tool_calls").fetchone()
conn.close()
blob = " ".join(str(x) for x in row)
assert "sk-ABCDEF" not in blob
assert "/Users/alice" not in blob
assert "SECRET" not in blob

53
tests/telemetry/test_policy_consent.py
View File

@ -1,53 +0,0 @@
"""Consent gate tests.
Consent is a single config field (``telemetry.consent_state``); the aggregate opt-in
is expressed by setting it to ``"aggregate"`` (via ``hermes config set`` or a
managed-scope pin). ``allow_aggregate`` is the hard gate. ``policy.may_upload_aggregate``
is the gate a future uploader must consult.
"""
from __future__ import annotations
from agent.telemetry import policy
def _cfg(**telemetry):
return {"telemetry": telemetry}
def test_default_posture_never_uploads():
# No consent recorded → unknown → never uploads.
assert policy.may_upload_aggregate(_cfg(local=True, consent_state="unknown")) is False
def test_missing_telemetry_block_never_uploads():
assert policy.may_upload_aggregate({}) is False
def test_opted_in_uploads():
assert policy.may_upload_aggregate(_cfg(consent_state="aggregate")) is True
def test_declined_does_not_upload():
assert policy.may_upload_aggregate(_cfg(consent_state="local")) is False
def test_allow_aggregate_false_overrides_opt_in():
# An admin pins telemetry.allow_aggregate: false via managed scope.
cfg = _cfg(consent_state="aggregate", allow_aggregate=False)
assert policy.may_upload_aggregate(cfg) is False # the hard gate wins
def test_local_off_makes_aggregate_inert():
# Aggregate metrics derive from the local tables; with local off there is
# nothing to aggregate, so opting in cannot upload.
cfg = _cfg(local=False, consent_state="aggregate", allow_aggregate=True)
assert policy.may_upload_aggregate(cfg) is False
def test_install_id_minted_when_empty_and_stable_when_set():
cfg = _cfg(install_id="")
minted = policy.ensure_install_id(cfg)
assert minted and len(minted) >= 32 # uuid4
cfg2 = _cfg(install_id="fixed-id")
assert policy.ensure_install_id(cfg2) == "fixed-id"

88
tests/telemetry/test_rollup.py
View File

@ -1,88 +0,0 @@
"""rollup tests: tel_* -> per-run summary events with REAL values (local only)."""
from __future__ import annotations
import sqlite3
import time
import hermes_state
from agent.telemetry import rollup
from agent.telemetry.emitter import TelemetryEmitter
from agent.telemetry.events import ModelCallEvent, RunEvent, ToolCallEvent
def _seed(tmp_path):
db = tmp_path / "state.db"
conn = sqlite3.connect(db)
conn.executescript(hermes_state.SCHEMA_SQL)
conn.close()
em = TelemetryEmitter(events_path=tmp_path / "tel" / "e.jsonl", db_path=db)
now = time.time_ns()
em.emit(RunEvent(run_id="r1", trace_id="t1", entrypoint="gateway",
platform="telegram", end_reason="completed",
start_ns=now - 90_000_000, end_ns=now,
model_call_count=2, tool_call_count=2))
em.emit(ModelCallEvent(span_id="m1", run_id="r1", provider="anthropic",
model="claude-opus-4", input_tokens=60000, output_tokens=8000))
em.emit(ModelCallEvent(span_id="m2", run_id="r1", provider="anthropic",
model="claude-opus-4", input_tokens=5000, output_tokens=500))
em.emit(ToolCallEvent(span_id="tc1", run_id="r1", tool_name="web_search",
result_class="ok"))
em.emit(ToolCallEvent(span_id="tc2", run_id="r1", tool_name="browser_navigate",
result_class="ok"))
# an in-progress run (no end_ns) must be excluded
em.emit(RunEvent(run_id="r2", trace_id="t2", entrypoint="cli", start_ns=now))
em.flush()
em.close()
return db
def test_builds_one_event_per_completed_run_with_real_values(tmp_path):
db = _seed(tmp_path)
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db,
include_heartbeat=False)
wf = [e for e in events if e["event_name"] == "workflow_completed"]
assert len(wf) == 1 # r2 (no end_ns) excluded
e = wf[0]
assert e["entrypoint"] == "gateway"
assert e["platform"] == "telegram"
# REAL model id + provider, not a bucket/class
models = {m["model"] for m in e["models_used"]}
assert models == {"claude-opus-4"}
assert e["models_used"][0]["provider"] == "anthropic"
assert sorted(e["tools_used"]) == ["browser_navigate", "web_search"]
# real token totals, not buckets
assert e["input_tokens"] == 65000
assert e["output_tokens"] == 8500
def test_real_model_and_tool_names_present(tmp_path):
db = _seed(tmp_path)
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db)
blob = " ".join(str(v) for e in events for v in e.values())
assert "claude-opus-4" in blob
assert "web_search" in blob
def test_heartbeat_included_by_default(tmp_path):
db = _seed(tmp_path)
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db)
assert any(e["event_name"] == "heartbeat" for e in events)
def test_summarize_counts_by_event_name(tmp_path):
db = _seed(tmp_path)
events = rollup.build_aggregate_events(install_id="fixed-id", db_path=db)
s = rollup.summarize(events)
assert s["total"] == len(events)
assert s["by_event_name"]["workflow_completed"] == 1
assert s["by_event_name"]["heartbeat"] == 1
def test_empty_db_yields_only_heartbeat(tmp_path):
db = tmp_path / "state.db"
conn = sqlite3.connect(db)
conn.executescript(hermes_state.SCHEMA_SQL)
conn.close()
events = rollup.build_aggregate_events(install_id="x", db_path=db)
assert [e["event_name"] for e in events] == ["heartbeat"]

45
tests/telemetry/test_schema_migration.py
View File

@ -1,45 +0,0 @@
"""Schema tests: tel_* tables exist after init; SCHEMA_VERSION bumped."""
from __future__ import annotations
import sqlite3
import hermes_state
TEL_TABLES = {
"tel_runs", "tel_spans", "tel_model_calls", "tel_tool_calls",
"tel_error_events",
}
def test_schema_version_is_17_or_higher():
assert hermes_state.SCHEMA_VERSION >= 17
def test_tel_tables_present_in_schema_sql():
for tbl in TEL_TABLES:
assert f"CREATE TABLE IF NOT EXISTS {tbl}" in hermes_state.SCHEMA_SQL
def test_tel_tables_created_on_executescript(tmp_path):
db = tmp_path / "state.db"
conn = sqlite3.connect(db)
conn.executescript(hermes_state.SCHEMA_SQL)
rows = {
r[0]
for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
}
conn.close()
assert TEL_TABLES.issubset(rows), f"missing: {TEL_TABLES - rows}"
def test_executescript_is_idempotent(tmp_path):
# IF NOT EXISTS means re-running on an existing DB is a no-op, not an error.
db = tmp_path / "state.db"
conn = sqlite3.connect(db)
conn.executescript(hermes_state.SCHEMA_SQL)
conn.executescript(hermes_state.SCHEMA_SQL) # second run must not raise
conn.close()

109
tests/telemetry/test_spans_trace.py
View File

@ -1,109 +0,0 @@
"""Trace/span layer: tel_spans is populated as a connected run -> calls tree.
Drives the real dispatch chain (discover_plugins -> invoke_hook) and asserts the
timing/lineage backbone in tel_spans:
- one root span per run (kind="run", parent_span_id NULL),
- one child span per model/tool call parented to the root,
- a single trace_id across the run,
- call detail rows (tel_model_calls / tel_tool_calls) JOIN to their span by span_id,
- reconstructed durations match the reported latency/duration.
This is the regression guard for the waterfall a desktop trace viewer renders.
"""
from __future__ import annotations
import sqlite3
import time
import pytest
import hermes_state
@pytest.fixture
def runtime(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
db = tmp_path / "state.db"
hermes_state.SessionDB(db_path=db)
import hermes_cli.plugins as plugins_mod
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
from agent.telemetry import emitter as emitter_mod
emitter_mod.reset_emitter_for_tests(None)
import plugins.telemetry as plug
plug._runs.clear()
yield db, plugins_mod, emitter_mod
try:
emitter_mod.get_emitter().flush()
except Exception:
pass
emitter_mod.reset_emitter_for_tests(None)
monkeypatch.setattr(plugins_mod, "_plugin_manager", None, raising=False)
def _one_turn(invoke_hook):
invoke_hook("on_session_start", session_id="s1",
model="anthropic/claude-opus-4", platform="cli")
invoke_hook("post_api_request", session_id="s1", platform="cli",
provider="anthropic", model="claude-opus-4", api_duration=0.9,
usage={"input_tokens": 1000, "output_tokens": 120})
invoke_hook("post_tool_call", session_id="s1", platform="cli",
function_name="web_search", duration_ms=210, result='{"data": "ok"}')
invoke_hook("on_session_finalize", session_id="s1", platform="cli",
reason="shutdown")
def test_tel_spans_forms_connected_trace(runtime):
db, plugins_mod, emitter_mod = runtime
plugins_mod.discover_plugins(force=True)
_one_turn(plugins_mod.invoke_hook)
time.sleep(0.5)
emitter_mod.get_emitter().flush()
conn = sqlite3.connect(db)
conn.row_factory = sqlite3.Row
spans = conn.execute(
"SELECT span_id, parent_span_id, kind, name, start_ns, end_ns, status, trace_id "
"FROM tel_spans"
).fetchall()
# root + model + tool
assert len(spans) == 3
roots = [s for s in spans if s["parent_span_id"] is None]
children = [s for s in spans if s["parent_span_id"] is not None]
assert len(roots) == 1
assert roots[0]["kind"] == "run"
assert len(children) == 2
# single trace, all children parented to the root
assert len({s["trace_id"] for s in spans}) == 1
assert all(c["parent_span_id"] == roots[0]["span_id"] for c in children)
# spans are time-ordered and carry real durations
by_kind = {s["kind"]: s for s in spans}
assert (by_kind["model"]["end_ns"] - by_kind["model"]["start_ns"]) == 900 * 1_000_000
assert (by_kind["tool"]["end_ns"] - by_kind["tool"]["start_ns"]) == 210 * 1_000_000
assert by_kind["run"]["end_ns"] >= by_kind["run"]["start_ns"]
def test_detail_rows_join_to_spans(runtime):
db, plugins_mod, emitter_mod = runtime
plugins_mod.discover_plugins(force=True)
_one_turn(plugins_mod.invoke_hook)
time.sleep(0.5)
emitter_mod.get_emitter().flush()
conn = sqlite3.connect(db)
conn.row_factory = sqlite3.Row
mc = conn.execute(
"SELECT m.model, s.kind, s.trace_id FROM tel_model_calls m "
"JOIN tel_spans s ON m.span_id = s.span_id"
).fetchone()
assert mc is not None and mc["model"] == "claude-opus-4" and mc["kind"] == "model"
tc = conn.execute(
"SELECT t.tool_name, s.kind FROM tel_tool_calls t "
"JOIN tel_spans s ON t.span_id = s.span_id"
).fetchone()
assert tc is not None and tc["tool_name"] == "web_search" and tc["kind"] == "tool"
conn.close()

18
tools/lazy_deps.py
View File

@ -116,6 +116,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = {
"search.firecrawl": ("firecrawl-py==4.17.0",),
"search.parallel": ("parallel-web==0.4.2",),
# ─── Monitoring ─────────────────────────────────────────────────────────
# OTLP gateway monitoring export. Lazily installed on first use of
# monitoring.gateway_health_export / monitoring.export.otlp. Tracks the
# `otlp` extra in pyproject.toml — bump both together.
"export.otlp": (
"opentelemetry-sdk==1.39.1",
"opentelemetry-exporter-otlp-proto-http==1.39.1",
),
# ─── TTS providers ─────────────────────────────────────────────────────
# Pinned to exact versions to match pyproject.toml's no-ranges policy
# (see comment at top of [project.dependencies]). When bumping, update
@ -140,15 +149,6 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = {
# ─── Image generation backends ─────────────────────────────────────────
"image.fal": ("fal-client==0.13.1",),
# ─── Observability ─────────────────────────────────────────────────────
# OTLP telemetry export. Lazily installed on first use of
# `hermes telemetry export --otlp`. Tracks the `otlp` extra in
# pyproject.toml — bump both together.
"export.otlp": (
"opentelemetry-sdk==1.30.0",
"opentelemetry-exporter-otlp-proto-http==1.30.0",
),
# ─── Memory providers ──────────────────────────────────────────────────
"memory.honcho": ("honcho-ai==2.2.0",),
"memory.hindsight": ("hindsight-client==0.6.1",),

14
uv.lock
View File

@ -1415,7 +1415,9 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/51/58/5404031044f55afad7aad1aff8be3f22b1bed03e237cfeabbc7e5c8cfde0/greenlet-3.5.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:aca9b4ce85b152b5524ef7d88170efdff80dc0032aa8b75f9aaf7f3479ea95b4", size = 287424, upload-time = "2026-06-26T18:20:31.469Z" },
{ url = "https://files.pythonhosted.org/packages/b4/bf/1c65e9b94a54d547068fa5b5a8a06f221f3316b48908e08668d29c77cb50/greenlet-3.5.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f71be4920368fe1fabeeaa53d1e3548337e2b223d9565f8ad5e392a75ba23fc", size = 606523, upload-time = "2026-06-26T19:07:08.859Z" },
{ url = "https://files.pythonhosted.org/packages/b8/c7/b66baacc95775ad511287acb0137b95574a9ce5491902372b7564799d790/greenlet-3.5.3-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d77e67f65f98449e3fb83f795b5d0a8437aead2f874ca89c96576caf4be3af6", size = 618315, upload-time = "2026-06-26T19:10:06.055Z" },
{ url = "https://files.pythonhosted.org/packages/b0/a0/68afd1ebad40db87dac0a28ffa120726b98bf9c7c40c481b0f63c105d298/greenlet-3.5.3-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e18619ba655ac05d78d80fc83cac4ba892bd6927b99e3b8237aee861aaacc8bb", size = 626155, upload-time = "2026-06-26T19:24:14.44Z" },
{ url = "https://files.pythonhosted.org/packages/78/2b/28ed29463522fdbe4c15b1f63922041626a7478316b34ab4adda3f0a4aba/greenlet-3.5.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8540f1e6205bd13ca0ce685581037219ca54a1b41a0a15d228c6c9b8ad5903d7", size = 617381, upload-time = "2026-06-26T18:32:16.077Z" },
{ url = "https://files.pythonhosted.org/packages/07/7f/e327d912239ec4b3b49999e3967389bcf1ee8722b9ee9194d2752ecd558a/greenlet-3.5.3-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:d27c0c653a60d9535f690226474a5cc1036a8b0d7b57504d1c4f89c44a07a80c", size = 421083, upload-time = "2026-06-26T19:25:35.804Z" },
{ url = "https://files.pythonhosted.org/packages/2a/7b/ad04e9d1337fc04965dc9fc616b6a72cb65a24b800a014c011ec812f5489/greenlet-3.5.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ef56fe650f50575bf843acde967b9c567687f3c22340941a899b7bc56e956a8", size = 1577771, upload-time = "2026-06-26T19:09:01.537Z" },
{ url = "https://files.pythonhosted.org/packages/d8/33/6c87ab7ba663f70ca21f3022aad1ffe56d3f3e0521e836c2415e13abcc3c/greenlet-3.5.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5121af01cf911e70056c00d4b46d5e9b5d1415550038573d744138bacb59e6b8", size = 1644048, upload-time = "2026-06-26T18:31:42.996Z" },
{ url = "https://files.pythonhosted.org/packages/1c/35/f0d8ee998b422cf8693b270f098e55d8d4ec8006b061b333f54f177d28d9/greenlet-3.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:0f41e4a05a3c0cb31b17023eff28dd111e1d16bf7d7d00406cd7df23f31398a7", size = 239137, upload-time = "2026-06-26T18:23:21.664Z" },
@ -1423,7 +1425,9 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/5d/6e/4c37d51a2b7f82d2ff11bb6b5f7d766d9a011726624af255e843727627a3/greenlet-3.5.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:719757059f5a53fd0dde23f78cffeafcdd97b21c850ddb7ca684a3c1a1f122e2", size = 288685, upload-time = "2026-06-26T18:22:08.977Z" },
{ url = "https://files.pythonhosted.org/packages/7a/73/815dd90131c1b71ebdf53dbc7c276cafec2a1173b97559f97aba72724a87/greenlet-3.5.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa9f765dd09f9d0cdac651ffdf631ee59ec5dc6ee7a73e0c012ba9c52fbdf5b", size = 604761, upload-time = "2026-06-26T19:07:10.114Z" },
{ url = "https://files.pythonhosted.org/packages/9f/57/079cfe76bcef36b153b25607ee91c6fcb58f17f8b23c86bbbeabe0c88d72/greenlet-3.5.3-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7faba15ac005376e02a0384504e0243be3370ce010296a44a820feb342b505ab", size = 617044, upload-time = "2026-06-26T19:10:07.25Z" },
{ url = "https://files.pythonhosted.org/packages/fb/fb/d97dc261209c80744b7c8132693a30d70ec6e7315e632cb0a10b3fec94dd/greenlet-3.5.3-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5795cd1101371140551c645f2d408b8d3c01a5a29cf8a9bce6e759c983682d23", size = 622351, upload-time = "2026-06-26T19:24:16.32Z" },
{ url = "https://files.pythonhosted.org/packages/37/87/b4d095775a3fb1bcafbb483fc206b27ebb785724c83051447737085dc54e/greenlet-3.5.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87142215824be6ac05e2e8e2786eec307ccbc27c36723c3881959df654af6861", size = 614244, upload-time = "2026-06-26T18:32:17.594Z" },
{ url = "https://files.pythonhosted.org/packages/8e/ac/e5fee13cbbd0e8de312d9a146584b8a51891c68847330ef9dc8b5109d23f/greenlet-3.5.3-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:af4923b3096e26a36d7e9cf24ab88083a20f97d191e3b97f253731ce9b41b28c", size = 425395, upload-time = "2026-06-26T19:25:37.144Z" },
{ url = "https://files.pythonhosted.org/packages/8a/70/7559b609683650fa2b95b8ab84b4ab0b26556a635d19675e12aa832d826d/greenlet-3.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:215275b1b49320987352e6c1b054acca0064f965a2c66992bed9a6f7d913f149", size = 1574210, upload-time = "2026-06-26T19:09:03.077Z" },
{ url = "https://files.pythonhosted.org/packages/ae/73/be55392074c60fc37655ca40fa6022457bfbf6718e9e342a7b0b41f96dd2/greenlet-3.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6b1b0eed82364b0e32c4ea0f221452d33e6bb17ae094d9f72aed9851812747ea", size = 1638627, upload-time = "2026-06-26T18:31:44.748Z" },
{ url = "https://files.pythonhosted.org/packages/14/40/c57489acf8e37d74e2913d4eff63aa0dba17acccc4bdeef874dde2dbbec9/greenlet-3.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:cde8adafa2365676f74a979744629589999093bc86e2484214f58e61df08902c", size = 239882, upload-time = "2026-06-26T18:23:27.518Z" },
@ -1431,7 +1435,9 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9b/ff/a620267401db30a50cc8450ee90730e2d4a85658c055c0e760d4ed47fb13/greenlet-3.5.3-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c8d87c2134d871df96ecdea9cec7cbaab286dadab0f56476e57aaf9e8ac11550", size = 287609, upload-time = "2026-06-26T18:21:14.724Z" },
{ url = "https://files.pythonhosted.org/packages/d6/fa/5401ac78021c826a25b6dde0c705e0a8f29b617509f9185a31dac15fbe1b/greenlet-3.5.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2d185dd1621757e70c3861cceffd5317ab4e7ed7eb09c82994828468527ade5", size = 607435, upload-time = "2026-06-26T19:07:11.412Z" },
{ url = "https://files.pythonhosted.org/packages/e9/76/1dc144a2e56e65d36405078ed774224375ea520a1870a6e46e08bb4ac7bf/greenlet-3.5.3-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1c514a468149bf8fbbab874188a3535cd8a48a3e353eb53a3d424296f8dbacd3", size = 619787, upload-time = "2026-06-26T19:10:08.396Z" },
{ url = "https://files.pythonhosted.org/packages/57/61/2f5b1adf256d039f5dab8005de8d3d7ad2b0070a3219c0e036b3fbfeb440/greenlet-3.5.3-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9ad04dd75458c6300b047c61b8639092433d205a25a14e310d6582a480efcca1", size = 625580, upload-time = "2026-06-26T19:24:18.344Z" },
{ url = "https://files.pythonhosted.org/packages/bf/87/c298cee62df1de4ad7fec32abda73526cff347fd143a6ed4ac369246668a/greenlet-3.5.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:915f887cf2682b66419b879423a2e072634aa7b7dce6f3ada4957cfced3f1e9a", size = 616786, upload-time = "2026-06-26T18:32:19.128Z" },
{ url = "https://files.pythonhosted.org/packages/3e/d9/ab7fc9e543e44d6879b0a6ef9a4b2188940fd180cc65d6f646883ddf7201/greenlet-3.5.3-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:afaabdd554cd7ae9bbb3ca070b0d7fdfd207dbf1d16865f7233837709d354bda", size = 427933, upload-time = "2026-06-26T19:25:38.219Z" },
{ url = "https://files.pythonhosted.org/packages/9e/2e/e6f009885ed0705ccf33fe0583c117cfd03cde77e31a596dd5785a30762b/greenlet-3.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:766cfd421c13e450feb340cd472a3ed9957d438727b7b4593ad7c76c5d2b0deb", size = 1574316, upload-time = "2026-06-26T19:09:04.273Z" },
{ url = "https://files.pythonhosted.org/packages/ef/fe/43fd110b01e40da0adb7c90ac7ea744bef2d43dca00de5095fd2351c2a68/greenlet-3.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2ecda9ec22edf38fa389369eaed8c3d37c05f3c54e69f69438dbb2cc1de1458b", size = 1638614, upload-time = "2026-06-26T18:31:46.297Z" },
{ url = "https://files.pythonhosted.org/packages/0f/7c/062447147a61f8b4337b156fe70d32a165fcf2f89d7ca6255e572806705c/greenlet-3.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:c82304750f057167ff60d188df1d0cc1764ce9567eadf03e6a7443bcedd0b30b", size = 239850, upload-time = "2026-06-26T18:21:54.613Z" },
@ -1666,6 +1672,10 @@ modal = [
nemo-relay = [
{ name = "nemo-relay" },
]
otlp = [
{ name = "opentelemetry-exporter-otlp-proto-http" },
{ name = "opentelemetry-sdk" },
]
parallel-web = [
{ name = "parallel-web" },
]
@ -1808,6 +1818,8 @@ requires-dist = [
{ name = "nemo-relay", marker = "extra == 'nemo-relay'", specifier = ">=0.5,<1.0" },
{ name = "numpy", marker = "extra == 'voice'", specifier = "==2.4.3" },
{ name = "openai", specifier = "==2.24.0" },
{ name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp'", specifier = "==1.39.1" },
{ name = "opentelemetry-sdk", marker = "extra == 'otlp'", specifier = "==1.39.1" },
{ name = "packaging", specifier = "==26.0" },
{ name = "parallel-web", marker = "extra == 'parallel-web'", specifier = "==0.4.2" },
{ name = "pathspec", specifier = "==1.1.1" },
@ -1855,7 +1867,7 @@ requires-dist = [
{ name = "websockets", specifier = "==15.0.1" },
{ name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" },
]
provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "honcho", "supermemory", "mem0", "vision", "pty", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"]
provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "supermemory", "mem0", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "teams", "computer-use", "acp", "mistral", "otlp", "bedrock", "vertex", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"]
[[package]]
name = "hf-xet"