telemetry: materialize dropped-event counter children at 0

A labeled Prometheus counter exports no series until its first labels()
call, so telemetry_events_dropped stayed invisible until an event was
actually dropped — impossible to alert on or graph, and "no drops" was
indistinguishable from "metric missing / scrape broken".

Pre-create the (namespace, reason) children at 0 on emitter start, for
each reason the emitter can emit, so the metric is always present.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Phil 2026-07-22 14:51:18 -04:00
parent e7cbcc8432
commit 0f47e84993
2 changed files with 35 additions and 0 deletions

View File

@ -172,6 +172,20 @@ class TelemetryEmitter:
)
self._running = True
self._flush_task = asyncio.create_task(self._periodic_flush())
# Pre-create the dropped-event counter children at 0 so the metric is
# visible in Prometheus/Grafana before any drop occurs — a labeled
# counter exports nothing until its first observation. Lets us alert on
# telemetry drops and tell "no drops" apart from "metric missing".
from src.telemetry.prometheus.metrics import prometheus_metrics
prometheus_metrics.initialize_telemetry_dropped_metrics(
reasons=[
f"{self.drop_reason_prefix}buffer_full",
f"{self.drop_reason_prefix}send_failed",
]
)
logger.info("Telemetry emitter started, endpoint: %s", self.endpoint)
async def shutdown(self) -> None:

View File

@ -325,6 +325,27 @@ class PrometheusMetrics:
except Exception as e:
self._handle_metric_error("record_telemetry_event_dropped", e)
def initialize_telemetry_dropped_metrics(self, *, reasons: list[str]) -> None:
"""Pre-create telemetry_events_dropped child series at 0.
A labeled Prometheus counter exports no time series until its first
``labels(...)`` call, so ``telemetry_events_dropped`` stays invisible in
Prometheus/Grafana until an event is actually dropped you cannot alert
on or graph a metric that does not exist yet. Materializing the
``(namespace, reason)`` children at startup keeps the metric present at 0,
so a missing series signals a broken scrape rather than "no drops".
Args:
reasons: The reason label values the calling emitter can produce.
"""
for reason in reasons:
try:
telemetry_events_dropped_counter.labels(reason=reason)
except Exception as e:
self._handle_metric_error(
"initialize_telemetry_dropped_metrics", e
)
def set_telemetry_buffer_size(self, *, size: int) -> None:
try:
telemetry_buffer_size_gauge.labels().set(size)