diff --git a/src/telemetry/emitter.py b/src/telemetry/emitter.py index 201d16aa..52b3cf91 100644 --- a/src/telemetry/emitter.py +++ b/src/telemetry/emitter.py @@ -172,6 +172,20 @@ class TelemetryEmitter: ) self._running = True self._flush_task = asyncio.create_task(self._periodic_flush()) + + # Pre-create the dropped-event counter children at 0 so the metric is + # visible in Prometheus/Grafana before any drop occurs — a labeled + # counter exports nothing until its first observation. Lets us alert on + # telemetry drops and tell "no drops" apart from "metric missing". + from src.telemetry.prometheus.metrics import prometheus_metrics + + prometheus_metrics.initialize_telemetry_dropped_metrics( + reasons=[ + f"{self.drop_reason_prefix}buffer_full", + f"{self.drop_reason_prefix}send_failed", + ] + ) + logger.info("Telemetry emitter started, endpoint: %s", self.endpoint) async def shutdown(self) -> None: diff --git a/src/telemetry/prometheus/metrics.py b/src/telemetry/prometheus/metrics.py index 01d7be4f..76a879ef 100644 --- a/src/telemetry/prometheus/metrics.py +++ b/src/telemetry/prometheus/metrics.py @@ -325,6 +325,27 @@ class PrometheusMetrics: except Exception as e: self._handle_metric_error("record_telemetry_event_dropped", e) + def initialize_telemetry_dropped_metrics(self, *, reasons: list[str]) -> None: + """Pre-create telemetry_events_dropped child series at 0. + + A labeled Prometheus counter exports no time series until its first + ``labels(...)`` call, so ``telemetry_events_dropped`` stays invisible in + Prometheus/Grafana until an event is actually dropped — you cannot alert + on or graph a metric that does not exist yet. Materializing the + ``(namespace, reason)`` children at startup keeps the metric present at 0, + so a missing series signals a broken scrape rather than "no drops". + + Args: + reasons: The reason label values the calling emitter can produce. + """ + for reason in reasons: + try: + telemetry_events_dropped_counter.labels(reason=reason) + except Exception as e: + self._handle_metric_error( + "initialize_telemetry_dropped_metrics", e + ) + def set_telemetry_buffer_size(self, *, size: int) -> None: try: telemetry_buffer_size_gauge.labels().set(size)