From 0f47e849934de5596bc89c0c22fbc24c9a2250b3 Mon Sep 17 00:00:00 2001
From: Phil <phil@plasticlabs.ai>
Date: Wed, 22 Jul 2026 14:51:18 -0400
Subject: [PATCH] telemetry: materialize dropped-event counter children at 0
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A labeled Prometheus counter exports no series until its first labels()
call, so telemetry_events_dropped stayed invisible until an event was
actually dropped — impossible to alert on or graph, and "no drops" was
indistinguishable from "metric missing / scrape broken".

Pre-create the (namespace, reason) children at 0 on emitter start, for
each reason the emitter can emit, so the metric is always present.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/telemetry/emitter.py            | 14 ++++++++++++++
 src/telemetry/prometheus/metrics.py | 21 +++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/src/telemetry/emitter.py b/src/telemetry/emitter.py
index 201d16aa..52b3cf91 100644
--- a/src/telemetry/emitter.py
+++ b/src/telemetry/emitter.py
@@ -172,6 +172,20 @@ class TelemetryEmitter:
         )
         self._running = True
         self._flush_task = asyncio.create_task(self._periodic_flush())
+
+        # Pre-create the dropped-event counter children at 0 so the metric is
+        # visible in Prometheus/Grafana before any drop occurs — a labeled
+        # counter exports nothing until its first observation. Lets us alert on
+        # telemetry drops and tell "no drops" apart from "metric missing".
+        from src.telemetry.prometheus.metrics import prometheus_metrics
+
+        prometheus_metrics.initialize_telemetry_dropped_metrics(
+            reasons=[
+                f"{self.drop_reason_prefix}buffer_full",
+                f"{self.drop_reason_prefix}send_failed",
+            ]
+        )
+
         logger.info("Telemetry emitter started, endpoint: %s", self.endpoint)
 
     async def shutdown(self) -> None:
diff --git a/src/telemetry/prometheus/metrics.py b/src/telemetry/prometheus/metrics.py
index 01d7be4f..76a879ef 100644
--- a/src/telemetry/prometheus/metrics.py
+++ b/src/telemetry/prometheus/metrics.py
@@ -325,6 +325,27 @@ class PrometheusMetrics:
         except Exception as e:
             self._handle_metric_error("record_telemetry_event_dropped", e)
 
+    def initialize_telemetry_dropped_metrics(self, *, reasons: list[str]) -> None:
+        """Pre-create telemetry_events_dropped child series at 0.
+
+        A labeled Prometheus counter exports no time series until its first
+        ``labels(...)`` call, so ``telemetry_events_dropped`` stays invisible in
+        Prometheus/Grafana until an event is actually dropped — you cannot alert
+        on or graph a metric that does not exist yet. Materializing the
+        ``(namespace, reason)`` children at startup keeps the metric present at 0,
+        so a missing series signals a broken scrape rather than "no drops".
+
+        Args:
+            reasons: The reason label values the calling emitter can produce.
+        """
+        for reason in reasons:
+            try:
+                telemetry_events_dropped_counter.labels(reason=reason)
+            except Exception as e:
+                self._handle_metric_error(
+                    "initialize_telemetry_dropped_metrics", e
+                )
+
     def set_telemetry_buffer_size(self, *, size: int) -> None:
         try:
             telemetry_buffer_size_gauge.labels().set(size)