telemetry: materialize dropped-event counter children at 0
A labeled Prometheus counter exports no series until its first labels() call, so telemetry_events_dropped stayed invisible until an event was actually dropped — impossible to alert on or graph, and "no drops" was indistinguishable from "metric missing / scrape broken". Pre-create the (namespace, reason) children at 0 on emitter start, for each reason the emitter can emit, so the metric is always present. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e7cbcc8432
commit
0f47e84993
|
|
@ -172,6 +172,20 @@ class TelemetryEmitter:
|
|||
)
|
||||
self._running = True
|
||||
self._flush_task = asyncio.create_task(self._periodic_flush())
|
||||
|
||||
# Pre-create the dropped-event counter children at 0 so the metric is
|
||||
# visible in Prometheus/Grafana before any drop occurs — a labeled
|
||||
# counter exports nothing until its first observation. Lets us alert on
|
||||
# telemetry drops and tell "no drops" apart from "metric missing".
|
||||
from src.telemetry.prometheus.metrics import prometheus_metrics
|
||||
|
||||
prometheus_metrics.initialize_telemetry_dropped_metrics(
|
||||
reasons=[
|
||||
f"{self.drop_reason_prefix}buffer_full",
|
||||
f"{self.drop_reason_prefix}send_failed",
|
||||
]
|
||||
)
|
||||
|
||||
logger.info("Telemetry emitter started, endpoint: %s", self.endpoint)
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
|
|
|
|||
|
|
@ -325,6 +325,27 @@ class PrometheusMetrics:
|
|||
except Exception as e:
|
||||
self._handle_metric_error("record_telemetry_event_dropped", e)
|
||||
|
||||
def initialize_telemetry_dropped_metrics(self, *, reasons: list[str]) -> None:
|
||||
"""Pre-create telemetry_events_dropped child series at 0.
|
||||
|
||||
A labeled Prometheus counter exports no time series until its first
|
||||
``labels(...)`` call, so ``telemetry_events_dropped`` stays invisible in
|
||||
Prometheus/Grafana until an event is actually dropped — you cannot alert
|
||||
on or graph a metric that does not exist yet. Materializing the
|
||||
``(namespace, reason)`` children at startup keeps the metric present at 0,
|
||||
so a missing series signals a broken scrape rather than "no drops".
|
||||
|
||||
Args:
|
||||
reasons: The reason label values the calling emitter can produce.
|
||||
"""
|
||||
for reason in reasons:
|
||||
try:
|
||||
telemetry_events_dropped_counter.labels(reason=reason)
|
||||
except Exception as e:
|
||||
self._handle_metric_error(
|
||||
"initialize_telemetry_dropped_metrics", e
|
||||
)
|
||||
|
||||
def set_telemetry_buffer_size(self, *, size: int) -> None:
|
||||
try:
|
||||
telemetry_buffer_size_gauge.labels().set(size)
|
||||
|
|
|
|||
Loading…
Reference in New Issue