feat: add route-level latency metrics

src/main.py

@@ -1,5 +1,6 @@
 import logging
 import re
+import time
 import uuid
 from collections.abc import Awaitable, Callable
 from contextlib import asynccontextmanager
@@ -250,6 +251,7 @@ async def track_request(
     token = request_context.set(f"api:{request_id}")
 
     try:
+        start_time = time.perf_counter()
         response = await call_next(request)
 
         # Track metrics if enabled
@@ -259,6 +261,7 @@ async def track_request(
                 method=request.method,
                 endpoint=template,
                 status_code=str(response.status_code),
+                duration_seconds=time.perf_counter() - start_time,
             )
 
         return response

src/telemetry/prometheus/metrics.py

@@ -12,6 +12,7 @@ from prometheus_client import (
     REGISTRY,
     Counter,
     Gauge,
+    Histogram,
     disable_created_metrics,
     generate_latest,
 )
@@ -38,6 +39,12 @@ class NamespacedGauge(Gauge):
         return super().labels(**kwargs)  # type: ignore[return-value]
 
 
+class NamespacedHistogram(Histogram):
+    def labels(self, **kwargs: str) -> NamespacedHistogram:
+        kwargs["namespace"] = cast(str, settings.METRICS.NAMESPACE)
+        return super().labels(**kwargs)  # type: ignore[return-value]
+
+
 class TokenTypes(Enum):
     INPUT = "input"
     OUTPUT = "output"
@@ -65,6 +72,15 @@ api_requests_counter = NamespacedCounter(
     ["namespace", "method", "endpoint", "status_code"],
 )
 
+# Per-route latency. Buckets are a geometric ladder spanning
+# the full range of API classes
+api_request_duration_seconds = NamespacedHistogram(
+    "api_request_duration_seconds",
+    "API request latency in seconds",
+    ["namespace", "method", "endpoint"],
+    buckets=(0.05, 0.1, 0.25, 0.5, 0.75, 1, 2, 5, 10, 20, 30, 60, 120),
+)
+
 messages_created_counter = NamespacedCounter(
     "messages_created",
     "Total messages created",
@@ -160,6 +176,7 @@ class PrometheusMetrics:
         method: str,
         endpoint: str,
         status_code: str,
+        duration_seconds: float,
     ) -> None:
         try:
             api_requests_counter.labels(
@@ -167,6 +184,10 @@ class PrometheusMetrics:
                 endpoint=endpoint,
                 status_code=status_code,
             ).inc()
+            api_request_duration_seconds.labels(
+                method=method,
+                endpoint=endpoint,
+            ).observe(duration_seconds)
         except Exception as e:
             self._handle_metric_error("record_api_request", e)