Vineeth/sentry filter consolidation (#934)

* fix: centralize sentry before_send filter config

* chore: comply with linter

* fix: default sentry filters

src/main.py

@@ -4,15 +4,12 @@ import time
 import uuid
 from collections.abc import Awaitable, Callable
 from contextlib import asynccontextmanager
-from typing import TYPE_CHECKING
 
 import sentry_sdk
 from fastapi import FastAPI, Request, Response
-from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from fastapi_pagination import add_pagination
-from pydantic import ValidationError
 from sentry_sdk.integrations.fastapi import FastApiIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 from sentry_sdk.integrations.starlette import StarletteIntegration
@@ -42,9 +39,6 @@ from src.telemetry import (
 from src.telemetry.logging import get_route_template
 from src.telemetry.sentry import initialize_sentry
 
-if TYPE_CHECKING:
-    from sentry_sdk._types import Event, Hint
-
 
 def get_log_level() -> int:
     """
@@ -88,30 +82,10 @@ class MetricsAccessFilter(logging.Filter):
 logging.getLogger("uvicorn.access").addFilter(MetricsAccessFilter())
 
 
-def before_send(event: "Event", hint: "Hint | None") -> "Event | None":
-    """Filter out events raised from known non-actionable exceptions before Sentry sees them."""
-    if not hint:
-        return event
-
-    exc_info = hint.get("exc_info")
-    if not exc_info:
-        return event
-
-    _, exc_value, _ = exc_info
-    if isinstance(exc_value, HonchoException):
-        return None
-
-    # Filters out ValidationErrors and RequestValidationErrors (typically coming from Pydantic)
-    if isinstance(exc_value, ValidationError | RequestValidationError):
-        logger.info(f"Filtering out validation error from Sentry: {exc_value}")
-        return None
-
-    return event
-
-
 # Sentry Setup
 SENTRY_ENABLED = settings.SENTRY.ENABLED
 if SENTRY_ENABLED:
+    # before_send defaults to sentry.default_before_send (shared with the deriver).
     initialize_sentry(
         integrations=[
             StarletteIntegration(
@@ -123,7 +97,6 @@ if SENTRY_ENABLED:
             # Explicit so DB-query spans are not reliant on auto-enabling.
             SqlalchemyIntegration(),
         ],
-        before_send=before_send,
     )
 
 

src/telemetry/sentry.py

@@ -9,19 +9,58 @@ from functools import wraps
 from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast
 
 import sentry_sdk
+from fastapi.exceptions import RequestValidationError
+from pydantic import ValidationError
+from sqlalchemy.exc import OperationalError
 
 from src.config import settings
+from src.exceptions import HonchoException
 
 P = ParamSpec("P")
 T = TypeVar("T")
 
 if TYPE_CHECKING:
-    from sentry_sdk._types import EventProcessor
+    from sentry_sdk._types import Event, EventProcessor, Hint
     from sentry_sdk.integrations import Integration
 
 logger = logging.getLogger(__name__)
 
 
+def default_before_send(event: Event, hint: Hint | None) -> Event | None:
+    """Filter/regroup known non-actionable events before Sentry ingests them.
+
+    Shared by every entrypoint (API + deriver) so filtering is process-agnostic.
+    """
+    if not hint:
+        return event
+
+    exc_info = hint.get("exc_info")
+    if not exc_info:
+        return event
+
+    _, exc_value, _ = exc_info
+    if isinstance(exc_value, HonchoException):
+        return None
+
+    # Filters out ValidationErrors and RequestValidationErrors (typically from Pydantic)
+    if isinstance(exc_value, ValidationError | RequestValidationError):
+        logger.info(f"Filtering out validation error from Sentry: {exc_value}")
+        return None
+
+    # DB connection-pool checkout timeouts are a fleet-wide saturation symptom, not a
+    # per-transaction bug. Collapse every occurrence into one issue (Sentry would otherwise
+    # split by transaction/endpoint) and drop to warning so it stops tripping error alerts.
+    # Watch it via a rate/spike metric alert instead. Root cause tracked in DEV-1852.
+    if isinstance(exc_value, OperationalError) and "connection timeout expired" in str(
+        exc_value
+    ):
+        event["fingerprint"] = ["honcho-db-connection-timeout"]
+        event["level"] = "warning"
+        return event
+
+    return event
+
+
 # Paths whose transactions carry no debugging value but are hit constantly
 # (health checks, Prometheus scrapes, OpenAPI schema, docs). Tracing them at the
 # same rate as real traffic drowns the signal and burns tracing/profiling quota.
@@ -81,13 +120,14 @@ def traces_sampler(sampling_context: dict[str, Any]) -> float:
 def initialize_sentry(
     *,
     integrations: Sequence[Integration],
-    before_send: EventProcessor | None = None,
+    before_send: EventProcessor | None = default_before_send,
 ) -> None:
     """Initialize Sentry SDK with project settings.
 
     Args:
         integrations: Sentry SDK integrations to enable (e.g., Starlette, FastAPI).
-        before_send: Optional event filter callback to suppress specific exceptions.
+        before_send: Event filter override. Defaults to ``default_before_send`` so
+            every entrypoint gets the shared filters; pass ``None`` to opt out.
     """
     sentry_sdk.init(
         dsn=settings.SENTRY.DSN,

tests/telemetry/test_sentry_before_send.py

@@ -0,0 +1,82 @@
+"""Tests for the shared Sentry before_send filter.
+
+default_before_send runs in every entrypoint (API + deriver). It drops known
+non-actionable exceptions and collapses DB connection-pool checkout timeouts
+into a single warning-level issue so they stop spawning a fresh error issue per
+transaction (fleet-wide saturation symptom, tracked in DEV-1852).
+"""
+
+from typing import TYPE_CHECKING, cast
+
+import pytest
+import sentry_sdk
+from fastapi.exceptions import RequestValidationError
+from pydantic import ValidationError
+from sqlalchemy.exc import OperationalError
+
+from src.exceptions import ResourceNotFoundException
+from src.telemetry.sentry import default_before_send, initialize_sentry
+
+if TYPE_CHECKING:
+    from sentry_sdk._types import Event, Hint
+
+
+def _hint(exc: BaseException) -> "Hint":
+    return cast("Hint", {"exc_info": (type(exc), exc, None)})
+
+
+def _event(**kwargs: object) -> "Event":
+    return cast("Event", cast(object, dict(kwargs)))
+
+
+def test_connection_timeout_is_consolidated_and_downgraded() -> None:
+    exc = OperationalError("SELECT 1", {}, Exception("connection timeout expired"))
+    out = default_before_send({}, _hint(exc))
+    assert out == {
+        "fingerprint": ["honcho-db-connection-timeout"],
+        "level": "warning",
+    }
+
+
+def test_unrelated_operational_error_passes_through() -> None:
+    exc = OperationalError("SELECT 1", {}, Exception("some other db failure"))
+    event = _event(level="error")
+    assert default_before_send(event, _hint(exc)) == {"level": "error"}
+
+
+def test_honcho_and_validation_errors_are_dropped() -> None:
+    assert default_before_send({}, _hint(ResourceNotFoundException("nope"))) is None
+    assert (
+        default_before_send({}, _hint(ValidationError.from_exception_data("x", [])))
+        is None
+    )
+    assert default_before_send({}, _hint(RequestValidationError([]))) is None
+
+
+def test_events_without_exc_info_pass_through() -> None:
+    event = _event(release="1.0")
+    assert default_before_send(event, None) == {"release": "1.0"}
+    assert default_before_send(event, cast("Hint", {})) == {"release": "1.0"}
+
+
+def _captured_before_send(monkeypatch: pytest.MonkeyPatch, **kwargs: object) -> object:
+    captured: dict[str, object] = {}
+
+    def fake_init(**init_kwargs: object) -> None:
+        captured.update(init_kwargs)
+
+    monkeypatch.setattr(sentry_sdk, "init", fake_init)
+    initialize_sentry(integrations=[], **kwargs)  # pyright: ignore[reportArgumentType]
+    return captured["before_send"]
+
+
+def test_initialize_sentry_defaults_to_shared_filter(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    assert _captured_before_send(monkeypatch) is default_before_send
+
+
+def test_initialize_sentry_explicit_none_bypasses_shared_filter(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    assert _captured_before_send(monkeypatch, before_send=None) is None