diff --git a/src/main.py b/src/main.py index b0611f59..3c8bf3e8 100644 --- a/src/main.py +++ b/src/main.py @@ -4,15 +4,12 @@ import time import uuid from collections.abc import Awaitable, Callable from contextlib import asynccontextmanager -from typing import TYPE_CHECKING import sentry_sdk from fastapi import FastAPI, Request, Response -from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from fastapi_pagination import add_pagination -from pydantic import ValidationError from sentry_sdk.integrations.fastapi import FastApiIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sentry_sdk.integrations.starlette import StarletteIntegration @@ -42,9 +39,6 @@ from src.telemetry import ( from src.telemetry.logging import get_route_template from src.telemetry.sentry import initialize_sentry -if TYPE_CHECKING: - from sentry_sdk._types import Event, Hint - def get_log_level() -> int: """ @@ -88,30 +82,10 @@ class MetricsAccessFilter(logging.Filter): logging.getLogger("uvicorn.access").addFilter(MetricsAccessFilter()) -def before_send(event: "Event", hint: "Hint | None") -> "Event | None": - """Filter out events raised from known non-actionable exceptions before Sentry sees them.""" - if not hint: - return event - - exc_info = hint.get("exc_info") - if not exc_info: - return event - - _, exc_value, _ = exc_info - if isinstance(exc_value, HonchoException): - return None - - # Filters out ValidationErrors and RequestValidationErrors (typically coming from Pydantic) - if isinstance(exc_value, ValidationError | RequestValidationError): - logger.info(f"Filtering out validation error from Sentry: {exc_value}") - return None - - return event - - # Sentry Setup SENTRY_ENABLED = settings.SENTRY.ENABLED if SENTRY_ENABLED: + # before_send defaults to sentry.default_before_send (shared with the deriver). initialize_sentry( integrations=[ StarletteIntegration( @@ -123,7 +97,6 @@ if SENTRY_ENABLED: # Explicit so DB-query spans are not reliant on auto-enabling. SqlalchemyIntegration(), ], - before_send=before_send, ) diff --git a/src/telemetry/sentry.py b/src/telemetry/sentry.py index d84837ed..7b17e156 100644 --- a/src/telemetry/sentry.py +++ b/src/telemetry/sentry.py @@ -9,19 +9,58 @@ from functools import wraps from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast import sentry_sdk +from fastapi.exceptions import RequestValidationError +from pydantic import ValidationError +from sqlalchemy.exc import OperationalError from src.config import settings +from src.exceptions import HonchoException P = ParamSpec("P") T = TypeVar("T") if TYPE_CHECKING: - from sentry_sdk._types import EventProcessor + from sentry_sdk._types import Event, EventProcessor, Hint from sentry_sdk.integrations import Integration logger = logging.getLogger(__name__) +def default_before_send(event: Event, hint: Hint | None) -> Event | None: + """Filter/regroup known non-actionable events before Sentry ingests them. + + Shared by every entrypoint (API + deriver) so filtering is process-agnostic. + """ + if not hint: + return event + + exc_info = hint.get("exc_info") + if not exc_info: + return event + + _, exc_value, _ = exc_info + if isinstance(exc_value, HonchoException): + return None + + # Filters out ValidationErrors and RequestValidationErrors (typically from Pydantic) + if isinstance(exc_value, ValidationError | RequestValidationError): + logger.info(f"Filtering out validation error from Sentry: {exc_value}") + return None + + # DB connection-pool checkout timeouts are a fleet-wide saturation symptom, not a + # per-transaction bug. Collapse every occurrence into one issue (Sentry would otherwise + # split by transaction/endpoint) and drop to warning so it stops tripping error alerts. + # Watch it via a rate/spike metric alert instead. Root cause tracked in DEV-1852. + if isinstance(exc_value, OperationalError) and "connection timeout expired" in str( + exc_value + ): + event["fingerprint"] = ["honcho-db-connection-timeout"] + event["level"] = "warning" + return event + + return event + + # Paths whose transactions carry no debugging value but are hit constantly # (health checks, Prometheus scrapes, OpenAPI schema, docs). Tracing them at the # same rate as real traffic drowns the signal and burns tracing/profiling quota. @@ -81,13 +120,14 @@ def traces_sampler(sampling_context: dict[str, Any]) -> float: def initialize_sentry( *, integrations: Sequence[Integration], - before_send: EventProcessor | None = None, + before_send: EventProcessor | None = default_before_send, ) -> None: """Initialize Sentry SDK with project settings. Args: integrations: Sentry SDK integrations to enable (e.g., Starlette, FastAPI). - before_send: Optional event filter callback to suppress specific exceptions. + before_send: Event filter override. Defaults to ``default_before_send`` so + every entrypoint gets the shared filters; pass ``None`` to opt out. """ sentry_sdk.init( dsn=settings.SENTRY.DSN, diff --git a/tests/telemetry/test_sentry_before_send.py b/tests/telemetry/test_sentry_before_send.py new file mode 100644 index 00000000..7e63b555 --- /dev/null +++ b/tests/telemetry/test_sentry_before_send.py @@ -0,0 +1,82 @@ +"""Tests for the shared Sentry before_send filter. + +default_before_send runs in every entrypoint (API + deriver). It drops known +non-actionable exceptions and collapses DB connection-pool checkout timeouts +into a single warning-level issue so they stop spawning a fresh error issue per +transaction (fleet-wide saturation symptom, tracked in DEV-1852). +""" + +from typing import TYPE_CHECKING, cast + +import pytest +import sentry_sdk +from fastapi.exceptions import RequestValidationError +from pydantic import ValidationError +from sqlalchemy.exc import OperationalError + +from src.exceptions import ResourceNotFoundException +from src.telemetry.sentry import default_before_send, initialize_sentry + +if TYPE_CHECKING: + from sentry_sdk._types import Event, Hint + + +def _hint(exc: BaseException) -> "Hint": + return cast("Hint", {"exc_info": (type(exc), exc, None)}) + + +def _event(**kwargs: object) -> "Event": + return cast("Event", cast(object, dict(kwargs))) + + +def test_connection_timeout_is_consolidated_and_downgraded() -> None: + exc = OperationalError("SELECT 1", {}, Exception("connection timeout expired")) + out = default_before_send({}, _hint(exc)) + assert out == { + "fingerprint": ["honcho-db-connection-timeout"], + "level": "warning", + } + + +def test_unrelated_operational_error_passes_through() -> None: + exc = OperationalError("SELECT 1", {}, Exception("some other db failure")) + event = _event(level="error") + assert default_before_send(event, _hint(exc)) == {"level": "error"} + + +def test_honcho_and_validation_errors_are_dropped() -> None: + assert default_before_send({}, _hint(ResourceNotFoundException("nope"))) is None + assert ( + default_before_send({}, _hint(ValidationError.from_exception_data("x", []))) + is None + ) + assert default_before_send({}, _hint(RequestValidationError([]))) is None + + +def test_events_without_exc_info_pass_through() -> None: + event = _event(release="1.0") + assert default_before_send(event, None) == {"release": "1.0"} + assert default_before_send(event, cast("Hint", {})) == {"release": "1.0"} + + +def _captured_before_send(monkeypatch: pytest.MonkeyPatch, **kwargs: object) -> object: + captured: dict[str, object] = {} + + def fake_init(**init_kwargs: object) -> None: + captured.update(init_kwargs) + + monkeypatch.setattr(sentry_sdk, "init", fake_init) + initialize_sentry(integrations=[], **kwargs) # pyright: ignore[reportArgumentType] + return captured["before_send"] + + +def test_initialize_sentry_defaults_to_shared_filter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert _captured_before_send(monkeypatch) is default_before_send + + +def test_initialize_sentry_explicit_none_bypasses_shared_filter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert _captured_before_send(monkeypatch, before_send=None) is None