v3.0.8 Release Candidate (#763)

* chore(docs): Update changelogs for v3.0.8

* chore: update configuration docs

.env.template

@@ -44,12 +44,18 @@ DB_CONNECTION_URI=postgresql+psycopg://postgres:postgres@localhost:5432/postgres
 # DB_POOL_CLASS=default
 # DB_POOL_SIZE=10
 # DB_MAX_OVERFLOW=20
-# DB_POOL_TIMEOUT=30
+# DB_POOL_TIMEOUT=5  # seconds; must stay under DB_CONNECTION_RETRY_MAX_DELAY_SECONDS for a pooled (non-null) DB_POOL_CLASS
 # DB_POOL_RECYCLE=300
 # DB_POOL_PRE_PING=true
 # DB_POOL_USE_LIFO=true
 # DB_SQL_DEBUG=false
 # DB_TRACING=false
+# Bounded exponential-backoff retry around connection checkout (guards against
+# transient transaction-pooler saturation). Applied lazily on first DB use.
+# DB_CONNECTION_RETRY_ENABLED=true
+# DB_CONNECTION_RETRY_MAX_DELAY_SECONDS=10.0
+# DB_CONNECTION_RETRY_BACKOFF_INITIAL_SECONDS=0.1
+# DB_CONNECTION_RETRY_BACKOFF_MAX_SECONDS=2.0
 
 # =============================================================================
 # Authentication Settings
@@ -104,6 +110,11 @@ LLM_OPENAI_API_KEY=your-api-key-here
 # DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1
 # DERIVER_WORKERS=1
 # DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0
+# Adaptive polling: grows the idle/error sleep from the base toward the max by
+# the multiplier each cycle, snapping back to base when work is found.
+# DERIVER_POLLING_BACKOFF_ENABLED=true
+# DERIVER_POLLING_SLEEP_MAX_INTERVAL_SECONDS=30.0
+# DERIVER_POLLING_BACKOFF_MULTIPLIER=2.0
 # DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5
 # DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000  # 30 days
 # DERIVER_MODEL_CONFIG__TEMPERATURE=

CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [3.0.8] - 2026-06-01
+
+### Added
+
+- Connection-checkout retry with bounded exponential backoff (tenacity) on `get_db`/`tracked_db`: transient transaction-pooler (Supavisor) rejections — SQLAlchemy `TimeoutError` and `OperationalError` — now retry with backoff instead of surfacing as 500s under client-connection saturation. Gated by
+  `DB_CONNECTION_RETRY_ENABLED` with configurable delay/backoff knobs; ~10s default budget (#758)
+- `HonchoAsyncSession` — a lazy `AsyncSession` that checks out its pooled connection (with retry) on the first DB-touching call rather than at construction. Request handlers doing non-DB work (embedding, file, LLM) before their first query no longer pin a pooler connection across it. Only the checkout is retried;
+  the statement still runs exactly once, so writes are never duplicated (#758)
+- Adaptive deriver queue polling: the poll interval backs off when the queue is idle or erroring (base → max, doubling each cycle) and snaps back to base the moment work is claimed, cutting steady-state query load against the DB. Gated by `DERIVER_POLLING_BACKOFF_ENABLED` with configurable max/multiplier (#758)
+- New Prometheus `db_pool_connections` gauge (checked_out / checked_in / size / overflow), labeled `api`|`deriver`, registered in both the API lifespan and the deriver metrics server (#758)
+- New Prometheus `db_connection_acquisitions{outcome=ok|retried|exhausted}` counter — the alertable early-warning signal that connection checkouts are retrying through pooler rejection, before requests start failing (#758)
+- New Prometheus `db_queries_in_flight` gauge — statements actually executing on the wire (via SQLAlchemy cursor-execute events). Paired with `checked_out`, the gap reveals connections held but parked (the "idle in transaction during an external call" antipattern). Gated on `METRICS.ENABLED` for zero overhead when
+  off (#758)
+- Explicit `SqlalchemyIntegration` in both the API and deriver Sentry inits; connection acquisition wrapped in a `db.pool.acquire` span with live pool stats captured on retry exhaustion (#758)
+
+### Changed
+
+- Default `POOL_TIMEOUT` lowered to 5s, with validation that it stays under the connection-retry budget when a pooled (non-null) `POOL_CLASS` is configured; `config.toml.example` and the v2/v3 configuration docs updated to match (#758)
+- `HonchoAsyncSession` wraps every DB-touching session method (execute / scalar / scalars / flush / merge / refresh / commit / get / get_one / stream / stream_scalars / delete) so the lazy-checkout-with-retry guarantee has no holes; the acquired flag resets on `close()`/`reset()` so a reused session re-acquires on
+  next use (#758)
+
+### Fixed
+
+- Roll the session back on a retryable checkout failure before retrying — a failed autobegin could otherwise leave it pending-rollback, making the next connection attempt raise instead of cleanly re-checking-out (#758)
+- Guard `DBPoolCollector.collect()` so a pool-read/import hiccup can't raise and abort the entire `/metrics` scrape (Prometheus drops all metrics if any collector raises) (#758)
+- Clamp the pool overflow gauge to ≥ 0 (it could report negative before the pool fills) (#758)
+- Removed a double-sleep in the deriver idle poll so the backoff cap is a true cap rather than 2× (#758)
+
 ## [3.0.7] - 2026-05-21
 
 ### Added

config.toml.example

@@ -88,6 +88,13 @@ model = "text-embedding-3-small"
 ENABLED = true
 WORKERS = 1
 POLLING_SLEEP_INTERVAL_SECONDS = 1.0
+# Adaptive polling: when idle/erroring, the sleep interval grows from
+# POLLING_SLEEP_INTERVAL_SECONDS toward POLLING_SLEEP_MAX_INTERVAL_SECONDS by
+# POLLING_BACKOFF_MULTIPLIER each cycle, then snaps back to base when work is
+# found. Cuts steady-state query load against the shared DB/pooler.
+POLLING_BACKOFF_ENABLED = true
+POLLING_SLEEP_MAX_INTERVAL_SECONDS = 30.0
+POLLING_BACKOFF_MULTIPLIER = 2.0
 STALE_SESSION_TIMEOUT_MINUTES = 5
 # QUEUE_ERROR_RETENTION_SECONDS = 2592000  # 30 days
 DEDUPLICATE = true

docs/changelog/compatibility-guide.mdx

@@ -30,7 +30,8 @@ This guide helps you match the right SDK version to your Honcho API version. New
 
 | Honcho API Version | TypeScript SDK | Python SDK |
 |-------------------|---------------|------------|
-| v3.0.7 (Current) | v2.1.2 | v2.1.2 |
+| v3.0.8 (Current) | v2.1.2 | v2.1.2 |
+| v3.0.7 | v2.1.2 | v2.1.2 |
 | v3.0.6 | v2.1.1 | v2.1.1 |
 | v3.0.5 | v2.1.0 | v2.1.0 |
 | v3.0.4 | v2.1.0 | v2.1.0 |

docs/changelog/introduction.mdx

@@ -27,7 +27,35 @@ Welcome to the Honcho changelog! This section documents all notable changes to t
 ### Honcho API and SDK Changelogs
 <Tabs>
     <Tab title="Honcho API">
-        <Update label="v3.0.7 (Current)">
+        <Update label="v3.0.8 (Current)">
+        ### Added
+
+        - Connection-checkout retry with bounded exponential backoff (tenacity) on `get_db`/`tracked_db`: transient transaction-pooler (Supavisor) rejections — SQLAlchemy `TimeoutError` and `OperationalError` — now retry with backoff instead of surfacing as 500s under client-connection saturation. Gated by
+          `DB_CONNECTION_RETRY_ENABLED` with configurable delay/backoff knobs; ~10s default budget (#758)
+        - `HonchoAsyncSession` — a lazy `AsyncSession` that checks out its pooled connection (with retry) on the first DB-touching call rather than at construction. Request handlers doing non-DB work (embedding, file, LLM) before their first query no longer pin a pooler connection across it. Only the checkout is retried;
+          the statement still runs exactly once, so writes are never duplicated (#758)
+        - Adaptive deriver queue polling: the poll interval backs off when the queue is idle or erroring (base → max, doubling each cycle) and snaps back to base the moment work is claimed, cutting steady-state query load against the DB. Gated by `DERIVER_POLLING_BACKOFF_ENABLED` with configurable max/multiplier (#758)
+        - New Prometheus `db_pool_connections` gauge (checked_out / checked_in / size / overflow), labeled `api`|`deriver`, registered in both the API lifespan and the deriver metrics server (#758)
+        - New Prometheus `db_connection_acquisitions{outcome=ok|retried|exhausted}` counter — the alertable early-warning signal that connection checkouts are retrying through pooler rejection, before requests start failing (#758)
+        - New Prometheus `db_queries_in_flight` gauge — statements actually executing on the wire (via SQLAlchemy cursor-execute events). Paired with `checked_out`, the gap reveals connections held but parked (the "idle in transaction during an external call" antipattern). Gated on `METRICS.ENABLED` for zero overhead when
+          off (#758)
+        - Explicit `SqlalchemyIntegration` in both the API and deriver Sentry inits; connection acquisition wrapped in a `db.pool.acquire` span with live pool stats captured on retry exhaustion (#758)
+
+        ### Changed
+
+        - Default `POOL_TIMEOUT` lowered to 5s, with validation that it stays under the connection-retry budget when a pooled (non-null) `POOL_CLASS` is configured; `config.toml.example` and the v2/v3 configuration docs updated to match (#758)
+        - `HonchoAsyncSession` wraps every DB-touching session method (execute / scalar / scalars / flush / merge / refresh / commit / get / get_one / stream / stream_scalars / delete) so the lazy-checkout-with-retry guarantee has no holes; the acquired flag resets on `close()`/`reset()` so a reused session re-acquires on
+          next use (#758)
+
+        ### Fixed
+
+        - Roll the session back on a retryable checkout failure before retrying — a failed autobegin could otherwise leave it pending-rollback, making the next connection attempt raise instead of cleanly re-checking-out (#758)
+        - Guard `DBPoolCollector.collect()` so a pool-read/import hiccup can't raise and abort the entire `/metrics` scrape (Prometheus drops all metrics if any collector raises) (#758)
+        - Clamp the pool overflow gauge to ≥ 0 (it could report negative before the pool fills) (#758)
+        - Removed a double-sleep in the deriver idle poll so the backoff cap is a true cap rather than 2× (#758)
+        </Update>
+
+        <Update label="v3.0.7">
         ### Added
 
         - New `src/llm/` package as the single owner of provider runtime: clients, backends, history adapters, tool loop, request builder, credentials, and caching policy (#459)

docs/docs.json

@@ -24,7 +24,7 @@
   "navigation": {
     "versions": [
       {
-        "version": "v3.0.7",
+        "version": "v3.0.8",
         "api": {
           "openapi": ["v3/openapi.json"]
         },

docs/v3/contributing/configuration.mdx

@@ -354,6 +354,12 @@ DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS=2000
 # Worker settings
 DERIVER_WORKERS=1  # Increase for higher throughput
 DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0
+# Adaptive polling: when idle/erroring, the sleep interval grows from the base
+# toward DERIVER_POLLING_SLEEP_MAX_INTERVAL_SECONDS by the multiplier each cycle,
+# then snaps back to base when work is found. Cuts steady-state query load.
+DERIVER_POLLING_BACKOFF_ENABLED=true
+DERIVER_POLLING_SLEEP_MAX_INTERVAL_SECONDS=30.0
+DERIVER_POLLING_BACKOFF_MULTIPLIER=2.0
 DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5
 
 # Queue management

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "honcho"
-version = "3.0.7"
+version = "3.0.8"
 description = "Honcho Server"
 authors = [
   {name = "Plastic Labs", email = "hello@plasticlabs.ai"},

uv.lock

@@ -8,7 +8,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-05-16T17:58:57.678125Z"
+exclude-newer = "2026-05-27T18:30:19.790621Z"
 exclude-newer-span = "P5D"
 
 [manifest]
@@ -1159,7 +1159,7 @@ wheels = [
 
 [[package]]
 name = "honcho"
-version = "3.0.7"
+version = "3.0.8"
 source = { virtual = "." }
 dependencies = [
     { name = "alembic" },