From bb6dad9157b7aae07de802ebca84d340261016eb Mon Sep 17 00:00:00 2001 From: Vineeth Voruganti <13438633+VVoruganti@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:37:05 -0400 Subject: [PATCH] v3.0.8 Release Candidate (#763) * chore(docs): Update changelogs for v3.0.8 * chore: update configuration docs --- .env.template | 13 ++++++++++- CHANGELOG.md | 28 ++++++++++++++++++++++++ config.toml.example | 7 ++++++ docs/changelog/compatibility-guide.mdx | 3 ++- docs/changelog/introduction.mdx | 30 +++++++++++++++++++++++++- docs/docs.json | 2 +- docs/v3/contributing/configuration.mdx | 6 ++++++ pyproject.toml | 2 +- uv.lock | 4 ++-- 9 files changed, 88 insertions(+), 7 deletions(-) diff --git a/.env.template b/.env.template index c13040cb..a4ae6c96 100644 --- a/.env.template +++ b/.env.template @@ -44,12 +44,18 @@ DB_CONNECTION_URI=postgresql+psycopg://postgres:postgres@localhost:5432/postgres # DB_POOL_CLASS=default # DB_POOL_SIZE=10 # DB_MAX_OVERFLOW=20 -# DB_POOL_TIMEOUT=30 +# DB_POOL_TIMEOUT=5 # seconds; must stay under DB_CONNECTION_RETRY_MAX_DELAY_SECONDS for a pooled (non-null) DB_POOL_CLASS # DB_POOL_RECYCLE=300 # DB_POOL_PRE_PING=true # DB_POOL_USE_LIFO=true # DB_SQL_DEBUG=false # DB_TRACING=false +# Bounded exponential-backoff retry around connection checkout (guards against +# transient transaction-pooler saturation). Applied lazily on first DB use. +# DB_CONNECTION_RETRY_ENABLED=true +# DB_CONNECTION_RETRY_MAX_DELAY_SECONDS=10.0 +# DB_CONNECTION_RETRY_BACKOFF_INITIAL_SECONDS=0.1 +# DB_CONNECTION_RETRY_BACKOFF_MAX_SECONDS=2.0 # ============================================================================= # Authentication Settings @@ -104,6 +110,11 @@ LLM_OPENAI_API_KEY=your-api-key-here # DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL=https://openrouter.ai/api/v1 # DERIVER_WORKERS=1 # DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0 +# Adaptive polling: grows the idle/error sleep from the base toward the max by +# the multiplier each cycle, snapping back to base when work is found. +# DERIVER_POLLING_BACKOFF_ENABLED=true +# DERIVER_POLLING_SLEEP_MAX_INTERVAL_SECONDS=30.0 +# DERIVER_POLLING_BACKOFF_MULTIPLIER=2.0 # DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 # DERIVER_QUEUE_ERROR_RETENTION_SECONDS=2592000 # 30 days # DERIVER_MODEL_CONFIG__TEMPERATURE= diff --git a/CHANGELOG.md b/CHANGELOG.md index 60085e5e..b4552f14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [3.0.8] - 2026-06-01 + +### Added + +- Connection-checkout retry with bounded exponential backoff (tenacity) on `get_db`/`tracked_db`: transient transaction-pooler (Supavisor) rejections — SQLAlchemy `TimeoutError` and `OperationalError` — now retry with backoff instead of surfacing as 500s under client-connection saturation. Gated by + `DB_CONNECTION_RETRY_ENABLED` with configurable delay/backoff knobs; ~10s default budget (#758) +- `HonchoAsyncSession` — a lazy `AsyncSession` that checks out its pooled connection (with retry) on the first DB-touching call rather than at construction. Request handlers doing non-DB work (embedding, file, LLM) before their first query no longer pin a pooler connection across it. Only the checkout is retried; + the statement still runs exactly once, so writes are never duplicated (#758) +- Adaptive deriver queue polling: the poll interval backs off when the queue is idle or erroring (base → max, doubling each cycle) and snaps back to base the moment work is claimed, cutting steady-state query load against the DB. Gated by `DERIVER_POLLING_BACKOFF_ENABLED` with configurable max/multiplier (#758) +- New Prometheus `db_pool_connections` gauge (checked_out / checked_in / size / overflow), labeled `api`|`deriver`, registered in both the API lifespan and the deriver metrics server (#758) +- New Prometheus `db_connection_acquisitions{outcome=ok|retried|exhausted}` counter — the alertable early-warning signal that connection checkouts are retrying through pooler rejection, before requests start failing (#758) +- New Prometheus `db_queries_in_flight` gauge — statements actually executing on the wire (via SQLAlchemy cursor-execute events). Paired with `checked_out`, the gap reveals connections held but parked (the "idle in transaction during an external call" antipattern). Gated on `METRICS.ENABLED` for zero overhead when + off (#758) +- Explicit `SqlalchemyIntegration` in both the API and deriver Sentry inits; connection acquisition wrapped in a `db.pool.acquire` span with live pool stats captured on retry exhaustion (#758) + +### Changed + +- Default `POOL_TIMEOUT` lowered to 5s, with validation that it stays under the connection-retry budget when a pooled (non-null) `POOL_CLASS` is configured; `config.toml.example` and the v2/v3 configuration docs updated to match (#758) +- `HonchoAsyncSession` wraps every DB-touching session method (execute / scalar / scalars / flush / merge / refresh / commit / get / get_one / stream / stream_scalars / delete) so the lazy-checkout-with-retry guarantee has no holes; the acquired flag resets on `close()`/`reset()` so a reused session re-acquires on + next use (#758) + +### Fixed + +- Roll the session back on a retryable checkout failure before retrying — a failed autobegin could otherwise leave it pending-rollback, making the next connection attempt raise instead of cleanly re-checking-out (#758) +- Guard `DBPoolCollector.collect()` so a pool-read/import hiccup can't raise and abort the entire `/metrics` scrape (Prometheus drops all metrics if any collector raises) (#758) +- Clamp the pool overflow gauge to ≥ 0 (it could report negative before the pool fills) (#758) +- Removed a double-sleep in the deriver idle poll so the backoff cap is a true cap rather than 2× (#758) + ## [3.0.7] - 2026-05-21 ### Added diff --git a/config.toml.example b/config.toml.example index 258172a7..0ddf4c23 100644 --- a/config.toml.example +++ b/config.toml.example @@ -88,6 +88,13 @@ model = "text-embedding-3-small" ENABLED = true WORKERS = 1 POLLING_SLEEP_INTERVAL_SECONDS = 1.0 +# Adaptive polling: when idle/erroring, the sleep interval grows from +# POLLING_SLEEP_INTERVAL_SECONDS toward POLLING_SLEEP_MAX_INTERVAL_SECONDS by +# POLLING_BACKOFF_MULTIPLIER each cycle, then snaps back to base when work is +# found. Cuts steady-state query load against the shared DB/pooler. +POLLING_BACKOFF_ENABLED = true +POLLING_SLEEP_MAX_INTERVAL_SECONDS = 30.0 +POLLING_BACKOFF_MULTIPLIER = 2.0 STALE_SESSION_TIMEOUT_MINUTES = 5 # QUEUE_ERROR_RETENTION_SECONDS = 2592000 # 30 days DEDUPLICATE = true diff --git a/docs/changelog/compatibility-guide.mdx b/docs/changelog/compatibility-guide.mdx index f12d3cee..dfd40be1 100644 --- a/docs/changelog/compatibility-guide.mdx +++ b/docs/changelog/compatibility-guide.mdx @@ -30,7 +30,8 @@ This guide helps you match the right SDK version to your Honcho API version. New | Honcho API Version | TypeScript SDK | Python SDK | |-------------------|---------------|------------| -| v3.0.7 (Current) | v2.1.2 | v2.1.2 | +| v3.0.8 (Current) | v2.1.2 | v2.1.2 | +| v3.0.7 | v2.1.2 | v2.1.2 | | v3.0.6 | v2.1.1 | v2.1.1 | | v3.0.5 | v2.1.0 | v2.1.0 | | v3.0.4 | v2.1.0 | v2.1.0 | diff --git a/docs/changelog/introduction.mdx b/docs/changelog/introduction.mdx index 29c15fdf..8e2febcb 100644 --- a/docs/changelog/introduction.mdx +++ b/docs/changelog/introduction.mdx @@ -27,7 +27,35 @@ Welcome to the Honcho changelog! This section documents all notable changes to t ### Honcho API and SDK Changelogs - + + ### Added + + - Connection-checkout retry with bounded exponential backoff (tenacity) on `get_db`/`tracked_db`: transient transaction-pooler (Supavisor) rejections — SQLAlchemy `TimeoutError` and `OperationalError` — now retry with backoff instead of surfacing as 500s under client-connection saturation. Gated by + `DB_CONNECTION_RETRY_ENABLED` with configurable delay/backoff knobs; ~10s default budget (#758) + - `HonchoAsyncSession` — a lazy `AsyncSession` that checks out its pooled connection (with retry) on the first DB-touching call rather than at construction. Request handlers doing non-DB work (embedding, file, LLM) before their first query no longer pin a pooler connection across it. Only the checkout is retried; + the statement still runs exactly once, so writes are never duplicated (#758) + - Adaptive deriver queue polling: the poll interval backs off when the queue is idle or erroring (base → max, doubling each cycle) and snaps back to base the moment work is claimed, cutting steady-state query load against the DB. Gated by `DERIVER_POLLING_BACKOFF_ENABLED` with configurable max/multiplier (#758) + - New Prometheus `db_pool_connections` gauge (checked_out / checked_in / size / overflow), labeled `api`|`deriver`, registered in both the API lifespan and the deriver metrics server (#758) + - New Prometheus `db_connection_acquisitions{outcome=ok|retried|exhausted}` counter — the alertable early-warning signal that connection checkouts are retrying through pooler rejection, before requests start failing (#758) + - New Prometheus `db_queries_in_flight` gauge — statements actually executing on the wire (via SQLAlchemy cursor-execute events). Paired with `checked_out`, the gap reveals connections held but parked (the "idle in transaction during an external call" antipattern). Gated on `METRICS.ENABLED` for zero overhead when + off (#758) + - Explicit `SqlalchemyIntegration` in both the API and deriver Sentry inits; connection acquisition wrapped in a `db.pool.acquire` span with live pool stats captured on retry exhaustion (#758) + + ### Changed + + - Default `POOL_TIMEOUT` lowered to 5s, with validation that it stays under the connection-retry budget when a pooled (non-null) `POOL_CLASS` is configured; `config.toml.example` and the v2/v3 configuration docs updated to match (#758) + - `HonchoAsyncSession` wraps every DB-touching session method (execute / scalar / scalars / flush / merge / refresh / commit / get / get_one / stream / stream_scalars / delete) so the lazy-checkout-with-retry guarantee has no holes; the acquired flag resets on `close()`/`reset()` so a reused session re-acquires on + next use (#758) + + ### Fixed + + - Roll the session back on a retryable checkout failure before retrying — a failed autobegin could otherwise leave it pending-rollback, making the next connection attempt raise instead of cleanly re-checking-out (#758) + - Guard `DBPoolCollector.collect()` so a pool-read/import hiccup can't raise and abort the entire `/metrics` scrape (Prometheus drops all metrics if any collector raises) (#758) + - Clamp the pool overflow gauge to ≥ 0 (it could report negative before the pool fills) (#758) + - Removed a double-sleep in the deriver idle poll so the backoff cap is a true cap rather than 2× (#758) + + + ### Added - New `src/llm/` package as the single owner of provider runtime: clients, backends, history adapters, tool loop, request builder, credentials, and caching policy (#459) diff --git a/docs/docs.json b/docs/docs.json index 64b56d86..47d5089f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -24,7 +24,7 @@ "navigation": { "versions": [ { - "version": "v3.0.7", + "version": "v3.0.8", "api": { "openapi": ["v3/openapi.json"] }, diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index d7a71b6c..04224e4b 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -354,6 +354,12 @@ DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS=2000 # Worker settings DERIVER_WORKERS=1 # Increase for higher throughput DERIVER_POLLING_SLEEP_INTERVAL_SECONDS=1.0 +# Adaptive polling: when idle/erroring, the sleep interval grows from the base +# toward DERIVER_POLLING_SLEEP_MAX_INTERVAL_SECONDS by the multiplier each cycle, +# then snaps back to base when work is found. Cuts steady-state query load. +DERIVER_POLLING_BACKOFF_ENABLED=true +DERIVER_POLLING_SLEEP_MAX_INTERVAL_SECONDS=30.0 +DERIVER_POLLING_BACKOFF_MULTIPLIER=2.0 DERIVER_STALE_SESSION_TIMEOUT_MINUTES=5 # Queue management diff --git a/pyproject.toml b/pyproject.toml index 31581efb..1079b9eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "honcho" -version = "3.0.7" +version = "3.0.8" description = "Honcho Server" authors = [ {name = "Plastic Labs", email = "hello@plasticlabs.ai"}, diff --git a/uv.lock b/uv.lock index d577957f..a2dac3b8 100644 --- a/uv.lock +++ b/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-05-16T17:58:57.678125Z" +exclude-newer = "2026-05-27T18:30:19.790621Z" exclude-newer-span = "P5D" [manifest] @@ -1159,7 +1159,7 @@ wheels = [ [[package]] name = "honcho" -version = "3.0.7" +version = "3.0.8" source = { virtual = "." } dependencies = [ { name = "alembic" },