From 9267c7823c382ae5fc1d6fb67dc4be3d7c8792c2 Mon Sep 17 00:00:00 2001 From: cicav Date: Fri, 22 May 2026 12:37:18 +0800 Subject: [PATCH] fix: exponential backoff for rate-limit fallback cooldown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the fixed 60-second cooldown with exponential backoff: 30min → 1h → 2h → 4h cap. The counter is reset by restore_primary_runtime on successful primary-provider recovery, so the backoff is strictly for consecutive failures within a single degradation window. Closes #29702 --- agent/agent_runtime_helpers.py | 1 + agent/chat_completion_helpers.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index fab406a3eb3c8..1bdce7a4989d4 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1702,6 +1702,7 @@ def restore_primary_runtime(agent) -> bool: # ── Reset fallback chain for the new turn ── agent._fallback_activated = False agent._fallback_index = 0 + agent._rate_limit_backoff_count = 0 # reset exponential backoff counter # Reset the stale-call circuit breaker (#58962): the streak measured # the FALLBACK provider we're leaving; the restored primary deserves diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index e6e8ab7fdc1fc..deab15b077e44 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1712,7 +1712,16 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool current_provider = (getattr(agent, "provider", "") or "").strip().lower() primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower() if (not fallback_already_active) or (primary_provider and current_provider == primary_provider): - agent._rate_limited_until = time.monotonic() + 60 + # Exponential backoff: 30min → 1h → 2h → 4h cap + # Counter is reset by restore_primary_runtime on successful restore. + backoff_count = getattr(agent, "_rate_limit_backoff_count", 0) + agent._rate_limit_backoff_count = backoff_count + 1 + backoff_seconds = min(1800 * (2 ** backoff_count), 14400) + agent._rate_limited_until = time.monotonic() + backoff_seconds + logging.info( + "Rate-limit backoff level %d: cooldown %d s (%.1f min, backoff#%d)", + backoff_count, backoff_seconds, backoff_seconds / 60, backoff_count + 1, + ) if agent._fallback_index >= len(agent._fallback_chain): # Chain exhausted. If we actually walked a non-empty chain and the # failure was NOT a rate-limit/billing event (those already armed