diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index fab406a3eb3c8..1bdce7a4989d4 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1702,6 +1702,7 @@ def restore_primary_runtime(agent) -> bool: # ── Reset fallback chain for the new turn ── agent._fallback_activated = False agent._fallback_index = 0 + agent._rate_limit_backoff_count = 0 # reset exponential backoff counter # Reset the stale-call circuit breaker (#58962): the streak measured # the FALLBACK provider we're leaving; the restored primary deserves diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index e6e8ab7fdc1fc..deab15b077e44 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1712,7 +1712,16 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool current_provider = (getattr(agent, "provider", "") or "").strip().lower() primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower() if (not fallback_already_active) or (primary_provider and current_provider == primary_provider): - agent._rate_limited_until = time.monotonic() + 60 + # Exponential backoff: 30min → 1h → 2h → 4h cap + # Counter is reset by restore_primary_runtime on successful restore. + backoff_count = getattr(agent, "_rate_limit_backoff_count", 0) + agent._rate_limit_backoff_count = backoff_count + 1 + backoff_seconds = min(1800 * (2 ** backoff_count), 14400) + agent._rate_limited_until = time.monotonic() + backoff_seconds + logging.info( + "Rate-limit backoff level %d: cooldown %d s (%.1f min, backoff#%d)", + backoff_count, backoff_seconds, backoff_seconds / 60, backoff_count + 1, + ) if agent._fallback_index >= len(agent._fallback_chain): # Chain exhausted. If we actually walked a non-empty chain and the # failure was NOT a rate-limit/billing event (those already armed