fix: exponential backoff for rate-limit fallback cooldown

Replace the fixed 60-second cooldown with exponential backoff:
30min → 1h → 2h → 4h cap.

The counter is reset by restore_primary_runtime on successful
primary-provider recovery, so the backoff is strictly for
consecutive failures within a single degradation window.

Closes #29702
This commit is contained in:
cicav 2026-05-22 12:37:18 +08:00 committed by kshitij
parent a7ad713f43
commit 9267c7823c
2 changed files with 11 additions and 1 deletions

View File

@ -1702,6 +1702,7 @@ def restore_primary_runtime(agent) -> bool:
# ── Reset fallback chain for the new turn ──
agent._fallback_activated = False
agent._fallback_index = 0
agent._rate_limit_backoff_count = 0 # reset exponential backoff counter
# Reset the stale-call circuit breaker (#58962): the streak measured
# the FALLBACK provider we're leaving; the restored primary deserves

View File

@ -1712,7 +1712,16 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
current_provider = (getattr(agent, "provider", "") or "").strip().lower()
primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower()
if (not fallback_already_active) or (primary_provider and current_provider == primary_provider):
agent._rate_limited_until = time.monotonic() + 60
# Exponential backoff: 30min → 1h → 2h → 4h cap
# Counter is reset by restore_primary_runtime on successful restore.
backoff_count = getattr(agent, "_rate_limit_backoff_count", 0)
agent._rate_limit_backoff_count = backoff_count + 1
backoff_seconds = min(1800 * (2 ** backoff_count), 14400)
agent._rate_limited_until = time.monotonic() + backoff_seconds
logging.info(
"Rate-limit backoff level %d: cooldown %d s (%.1f min, backoff#%d)",
backoff_count, backoff_seconds, backoff_seconds / 60, backoff_count + 1,
)
if agent._fallback_index >= len(agent._fallback_chain):
# Chain exhausted. If we actually walked a non-empty chain and the
# failure was NOT a rate-limit/billing event (those already armed