fix: exponential backoff for rate-limit fallback cooldown
Replace the fixed 60-second cooldown with exponential backoff: 30min → 1h → 2h → 4h cap. The counter is reset by restore_primary_runtime on successful primary-provider recovery, so the backoff is strictly for consecutive failures within a single degradation window. Closes #29702
This commit is contained in:
parent
a7ad713f43
commit
9267c7823c
|
|
@ -1702,6 +1702,7 @@ def restore_primary_runtime(agent) -> bool:
|
|||
# ── Reset fallback chain for the new turn ──
|
||||
agent._fallback_activated = False
|
||||
agent._fallback_index = 0
|
||||
agent._rate_limit_backoff_count = 0 # reset exponential backoff counter
|
||||
|
||||
# Reset the stale-call circuit breaker (#58962): the streak measured
|
||||
# the FALLBACK provider we're leaving; the restored primary deserves
|
||||
|
|
|
|||
|
|
@ -1712,7 +1712,16 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
|
|||
current_provider = (getattr(agent, "provider", "") or "").strip().lower()
|
||||
primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower()
|
||||
if (not fallback_already_active) or (primary_provider and current_provider == primary_provider):
|
||||
agent._rate_limited_until = time.monotonic() + 60
|
||||
# Exponential backoff: 30min → 1h → 2h → 4h cap
|
||||
# Counter is reset by restore_primary_runtime on successful restore.
|
||||
backoff_count = getattr(agent, "_rate_limit_backoff_count", 0)
|
||||
agent._rate_limit_backoff_count = backoff_count + 1
|
||||
backoff_seconds = min(1800 * (2 ** backoff_count), 14400)
|
||||
agent._rate_limited_until = time.monotonic() + backoff_seconds
|
||||
logging.info(
|
||||
"Rate-limit backoff level %d: cooldown %d s (%.1f min, backoff#%d)",
|
||||
backoff_count, backoff_seconds, backoff_seconds / 60, backoff_count + 1,
|
||||
)
|
||||
if agent._fallback_index >= len(agent._fallback_chain):
|
||||
# Chain exhausted. If we actually walked a non-empty chain and the
|
||||
# failure was NOT a rate-limit/billing event (those already armed
|
||||
|
|
|
|||
Loading…
Reference in New Issue