From ad8c06047d6c820292b4d2d7f351afdbf732ec81 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:25:57 -0700 Subject: [PATCH] test: cover api_key_hint in strict pool doubles + real-pool routing regression Follow-up to the #43755 salvage: - Update the strict _Pool doubles in tests/run_agent/test_run_agent.py to accept api_key_hint and assert it carries the agent's failed key. - Add a real-CredentialPool regression (no mocks) proving the hint routes exhaustion to the entry whose key actually failed, not pool.current(), plus the no-hint baseline (#43747 wrong-entry marking). --- tests/agent/test_credential_pool_routing.py | 77 +++++++++++++++++++++ tests/run_agent/test_run_agent.py | 19 +++-- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/tests/agent/test_credential_pool_routing.py b/tests/agent/test_credential_pool_routing.py index 95b07a7039d9a..4345a043eb446 100644 --- a/tests/agent/test_credential_pool_routing.py +++ b/tests/agent/test_credential_pool_routing.py @@ -275,3 +275,80 @@ class TestPoolRotationCycle: pool.mark_exhausted_and_rotate.assert_called_once_with( status_code=402, error_context=None, api_key_hint="pool-current-key" ) + + +# --------------------------------------------------------------------------- +# 6. Real-pool regression: the hint routes exhaustion to the FAILED entry +# --------------------------------------------------------------------------- + +class TestApiKeyHintRealPool: + """Prove the routing guarantee through the real CredentialPool selector: + when the failed key differs from the pool's current/first entry, only the + failed entry is marked exhausted (#43747, wrong-entry marking).""" + + def _seed_pool(self, tmp_path, monkeypatch): + import json + + hermes_home = tmp_path / "hermes" + hermes_home.mkdir(parents=True, exist_ok=True) + (hermes_home / "auth.json").write_text( + json.dumps( + { + "version": 1, + "providers": {}, + "credential_pool": { + "openrouter": [ + { + "id": "cred-healthy", + "label": "healthy", + "auth_type": "api_key", + "priority": 0, + "source": "manual", + "access_token": "sk-or-healthy", + }, + { + "id": "cred-failed", + "label": "failed", + "auth_type": "api_key", + "priority": 1, + "source": "manual", + "access_token": "sk-or-failed", + }, + ] + }, + } + ) + ) + monkeypatch.setenv("HERMES_HOME", str(hermes_home)) + from agent.credential_pool import load_pool + + return load_pool("openrouter") + + def test_hint_marks_failed_entry_not_current(self, tmp_path, monkeypatch): + pool = self._seed_pool(tmp_path, monkeypatch) + # Another process/pool instance issued sk-or-failed; THIS pool's + # current() would resolve to the first (healthy) entry. + assert pool.select().access_token == "sk-or-healthy" + + next_entry = pool.mark_exhausted_and_rotate( + status_code=429, + error_context={"reason": "rate_limit_exceeded"}, + api_key_hint="sk-or-failed", + ) + + statuses = {e.id: e.last_status for e in pool._entries} + assert statuses["cred-failed"] == "exhausted" + assert statuses["cred-healthy"] in (None, "ok") + assert next_entry is not None + assert next_entry.access_token == "sk-or-healthy" + + def test_without_hint_current_entry_is_marked(self, tmp_path, monkeypatch): + """Baseline: no hint falls back to current() — the pre-fix behavior.""" + pool = self._seed_pool(tmp_path, monkeypatch) + assert pool.select().access_token == "sk-or-healthy" + + pool.mark_exhausted_and_rotate(status_code=429, error_context=None) + + statuses = {e.id: e.last_status for e in pool._entries} + assert statuses["cred-healthy"] == "exhausted" + assert statuses["cred-failed"] in (None, "ok") diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 26d82cbcfdf62..0dc7a12045fa1 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -6413,9 +6413,12 @@ class TestCredentialPoolRecovery: def current(self): return SimpleNamespace(label="primary") - def mark_exhausted_and_rotate(self, *, status_code, error_context=None): + def mark_exhausted_and_rotate( + self, *, status_code, error_context=None, api_key_hint=None + ): assert status_code == 429 assert error_context is None + assert api_key_hint == agent.api_key return next_entry agent._credential_pool = _Pool() @@ -6518,9 +6521,12 @@ class TestCredentialPoolRecovery: def try_refresh_current(self): return None # refresh failed - def mark_exhausted_and_rotate(self, *, status_code, error_context=None): + def mark_exhausted_and_rotate( + self, *, status_code, error_context=None, api_key_hint=None + ): assert status_code == 401 assert error_context is None + assert api_key_hint == agent.api_key return next_entry agent._credential_pool = _Pool() @@ -6542,7 +6548,9 @@ class TestCredentialPoolRecovery: def try_refresh_current(self): return None - def mark_exhausted_and_rotate(self, *, status_code, error_context=None): + def mark_exhausted_and_rotate( + self, *, status_code, error_context=None, api_key_hint=None + ): assert error_context is None return None # no more credentials @@ -6619,9 +6627,12 @@ class TestCredentialPoolRecovery: def current(self): return SimpleNamespace(label="primary") - def mark_exhausted_and_rotate(self, *, status_code, error_context=None): + def mark_exhausted_and_rotate( + self, *, status_code, error_context=None, api_key_hint=None + ): captured["status_code"] = status_code captured["error_context"] = error_context + captured["api_key_hint"] = api_key_hint return next_entry agent._credential_pool = _Pool()