diff --git a/plugins/memory/honcho/__init__.py b/plugins/memory/honcho/__init__.py index 78d24e90b3d9a..9246f26b6e0b5 100644 --- a/plugins/memory/honcho/__init__.py +++ b/plugins/memory/honcho/__init__.py @@ -522,7 +522,7 @@ class HonchoMemoryProvider(MemoryProvider): ) except Exception as exc: logger.debug("Honcho dialectic prewarm failed: %s", exc) - self._dialectic_empty_streak += 1 + self._note_dialectic_failure(exc) return if r and r.strip(): with self._prefetch_lock: @@ -715,6 +715,11 @@ class HonchoMemoryProvider(MemoryProvider): parts = [] + # One-time notice, relayed by the model, that auth is dead and memory is paused. + auth_notice = self._pop_auth_notice() + if auth_notice: + parts.append(auth_notice) + # ----- Layer 1: Base context (representation + card) ----- if not _skip_base: # The first base fetch gets the remaining turn-1 budget. Later @@ -804,7 +809,7 @@ class HonchoMemoryProvider(MemoryProvider): r = self._run_dialectic_depth(query) except Exception as exc: logger.debug("Honcho first-turn dialectic failed: %s", exc) - self._dialectic_empty_streak += 1 + self._note_dialectic_failure(exc) return if r and r.strip(): with self._prefetch_lock: @@ -845,6 +850,25 @@ class HonchoMemoryProvider(MemoryProvider): return result + def _pop_auth_notice(self) -> str: + """One-time model-facing notice that Honcho auth expired and memory is paused.""" + pop = getattr(self._manager, "pop_auth_notice", None) + if not callable(pop): + return "" + try: + msg = pop() + except Exception: + return "" + if not isinstance(msg, str) or not msg: + return "" + return ( + "[Honcho memory status] Authentication with the Honcho memory backend " + "has expired and automatic token refresh failed, so memory sync and " + f"recall are paused. Reason: {msg}\n" + "Tell the user (once) that Honcho memory is paused and that running " + "'hermes honcho setup' to re-authenticate will restore it." + ) + def _consume_pending_dialectic(self) -> str: """Pop any pending dialectic result, applying the stale-discard guard. @@ -941,7 +965,7 @@ class HonchoMemoryProvider(MemoryProvider): result = self._run_dialectic_depth(query) except Exception as e: logger.debug("Honcho prefetch failed: %s", e) - self._dialectic_empty_streak += 1 + self._note_dialectic_failure(e) return if result and result.strip(): with self._prefetch_lock: @@ -1020,6 +1044,22 @@ class HonchoMemoryProvider(MemoryProvider): ceiling = self._dialectic_cadence * self._BACKOFF_MAX return min(widened, ceiling) + def _note_dialectic_failure(self, exc: BaseException) -> None: + """Widen the empty-streak backoff after a failed dialectic cycle. + + Auth failures are exempt: waiting cannot fix a dead token, and backoff + would delay recovery after the user re-authenticates. + """ + from plugins.memory.honcho.session import HonchoAuthError + + if isinstance(exc, HonchoAuthError): + logger.warning( + "Honcho dialectic auth failure (not counted toward cadence backoff): %s", + exc, + ) + return + self._dialectic_empty_streak += 1 + def liveness_snapshot(self) -> dict: """In-process snapshot of dialectic liveness state for diagnostics. diff --git a/plugins/memory/honcho/client.py b/plugins/memory/honcho/client.py index 1945a38a15fe7..fd2e593faf102 100644 --- a/plugins/memory/honcho/client.py +++ b/plugins/memory/honcho/client.py @@ -934,8 +934,9 @@ def _resolve_timeout_from_sources(config: HonchoClientConfig | None) -> float: def _apply_fresh_oauth_token(config: HonchoClientConfig) -> None: """Refresh a near-expiry OAuth grant and point ``config.api_key`` at it. - No-op for static API keys or when refresh fails (fail-open: the stale token - is left in place and the existing 401 handling degrades gracefully). + No-op for static API keys or when refresh fails: the stale token stays in + place and the first rejected call triggers the post-401 recovery in + session.py (forced rotation, one retry). """ try: from plugins.memory.honcho import oauth diff --git a/plugins/memory/honcho/oauth.py b/plugins/memory/honcho/oauth.py index 3d7cbbd02c82f..cdd31daf669bf 100644 --- a/plugins/memory/honcho/oauth.py +++ b/plugins/memory/honcho/oauth.py @@ -6,15 +6,22 @@ expiry to keep it live. Refresh tokens rotate with single-use reuse detection: a replayed stale token revokes the whole grant. So every refresh must persist the rotated token -atomically and be serialized — and a failed refresh never raises into the -agent (stale token stays; the fail-open path absorbs the eventual 401). +atomically and be serialized. A failed exchange never raises into the agent: +transient failures retry once immediately (the server re-rotates a replayed +refresh token only within a short grace window, so waiting for the next +memory call is too late), and a permanent OAuth error such as invalid_grant +marks the grant dead so nothing keeps hitting the token endpoint — callers +surface a re-login prompt instead. A server-side 401 on a locally-valid +token is recovered via ``force_refresh_token``. """ from __future__ import annotations +import hashlib import json import logging import os +import re import threading import time from contextlib import contextmanager @@ -35,6 +42,29 @@ _REFRESH_SKEW_SECONDS = 120 # on the path to a memory call, and a stalled auth server must not hang it. _REFRESH_TIMEOUT_SECONDS = 15.0 +# Retry pause, kept short: the server honors a replayed refresh token only briefly after rotating it. +_REFRESH_RETRY_DELAY_SECONDS = 2.0 + +# OAuth error codes that a retry can never fix — the grant itself is dead. +_PERMANENT_OAUTH_ERRORS = frozenset({"invalid_grant", "invalid_client", "unauthorized_client"}) + +# Token values are secret even though their prefixes are not; redact before logging. +_TOKEN_VALUE_RE = re.compile(r"hch-(at|rt)-[A-Za-z0-9._~+/=-]+") + + +def _redact_tokens(text: str) -> str: + """Replace any embedded token values with their prefix plus a placeholder.""" + return _TOKEN_VALUE_RE.sub(lambda m: f"hch-{m.group(1)}-[redacted]", text) + + +class OAuthRefreshError(Exception): + """Token endpoint rejected the refresh. ``permanent`` means re-login is required.""" + + def __init__(self, message: str, *, error: str = "", permanent: bool = False): + super().__init__(message) + self.error = error + self.permanent = permanent + # Serializes refresh across threads sharing one process's config. Re-checked # under the lock (double-checked) so racing callers don't replay a rotated # refresh token and trip reuse detection. @@ -99,6 +129,31 @@ def _config_refresh_lock(path: Path): # until the token nears expiry and disk is read again. _expiry_cache: dict[tuple[str, str], tuple[float, str]] = {} +# Permanently rejected grants: (config path, host) → sha256 of the dead refresh token; a re-login rotates the token, so the digest check self-clears. +_dead_grants: dict[tuple[str, str], str] = {} + + +def _refresh_token_digest(cred: OAuthCredential) -> str: + return hashlib.sha256(cred.refresh_token.encode("utf-8")).hexdigest() + + +def _grant_is_dead(key: tuple[str, str], cred: OAuthCredential) -> bool: + return _dead_grants.get(key) == _refresh_token_digest(cred) + + +def _mark_grant_dead(key: tuple[str, str], cred: OAuthCredential) -> None: + _dead_grants[key] = _refresh_token_digest(cred) + + +def reauth_required(path: Path, host: str) -> bool: + """True when ``host``'s OAuth grant is dead and only a new login fixes it.""" + key = (str(path), host) + if key not in _dead_grants: + return False + block = (_read_config(path).get("hosts") or {}).get(host) or {} + cred = OAuthCredential.from_host_block(block) + return cred is not None and _grant_is_dead(key, cred) + def is_oauth_access_token(value: str | None) -> bool: """True when ``value`` is an OAuth access token (vs a static API key).""" @@ -208,9 +263,10 @@ def _http_get_json(url: str, timeout: float) -> dict[str, Any]: def _exchange_refresh_token(cred: OAuthCredential, *, now: float) -> OAuthCredential: """Run the refresh_token grant and return the rotated credential. - Raises on any transport/protocol failure; callers fail open. + Raises ``OAuthRefreshError`` (with the endpoint's error body) on an error + response, transport errors as-is; callers fail open. """ - body = _http_post_form( + status, body = _http_post_form_status( cred.token_endpoint, { "grant_type": "refresh_token", @@ -219,6 +275,15 @@ def _exchange_refresh_token(cred: OAuthCredential, *, now: float) -> OAuthCreden }, _REFRESH_TIMEOUT_SECONDS, ) + if status >= 400: + error = str(body.get("error") or "") + description = str(body.get("error_description") or "") + detail = " — ".join(p for p in (error, description) if p) or "no error body" + raise OAuthRefreshError( + _redact_tokens(f"token endpoint returned HTTP {status}: {detail}"), + error=error, + permanent=error in _PERMANENT_OAUTH_ERRORS, + ) access = body.get("access_token") refresh = body.get("refresh_token") if not is_oauth_access_token(access) or not refresh: @@ -238,6 +303,29 @@ def _exchange_refresh_token(cred: OAuthCredential, *, now: float) -> OAuthCreden ) +def _exchange_with_retry(cred: OAuthCredential, *, now: float) -> OAuthCredential: + """Exchange the refresh token, retrying once immediately on transient failure. + + The server honors a replayed refresh token only within a short grace + window after rotating it, so the retry must happen now — deferring to the + next memory call lands outside the window and revokes the grant. + """ + try: + return _exchange_refresh_token(cred, now=now) + except OAuthRefreshError as exc: + if exc.permanent: + raise + first: Exception = exc + except Exception as exc: + first = exc + logger.warning( + "Honcho OAuth token exchange failed, retrying once: %s", + _redact_tokens(str(first)), + ) + time.sleep(_REFRESH_RETRY_DELAY_SECONDS) + return _exchange_refresh_token(cred, now=now) + + def _read_config(path: Path) -> dict[str, Any]: try: return json.loads(path.read_text(encoding="utf-8")) @@ -279,6 +367,7 @@ def _persist_credential(path: Path, host: str, cred: OAuthCredential) -> None: block["oauth"] = cred.oauth_block() _atomic_write_config(path, raw) _expiry_cache[(str(path), host)] = (cred.expires_at, cred.access_token) + _dead_grants.pop((str(path), host), None) def ensure_fresh_token( @@ -293,7 +382,9 @@ def ensure_fresh_token( Returns ``(None, False)`` when the host has no OAuth credential (e.g. a plain API key) so callers leave the existing token untouched. Refresh failures are swallowed: the current (possibly stale) token is returned with - ``refreshed=False`` and the fail-open path handles any resulting 401. + ``refreshed=False``, transient failures retry once immediately, and a + permanently rejected grant is marked dead so later calls skip the endpoint. + The 401 recovery in session.py escalates dead grants to the user. """ now = time.time() if now is None else now key = (str(path), host) @@ -323,16 +414,78 @@ def ensure_fresh_token( current = OAuthCredential.from_host_block(fresh_block) or cred if not current.is_expired(now=now): return current.access_token, current.access_token != cred.access_token + if _grant_is_dead(key, current): + return current.access_token, False try: - rotated = _exchange_refresh_token(current, now=now) + rotated = _exchange_with_retry(current, now=now) + except OAuthRefreshError as exc: + if exc.permanent: + _mark_grant_dead(key, current) + logger.error( + "Honcho OAuth grant for host %s is no longer valid (%s); " + "run 'hermes honcho setup' to re-authenticate", host, exc, + ) + else: + logger.warning("Honcho OAuth refresh failed for host %s: %s", host, exc) + return current.access_token, False except Exception as exc: - logger.warning("Honcho OAuth refresh failed for host %s: %s", host, exc) + logger.warning( + "Honcho OAuth refresh failed for host %s: %s", + host, _redact_tokens(str(exc)), + ) return current.access_token, False _persist_credential(path, host, rotated) logger.info("Honcho OAuth token refreshed for host %s", host) return rotated.access_token, True +def force_refresh_token(path: Path, host: str) -> str | None: + """Rotate ``host``'s access token now, ignoring local expiry math. + + Recovery path for a server-side 401 on a token the local clock still + considers valid (clock skew, server-side rotation, revocation). Returns + the new access token, or ``None`` when the host has no OAuth credential, + the grant is dead, or the exchange fails — callers surface the original + auth error in that case. + """ + now = time.time() + key = (str(path), host) + with _refresh_lock, _config_refresh_lock(path): + block = (_read_config(path).get("hosts") or {}).get(host) or {} + cred = OAuthCredential.from_host_block(block) + if cred is None: + _expiry_cache.pop(key, None) + return None + if _grant_is_dead(key, cred): + return None + cached = _expiry_cache.get(key) + # Another thread or process already rotated: adopt the newer on-disk token. + if cached is not None and cred.access_token != cached[1] and not cred.is_expired(now=now): + _expiry_cache[key] = (cred.expires_at, cred.access_token) + return cred.access_token + try: + rotated = _exchange_with_retry(cred, now=now) + except OAuthRefreshError as exc: + if exc.permanent: + _mark_grant_dead(key, cred) + logger.error( + "Honcho OAuth grant for host %s is no longer valid (%s); " + "run 'hermes honcho setup' to re-authenticate", host, exc, + ) + else: + logger.warning("Honcho OAuth forced refresh failed for host %s: %s", host, exc) + return None + except Exception as exc: + logger.warning( + "Honcho OAuth forced refresh failed for host %s: %s", + host, _redact_tokens(str(exc)), + ) + return None + _persist_credential(path, host, rotated) + logger.info("Honcho OAuth token force-refreshed for host %s after an auth failure", host) + return rotated.access_token + + def install_grant( path: Path, host: str, @@ -378,6 +531,7 @@ def install_grant( if apply_config: _deep_merge(raw, granted_config) _expiry_cache[(str(path), host)] = (cred.expires_at, cred.access_token) + _dead_grants.pop((str(path), host), None) hosts = raw.setdefault("hosts", {}) block = hosts.setdefault(host, {}) block["apiKey"] = cred.access_token diff --git a/plugins/memory/honcho/session.py b/plugins/memory/honcho/session.py index d4a53a6e42436..76cf4e94598b4 100644 --- a/plugins/memory/honcho/session.py +++ b/plugins/memory/honcho/session.py @@ -24,6 +24,38 @@ _PEER_ID_HASH_LEN = 8 _PEER_ID_HASH_ESCALATION_LENGTHS = (_PEER_ID_HASH_LEN, 12, 16, 24, 32, 64) +class HonchoAuthError(RuntimeError): + """Auth failure that survived a forced token refresh and one retry. + + Raised instead of swallowed so callers can tell a rejected credential + apart from an empty result; cadence backoff must not count it as empty. + """ + + +# The SDK raises different exception classes per transport, but an auth failure always carries one of these markers. +_AUTH_ERROR_MARKERS = ( + "invalid or expired access token", + "authentication", + "unauthorized", + "401", +) + + +def _is_auth_error(exc: BaseException) -> bool: + status = getattr(exc, "status_code", None) or getattr(exc, "status", None) + if status == 401: + return True + text = str(exc).lower() + return any(marker in text for marker in _AUTH_ERROR_MARKERS) + + +def _auth_error_message(exc: BaseException) -> str: + return ( + f"Honcho rejected our credentials and a forced token refresh did not recover: {exc}. " + "Re-authenticate with 'hermes honcho setup'." + ) + + @dataclass class HonchoSession: """ @@ -105,6 +137,10 @@ class HonchoSessionManager: self._peers_cache: dict[str, Any] = {} self._sessions_cache: dict[str, Any] = {} + # Set when a call still fails auth after a forced token refresh; cleared on the next success. + self._auth_failure: str | None = None + self._auth_notice_emitted = False + # Write frequency state write_frequency = (config.write_frequency if config else "async") self._write_frequency = write_frequency @@ -161,6 +197,59 @@ class HonchoSessionManager: self._honcho = get_honcho_client() return self._honcho + def _record_auth_failure(self, exc: BaseException) -> None: + if self._auth_failure is None: + logger.error( + "Honcho authentication failed and token refresh did not recover; " + "memory sync and recall are paused until the user re-authenticates: %s", + exc, + ) + self._auth_failure = str(exc) + + def _clear_auth_failure(self) -> None: + if self._auth_failure is not None: + logger.info("Honcho authentication recovered; memory sync and recall resumed") + self._auth_failure = None + self._auth_notice_emitted = False + + def pop_auth_notice(self) -> str | None: + """Return the pending auth-failure message once; later calls return None.""" + if self._auth_failure is None or self._auth_notice_emitted: + return None + self._auth_notice_emitted = True + return self._auth_failure + + def _force_reauth(self) -> bool: + """Rotate the OAuth token after a server-side 401 and rebind the client. + + Returns True when a fresh token was obtained and applied. False for + static API keys (no OAuth credential), a dead grant, or a failed + exchange — callers surface the original auth error in that case. + """ + try: + from plugins.memory.honcho import oauth + from plugins.memory.honcho.client import ( + reset_honcho_client, + resolve_config_path, + ) + + host = getattr(self._config, "host", "") or "" + if not host: + return False + token = oauth.force_refresh_token(resolve_config_path(), host) + if not token: + return False + if not oauth.apply_token_to_client(self.honcho, token): + # SDK shape changed: rebuild the client and drop objects holding the old transport. + reset_honcho_client() + with self._cache_lock: + self._peers_cache.clear() + self._sessions_cache.clear() + return True + except Exception: + logger.warning("Honcho post-401 token refresh failed", exc_info=True) + return False + def _get_or_create_peer(self, peer_id: str) -> Any: """ Get or create a Honcho peer. @@ -442,7 +531,19 @@ class HonchoSessionManager: honcho_messages.append(peer.message(msg["content"])) try: - honcho_session.add_messages(honcho_messages) + try: + honcho_session.add_messages(honcho_messages) + except Exception as e: + if not _is_auth_error(e): + raise + logger.warning( + "Honcho message sync hit an auth error; forcing token " + "refresh and retrying once: %s", e, + ) + if not self._force_reauth(): + raise + honcho_session.add_messages(honcho_messages) + self._clear_auth_failure() for msg in new_messages: msg["_synced"] = True logger.debug("Synced %d messages to Honcho for %s", len(honcho_messages), session.key) @@ -450,6 +551,8 @@ class HonchoSessionManager: self._cache[session.key] = session return True except Exception as e: + if _is_auth_error(e): + self._record_auth_failure(e) for msg in new_messages: msg["_synced"] = False logger.error("Failed to sync messages to Honcho: %s", e) @@ -638,6 +741,10 @@ class HonchoSessionManager: Returns: Honcho's synthesized answer, or empty string on failure. + + Raises: + HonchoAuthError: the backend rejected our credentials and a forced + token refresh plus one retry did not recover. """ session = self._cache.get(session_key) if not session: @@ -656,23 +763,43 @@ class HonchoSessionManager: else: level = self._default_reasoning_level() - try: + def _chat_once() -> str: if self._ai_observe_others: # AI peer can observe other peers — use assistant as observer. ai_peer_obj = self._get_or_create_peer(session.assistant_peer_id) if target_peer_id == session.assistant_peer_id: - result = ai_peer_obj.chat(query, reasoning_level=level) or "" - else: - result = ai_peer_obj.chat( - query, - target=target_peer_id, - reasoning_level=level, - ) or "" - else: - # Without cross-observation, each peer queries its own context. - target_peer = self._get_or_create_peer(target_peer_id) - result = target_peer.chat(query, reasoning_level=level) or "" + return ai_peer_obj.chat(query, reasoning_level=level) or "" + return ai_peer_obj.chat( + query, + target=target_peer_id, + reasoning_level=level, + ) or "" + # Without cross-observation, each peer queries its own context. + target_peer = self._get_or_create_peer(target_peer_id) + return target_peer.chat(query, reasoning_level=level) or "" + try: + try: + result = _chat_once() + except Exception as e: + if not _is_auth_error(e): + raise + logger.warning( + "Honcho dialectic query hit an auth error; forcing token " + "refresh and retrying once: %s", e, + ) + if not self._force_reauth(): + self._record_auth_failure(e) + raise HonchoAuthError(_auth_error_message(e)) from e + try: + result = _chat_once() + except Exception as retry_exc: + if _is_auth_error(retry_exc): + self._record_auth_failure(retry_exc) + raise HonchoAuthError(_auth_error_message(retry_exc)) from retry_exc + raise + + self._clear_auth_failure() # Only automatic injection uses the Hermes-side character cap. if ( apply_injection_cap @@ -682,6 +809,8 @@ class HonchoSessionManager: ): result = result[:self._dialectic_max_chars].rsplit(" ", 1)[0] + " …" return result + except HonchoAuthError: + raise except Exception as e: logger.warning("Honcho dialectic query failed: %s", e) return "" diff --git a/tests/honcho_plugin/test_auth_recovery.py b/tests/honcho_plugin/test_auth_recovery.py new file mode 100644 index 0000000000000..15a0e4dd4a396 --- /dev/null +++ b/tests/honcho_plugin/test_auth_recovery.py @@ -0,0 +1,465 @@ +"""Tests for OAuth 401 recovery: prompt exchange retry, invalid_grant handling, +forced refresh + single retry on sync and dialectic, backoff exemption, and +the one-time user-facing notice.""" + +import json +import logging +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from plugins.memory.honcho import HonchoMemoryProvider +from plugins.memory.honcho import oauth +from plugins.memory.honcho.client import HonchoClientConfig +from plugins.memory.honcho.session import ( + HonchoAuthError, + HonchoSession, + HonchoSessionManager, + _is_auth_error, +) + + +def _host_block(refresh="hch-rt-old", expires_at=100): + return { + "apiKey": "hch-at-old", + "oauth": { + "refreshToken": refresh, + "expiresAt": expires_at, + "clientId": "hermes-desktop", + "tokenEndpoint": "http://localhost:8000/oauth/token", + "scope": "write", + "tokenType": "Bearer", + }, + } + + +def _write(path: Path, raw: dict) -> None: + path.write_text(json.dumps(raw), encoding="utf-8") + + +def _rotated_body(n=1): + return { + "access_token": f"hch-at-new{n}", + "refresh_token": f"hch-rt-new{n}", + "expires_in": 3600, + "scope": "write", + "token_type": "Bearer", + } + + +# --------------------------------------------------------------------------- +# oauth: transient vs permanent exchange failures +# --------------------------------------------------------------------------- + + +class TestExchangeRetry: + def test_transient_failure_recovers_on_immediate_retry(self, tmp_path, monkeypatch): + """A timed-out exchange retries right away — the server honors the + replayed refresh token only within its rotation grace window.""" + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + calls = [] + + def flaky(url, data, timeout): + calls.append(data["refresh_token"]) + if len(calls) == 1: + raise TimeoutError("token exchange timed out") + return 200, _rotated_body() + + monkeypatch.setattr(oauth, "_http_post_form_status", flaky) + token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) + + assert token == "hch-at-new1" and refreshed is True + assert calls == ["hch-rt-old", "hch-rt-old"] + saved = json.loads(path.read_text())["hosts"]["hermes"] + assert saved["oauth"]["refreshToken"] == "hch-rt-new1" + + def test_invalid_grant_stops_retries_and_marks_reauth_required(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + calls = [] + + def revoked(url, data, timeout): + calls.append(1) + return 400, {"error": "invalid_grant", "error_description": "grant revoked"} + + monkeypatch.setattr(oauth, "_http_post_form_status", revoked) + token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) + + # Fail-open return, but no retry of a permanently rejected grant. + assert token == "hch-at-old" and refreshed is False + assert len(calls) == 1 + assert oauth.reauth_required(path, "hermes") is True + + # Later refresh attempts skip the endpoint entirely. + token2, refreshed2 = oauth.ensure_fresh_token(path, "hermes", now=2000) + assert token2 == "hch-at-old" and refreshed2 is False + assert len(calls) == 1 + + # The forced (post-401) path refuses a dead grant too. + assert oauth.force_refresh_token(path, "hermes") is None + assert len(calls) == 1 + + def test_relogin_clears_the_dead_grant(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + monkeypatch.setattr( + oauth, "_http_post_form_status", + lambda *a, **k: (400, {"error": "invalid_grant"}), + ) + oauth.ensure_fresh_token(path, "hermes", now=1000) + assert oauth.reauth_required(path, "hermes") is True + + oauth.install_grant( + path, "hermes", + {"access_token": "hch-at-fresh", "refresh_token": "hch-rt-fresh", "expires_in": 3600}, + client_id="hermes-desktop", + token_endpoint="http://localhost:8000/oauth/token", + now=2000, + ) + assert oauth.reauth_required(path, "hermes") is False + token, _ = oauth.ensure_fresh_token(path, "hermes", now=2000) + assert token == "hch-at-fresh" + + def test_error_body_is_logged(self, tmp_path, monkeypatch, caplog): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + monkeypatch.setattr( + oauth, "_http_post_form_status", + lambda *a, **k: (400, {"error": "invalid_grant", "error_description": "grant revoked"}), + ) + with caplog.at_level(logging.WARNING, logger="plugins.memory.honcho.oauth"): + oauth.ensure_fresh_token(path, "hermes", now=1000) + assert "invalid_grant" in caplog.text + assert "grant revoked" in caplog.text + + def test_redaction_strips_token_values(self): + redacted = oauth._redact_tokens( + "exchange failed for hch-rt-supersecret123 got hch-at-alsosecret456" + ) + assert "supersecret123" not in redacted + assert "alsosecret456" not in redacted + assert "hch-rt-[redacted]" in redacted + assert "hch-at-[redacted]" in redacted + + +class TestForceRefreshToken: + def test_rotates_despite_local_validity(self, tmp_path, monkeypatch): + """A server-side 401 forces a rotation even when the local clock says + the token is still live.""" + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block(expires_at=time.time() + 3600)}}) + monkeypatch.setattr( + oauth, "_http_post_form_status", lambda *a, **k: (200, _rotated_body()) + ) + token = oauth.force_refresh_token(path, "hermes") + assert token == "hch-at-new1" + saved = json.loads(path.read_text())["hosts"]["hermes"] + assert saved["apiKey"] == "hch-at-new1" + + def test_adopts_concurrent_rotation_without_exchange(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + far = time.time() + 7200 + _write(path, {"hosts": {"hermes": _host_block(expires_at=far)}}) + # Seed the expiry cache with the old token. + oauth.ensure_fresh_token(path, "hermes") + # Another process rotated the credential on disk. + rotated = _host_block(refresh="hch-rt-2", expires_at=far) + rotated["apiKey"] = "hch-at-2" + _write(path, {"hosts": {"hermes": rotated}}) + monkeypatch.setattr( + oauth, "_http_post_form_status", + lambda *a, **k: pytest.fail("must adopt the on-disk rotation, not exchange"), + ) + assert oauth.force_refresh_token(path, "hermes") == "hch-at-2" + + def test_transient_failure_returns_none(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block(expires_at=time.time() + 3600)}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + def boom(*a, **k): + raise ConnectionError("network down") + + monkeypatch.setattr(oauth, "_http_post_form_status", boom) + assert oauth.force_refresh_token(path, "hermes") is None + # Not permanent: a later attempt may exchange again. + assert oauth.reauth_required(path, "hermes") is False + + def test_static_api_key_is_noop(self, tmp_path): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": {"apiKey": "hch-v3-static"}}}) + assert oauth.force_refresh_token(path, "hermes") is None + + +# --------------------------------------------------------------------------- +# session: auth error detection +# --------------------------------------------------------------------------- + + +class TestAuthErrorDetection: + def test_matches_honcho_token_message(self): + assert _is_auth_error(Exception("Invalid or expired access token")) + + def test_matches_status_code_attr(self): + exc = Exception("boom") + exc.status_code = 401 + assert _is_auth_error(exc) + + def test_matches_401_text(self): + assert _is_auth_error(Exception("HTTP 401 Unauthorized")) + + def test_ignores_other_errors(self): + assert not _is_auth_error(Exception("connection reset by peer")) + assert not _is_auth_error(Exception("HTTP 500 internal error")) + + +# --------------------------------------------------------------------------- +# session: dialectic 401 recovery +# --------------------------------------------------------------------------- + + +class _FlakyPeer: + """chat() raises an auth error N times, then succeeds.""" + + def __init__(self, failures: int, result: str = "synthesized answer"): + self.failures = failures + self.result = result + self.calls = 0 + + def chat(self, query, **kw): + self.calls += 1 + if self.calls <= self.failures: + raise Exception("Invalid or expired access token") + return self.result + + +def _make_manager(peer, *, reauth_ok=True): + cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True) + mgr = HonchoSessionManager(config=cfg) + session = HonchoSession( + key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s" + ) + mgr._cache["k"] = session + mgr._get_or_create_peer = lambda peer_id: peer + mgr._force_reauth = lambda: reauth_ok + return mgr + + +class TestDialecticAuthRetry: + def test_401_forces_refresh_and_retries_once(self): + peer = _FlakyPeer(failures=1) + mgr = _make_manager(peer) + assert mgr.dialectic_query("k", "who is this user?") == "synthesized answer" + assert peer.calls == 2 # original + one retry + + def test_persistent_401_raises_auth_error(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert peer.calls == 2 # exactly one retry, no loop + + def test_failed_reauth_raises_without_retry(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer, reauth_ok=False) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert peer.calls == 1 # no retry without a fresh token + + def test_non_auth_errors_stay_fail_open(self): + class _BrokenPeer: + def chat(self, *a, **kw): + raise Exception("connection reset by peer") + + mgr = _make_manager(_BrokenPeer()) + assert mgr.dialectic_query("k", "q") == "" + + def test_success_after_failure_clears_auth_state(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer, reauth_ok=False) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert mgr._auth_failure is not None + + peer.failures = 0 + mgr._force_reauth = lambda: True + assert mgr.dialectic_query("k", "q") == "synthesized answer" + assert mgr._auth_failure is None + assert mgr.pop_auth_notice() is None + + +class TestForceReauth: + def test_rotates_and_applies_to_live_client(self, tmp_path, monkeypatch): + from plugins.memory.honcho import client as client_mod + from plugins.memory.honcho import session as session_mod + + fake_client = object() + applied = {} + monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: fake_client) + monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json") + monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h: "hch-at-new") + + def apply(client, token): + applied["client"] = client + applied["token"] = token + return True + + monkeypatch.setattr(oauth, "apply_token_to_client", apply) + + mgr = HonchoSessionManager(config=HonchoClientConfig(host="hermes")) + assert mgr._force_reauth() is True + assert applied == {"client": fake_client, "token": "hch-at-new"} + + def test_returns_false_when_refresh_yields_nothing(self, tmp_path, monkeypatch): + from plugins.memory.honcho import client as client_mod + + monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json") + monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h: None) + mgr = HonchoSessionManager(config=HonchoClientConfig(host="hermes")) + assert mgr._force_reauth() is False + + +# --------------------------------------------------------------------------- +# session: message sync 401 recovery +# --------------------------------------------------------------------------- + + +class _FlakyHonchoSession: + """add_messages() raises an auth error N times, then succeeds.""" + + def __init__(self, failures: int): + self.failures = failures + self.calls = 0 + + def add_messages(self, messages): + self.calls += 1 + if self.calls <= self.failures: + raise Exception("Invalid or expired access token") + + +def _make_sync_manager(flaky_session, *, reauth_ok=True): + cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True) + mgr = HonchoSessionManager(config=cfg) + peer = MagicMock() + peer.message.side_effect = lambda content: content + mgr._get_or_create_peer = lambda peer_id: peer + mgr._sessions_cache["s"] = flaky_session + mgr._force_reauth = lambda: reauth_ok + session = HonchoSession( + key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s" + ) + session.add_message("user", "hello") + session.add_message("assistant", "hi") + return mgr, session + + +class TestSyncAuthRetry: + def test_401_forces_refresh_and_retries_once(self): + flaky = _FlakyHonchoSession(failures=1) + mgr, session = _make_sync_manager(flaky) + assert mgr._flush_session(session) is True + assert flaky.calls == 2 + assert all(m["_synced"] for m in session.messages) + + def test_persistent_401_fails_and_records_auth_failure(self): + flaky = _FlakyHonchoSession(failures=99) + mgr, session = _make_sync_manager(flaky) + assert mgr._flush_session(session) is False + assert flaky.calls == 2 # exactly one retry, no loop + assert not any(m.get("_synced") for m in session.messages) + assert mgr._auth_failure is not None + + def test_failed_reauth_fails_without_retry(self): + flaky = _FlakyHonchoSession(failures=99) + mgr, session = _make_sync_manager(flaky, reauth_ok=False) + assert mgr._flush_session(session) is False + assert flaky.calls == 1 + assert mgr._auth_failure is not None + + def test_later_success_recovers_and_clears_auth_state(self): + flaky = _FlakyHonchoSession(failures=2) + mgr, session = _make_sync_manager(flaky, reauth_ok=False) + assert mgr._flush_session(session) is False + assert mgr._auth_failure is not None + + mgr._force_reauth = lambda: True + assert mgr._flush_session(session) is True + assert all(m["_synced"] for m in session.messages) + assert mgr._auth_failure is None + + +# --------------------------------------------------------------------------- +# one-time user-facing notice +# --------------------------------------------------------------------------- + + +class TestAuthNotice: + def test_manager_emits_notice_exactly_once(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer, reauth_ok=False) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + + first = mgr.pop_auth_notice() + assert first and "Invalid or expired access token" in first + assert mgr.pop_auth_notice() is None + + # A second failure inside the same episode does not re-arm the notice. + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert mgr.pop_auth_notice() is None + + def test_provider_prefetch_injects_notice_once(self): + class _FakeManager: + def __init__(self): + self.notices = ["Invalid or expired access token"] + + def pop_auth_notice(self): + return self.notices.pop() if self.notices else None + + def pop_context_result(self, session_key): + return {} + + provider = HonchoMemoryProvider() + provider._manager = _FakeManager() + provider._config = SimpleNamespace(timeout=0.01, context_tokens=0) + provider._session_key = "k" + provider._session_initialized = True + provider._recall_mode = "context" + provider._turn_count = 2 + provider._last_dialectic_turn = 0 + provider._base_context_cache = "" + + first = provider.prefetch("what did we decide about the schema?") + assert "hermes honcho setup" in first + assert "paused" in first + + second = provider.prefetch("and the follow-up question?") + assert second == "" + + +# --------------------------------------------------------------------------- +# cadence backoff exemption +# --------------------------------------------------------------------------- + + +class TestBackoffExemption: + def test_auth_error_does_not_widen_backoff(self): + provider = HonchoMemoryProvider() + provider._note_dialectic_failure(HonchoAuthError("still 401 after refresh")) + assert provider._dialectic_empty_streak == 0 + + def test_other_errors_still_widen_backoff(self): + provider = HonchoMemoryProvider() + provider._note_dialectic_failure(RuntimeError("timeout")) + assert provider._dialectic_empty_streak == 1 diff --git a/tests/honcho_plugin/test_oauth.py b/tests/honcho_plugin/test_oauth.py index 58ca9d0b5ba65..bdaebe3ab02a6 100644 --- a/tests/honcho_plugin/test_oauth.py +++ b/tests/honcho_plugin/test_oauth.py @@ -69,7 +69,7 @@ class TestEnsureFreshToken: path = tmp_path / "honcho.json" _write(path, {"hosts": {"hermes": _host_block(expires_at=10_000)}}) monkeypatch.setattr( - oauth, "_http_post_form", + oauth, "_http_post_form_status", lambda *a, **k: pytest.fail("refresh must not be called when fresh"), ) token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=0) @@ -84,7 +84,7 @@ class TestEnsureFreshToken: assert data["grant_type"] == "refresh_token" assert data["refresh_token"] == "hch-rt-old" assert data["client_id"] == "hermes-desktop" - return { + return 200, { "access_token": "hch-at-new", "refresh_token": "hch-rt-new", "expires_in": 3600, @@ -92,7 +92,7 @@ class TestEnsureFreshToken: "token_type": "Bearer", } - monkeypatch.setattr(oauth, "_http_post_form", fake_post) + monkeypatch.setattr(oauth, "_http_post_form_status", fake_post) token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) assert token == "hch-at-new" and refreshed is True @@ -105,14 +105,20 @@ class TestEnsureFreshToken: def test_refresh_failure_fails_open(self, tmp_path, monkeypatch): path = tmp_path / "honcho.json" _write(path, {"hosts": {"hermes": _host_block(expires_at=100)}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + calls = [] def boom(*a, **k): + calls.append(a) raise RuntimeError("network down") - monkeypatch.setattr(oauth, "_http_post_form", boom) + monkeypatch.setattr(oauth, "_http_post_form_status", boom) token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) - # Stale token returned, no crash, file untouched. + # Stale token returned, no crash, file untouched. Transient failures + # retry exactly once, then fail open. assert token == "hch-at-old" and refreshed is False + assert len(calls) == 2 assert json.loads(path.read_text())["hosts"]["hermes"]["apiKey"] == "hch-at-old" def test_double_check_uses_disk_when_already_rotated(self, tmp_path, monkeypatch): @@ -123,7 +129,7 @@ class TestEnsureFreshToken: stale_raw = {"hosts": {"hermes": _host_block(refresh="hch-rt-old", expires_at=100)}} stale_raw["hosts"]["hermes"]["apiKey"] = "hch-at-stale" monkeypatch.setattr( - oauth, "_http_post_form", + oauth, "_http_post_form_status", lambda *a, **k: pytest.fail("must not refresh; disk token is fresh"), ) token, refreshed = oauth.ensure_fresh_token(path, "hermes", stale_raw, now=1000)