From 6ea01262fc4223879855b8ececb7887be5b86f26 Mon Sep 17 00:00:00 2001 From: Erosika Date: Thu, 6 Aug 2026 13:46:23 -0400 Subject: [PATCH] fix(honcho): recover memory from mid-session oauth 401s and tell the user once An expired access token could pause Honcho memory for hours with no user-facing signal: ensure_fresh_token swallowed every exchange failure and returned the stale token, no code handled a 401 from the Honcho API, and each failed dialectic cycle widened the cadence backoff. Hypothesis for the trigger (not confirmed): the refresh POST times out after the server already rotated the token pair, Hermes keeps the old refresh token, and the eventual replay lands outside the server's 60-second rotation grace window, which revokes the whole grant. - oauth: the exchange reads the token endpoint's error body instead of discarding it. invalid_grant and other permanent OAuth errors mark the grant dead so no code retries a revoked grant; transient failures retry once immediately, which keeps a replayed refresh token inside the grace window. Log lines redact token values. - oauth: force_refresh_token() rotates the token now, ignoring local expiry, to recover from a server-side 401. - session: dialectic_query and _flush_session treat a 401 as a trigger to force one token rotation and retry the call exactly once. A persistent auth failure raises HonchoAuthError (dialectic) or records the failure (sync) instead of being returned as an empty result. - provider: injects a one-time notice into the memory context so the model tells the user memory is paused and 'hermes honcho setup' restores it. Auth failures no longer widen the dialectic cadence backoff. New tests cover the exchange retry, invalid_grant terminality plus re-login recovery, forced refresh, 401 retry on both the sync and dialectic paths, the one-time notice, and the backoff exemption. --- plugins/memory/honcho/__init__.py | 46 ++- plugins/memory/honcho/client.py | 5 +- plugins/memory/honcho/oauth.py | 168 +++++++- plugins/memory/honcho/session.py | 155 +++++++- tests/honcho_plugin/test_auth_recovery.py | 465 ++++++++++++++++++++++ tests/honcho_plugin/test_oauth.py | 18 +- 6 files changed, 826 insertions(+), 31 deletions(-) create mode 100644 tests/honcho_plugin/test_auth_recovery.py diff --git a/plugins/memory/honcho/__init__.py b/plugins/memory/honcho/__init__.py index 78d24e90b3d9a..9246f26b6e0b5 100644 --- a/plugins/memory/honcho/__init__.py +++ b/plugins/memory/honcho/__init__.py @@ -522,7 +522,7 @@ class HonchoMemoryProvider(MemoryProvider): ) except Exception as exc: logger.debug("Honcho dialectic prewarm failed: %s", exc) - self._dialectic_empty_streak += 1 + self._note_dialectic_failure(exc) return if r and r.strip(): with self._prefetch_lock: @@ -715,6 +715,11 @@ class HonchoMemoryProvider(MemoryProvider): parts = [] + # One-time notice, relayed by the model, that auth is dead and memory is paused. + auth_notice = self._pop_auth_notice() + if auth_notice: + parts.append(auth_notice) + # ----- Layer 1: Base context (representation + card) ----- if not _skip_base: # The first base fetch gets the remaining turn-1 budget. Later @@ -804,7 +809,7 @@ class HonchoMemoryProvider(MemoryProvider): r = self._run_dialectic_depth(query) except Exception as exc: logger.debug("Honcho first-turn dialectic failed: %s", exc) - self._dialectic_empty_streak += 1 + self._note_dialectic_failure(exc) return if r and r.strip(): with self._prefetch_lock: @@ -845,6 +850,25 @@ class HonchoMemoryProvider(MemoryProvider): return result + def _pop_auth_notice(self) -> str: + """One-time model-facing notice that Honcho auth expired and memory is paused.""" + pop = getattr(self._manager, "pop_auth_notice", None) + if not callable(pop): + return "" + try: + msg = pop() + except Exception: + return "" + if not isinstance(msg, str) or not msg: + return "" + return ( + "[Honcho memory status] Authentication with the Honcho memory backend " + "has expired and automatic token refresh failed, so memory sync and " + f"recall are paused. Reason: {msg}\n" + "Tell the user (once) that Honcho memory is paused and that running " + "'hermes honcho setup' to re-authenticate will restore it." + ) + def _consume_pending_dialectic(self) -> str: """Pop any pending dialectic result, applying the stale-discard guard. @@ -941,7 +965,7 @@ class HonchoMemoryProvider(MemoryProvider): result = self._run_dialectic_depth(query) except Exception as e: logger.debug("Honcho prefetch failed: %s", e) - self._dialectic_empty_streak += 1 + self._note_dialectic_failure(e) return if result and result.strip(): with self._prefetch_lock: @@ -1020,6 +1044,22 @@ class HonchoMemoryProvider(MemoryProvider): ceiling = self._dialectic_cadence * self._BACKOFF_MAX return min(widened, ceiling) + def _note_dialectic_failure(self, exc: BaseException) -> None: + """Widen the empty-streak backoff after a failed dialectic cycle. + + Auth failures are exempt: waiting cannot fix a dead token, and backoff + would delay recovery after the user re-authenticates. + """ + from plugins.memory.honcho.session import HonchoAuthError + + if isinstance(exc, HonchoAuthError): + logger.warning( + "Honcho dialectic auth failure (not counted toward cadence backoff): %s", + exc, + ) + return + self._dialectic_empty_streak += 1 + def liveness_snapshot(self) -> dict: """In-process snapshot of dialectic liveness state for diagnostics. diff --git a/plugins/memory/honcho/client.py b/plugins/memory/honcho/client.py index 1945a38a15fe7..fd2e593faf102 100644 --- a/plugins/memory/honcho/client.py +++ b/plugins/memory/honcho/client.py @@ -934,8 +934,9 @@ def _resolve_timeout_from_sources(config: HonchoClientConfig | None) -> float: def _apply_fresh_oauth_token(config: HonchoClientConfig) -> None: """Refresh a near-expiry OAuth grant and point ``config.api_key`` at it. - No-op for static API keys or when refresh fails (fail-open: the stale token - is left in place and the existing 401 handling degrades gracefully). + No-op for static API keys or when refresh fails: the stale token stays in + place and the first rejected call triggers the post-401 recovery in + session.py (forced rotation, one retry). """ try: from plugins.memory.honcho import oauth diff --git a/plugins/memory/honcho/oauth.py b/plugins/memory/honcho/oauth.py index 3d7cbbd02c82f..cdd31daf669bf 100644 --- a/plugins/memory/honcho/oauth.py +++ b/plugins/memory/honcho/oauth.py @@ -6,15 +6,22 @@ expiry to keep it live. Refresh tokens rotate with single-use reuse detection: a replayed stale token revokes the whole grant. So every refresh must persist the rotated token -atomically and be serialized — and a failed refresh never raises into the -agent (stale token stays; the fail-open path absorbs the eventual 401). +atomically and be serialized. A failed exchange never raises into the agent: +transient failures retry once immediately (the server re-rotates a replayed +refresh token only within a short grace window, so waiting for the next +memory call is too late), and a permanent OAuth error such as invalid_grant +marks the grant dead so nothing keeps hitting the token endpoint — callers +surface a re-login prompt instead. A server-side 401 on a locally-valid +token is recovered via ``force_refresh_token``. """ from __future__ import annotations +import hashlib import json import logging import os +import re import threading import time from contextlib import contextmanager @@ -35,6 +42,29 @@ _REFRESH_SKEW_SECONDS = 120 # on the path to a memory call, and a stalled auth server must not hang it. _REFRESH_TIMEOUT_SECONDS = 15.0 +# Retry pause, kept short: the server honors a replayed refresh token only briefly after rotating it. +_REFRESH_RETRY_DELAY_SECONDS = 2.0 + +# OAuth error codes that a retry can never fix — the grant itself is dead. +_PERMANENT_OAUTH_ERRORS = frozenset({"invalid_grant", "invalid_client", "unauthorized_client"}) + +# Token values are secret even though their prefixes are not; redact before logging. +_TOKEN_VALUE_RE = re.compile(r"hch-(at|rt)-[A-Za-z0-9._~+/=-]+") + + +def _redact_tokens(text: str) -> str: + """Replace any embedded token values with their prefix plus a placeholder.""" + return _TOKEN_VALUE_RE.sub(lambda m: f"hch-{m.group(1)}-[redacted]", text) + + +class OAuthRefreshError(Exception): + """Token endpoint rejected the refresh. ``permanent`` means re-login is required.""" + + def __init__(self, message: str, *, error: str = "", permanent: bool = False): + super().__init__(message) + self.error = error + self.permanent = permanent + # Serializes refresh across threads sharing one process's config. Re-checked # under the lock (double-checked) so racing callers don't replay a rotated # refresh token and trip reuse detection. @@ -99,6 +129,31 @@ def _config_refresh_lock(path: Path): # until the token nears expiry and disk is read again. _expiry_cache: dict[tuple[str, str], tuple[float, str]] = {} +# Permanently rejected grants: (config path, host) → sha256 of the dead refresh token; a re-login rotates the token, so the digest check self-clears. +_dead_grants: dict[tuple[str, str], str] = {} + + +def _refresh_token_digest(cred: OAuthCredential) -> str: + return hashlib.sha256(cred.refresh_token.encode("utf-8")).hexdigest() + + +def _grant_is_dead(key: tuple[str, str], cred: OAuthCredential) -> bool: + return _dead_grants.get(key) == _refresh_token_digest(cred) + + +def _mark_grant_dead(key: tuple[str, str], cred: OAuthCredential) -> None: + _dead_grants[key] = _refresh_token_digest(cred) + + +def reauth_required(path: Path, host: str) -> bool: + """True when ``host``'s OAuth grant is dead and only a new login fixes it.""" + key = (str(path), host) + if key not in _dead_grants: + return False + block = (_read_config(path).get("hosts") or {}).get(host) or {} + cred = OAuthCredential.from_host_block(block) + return cred is not None and _grant_is_dead(key, cred) + def is_oauth_access_token(value: str | None) -> bool: """True when ``value`` is an OAuth access token (vs a static API key).""" @@ -208,9 +263,10 @@ def _http_get_json(url: str, timeout: float) -> dict[str, Any]: def _exchange_refresh_token(cred: OAuthCredential, *, now: float) -> OAuthCredential: """Run the refresh_token grant and return the rotated credential. - Raises on any transport/protocol failure; callers fail open. + Raises ``OAuthRefreshError`` (with the endpoint's error body) on an error + response, transport errors as-is; callers fail open. """ - body = _http_post_form( + status, body = _http_post_form_status( cred.token_endpoint, { "grant_type": "refresh_token", @@ -219,6 +275,15 @@ def _exchange_refresh_token(cred: OAuthCredential, *, now: float) -> OAuthCreden }, _REFRESH_TIMEOUT_SECONDS, ) + if status >= 400: + error = str(body.get("error") or "") + description = str(body.get("error_description") or "") + detail = " — ".join(p for p in (error, description) if p) or "no error body" + raise OAuthRefreshError( + _redact_tokens(f"token endpoint returned HTTP {status}: {detail}"), + error=error, + permanent=error in _PERMANENT_OAUTH_ERRORS, + ) access = body.get("access_token") refresh = body.get("refresh_token") if not is_oauth_access_token(access) or not refresh: @@ -238,6 +303,29 @@ def _exchange_refresh_token(cred: OAuthCredential, *, now: float) -> OAuthCreden ) +def _exchange_with_retry(cred: OAuthCredential, *, now: float) -> OAuthCredential: + """Exchange the refresh token, retrying once immediately on transient failure. + + The server honors a replayed refresh token only within a short grace + window after rotating it, so the retry must happen now — deferring to the + next memory call lands outside the window and revokes the grant. + """ + try: + return _exchange_refresh_token(cred, now=now) + except OAuthRefreshError as exc: + if exc.permanent: + raise + first: Exception = exc + except Exception as exc: + first = exc + logger.warning( + "Honcho OAuth token exchange failed, retrying once: %s", + _redact_tokens(str(first)), + ) + time.sleep(_REFRESH_RETRY_DELAY_SECONDS) + return _exchange_refresh_token(cred, now=now) + + def _read_config(path: Path) -> dict[str, Any]: try: return json.loads(path.read_text(encoding="utf-8")) @@ -279,6 +367,7 @@ def _persist_credential(path: Path, host: str, cred: OAuthCredential) -> None: block["oauth"] = cred.oauth_block() _atomic_write_config(path, raw) _expiry_cache[(str(path), host)] = (cred.expires_at, cred.access_token) + _dead_grants.pop((str(path), host), None) def ensure_fresh_token( @@ -293,7 +382,9 @@ def ensure_fresh_token( Returns ``(None, False)`` when the host has no OAuth credential (e.g. a plain API key) so callers leave the existing token untouched. Refresh failures are swallowed: the current (possibly stale) token is returned with - ``refreshed=False`` and the fail-open path handles any resulting 401. + ``refreshed=False``, transient failures retry once immediately, and a + permanently rejected grant is marked dead so later calls skip the endpoint. + The 401 recovery in session.py escalates dead grants to the user. """ now = time.time() if now is None else now key = (str(path), host) @@ -323,16 +414,78 @@ def ensure_fresh_token( current = OAuthCredential.from_host_block(fresh_block) or cred if not current.is_expired(now=now): return current.access_token, current.access_token != cred.access_token + if _grant_is_dead(key, current): + return current.access_token, False try: - rotated = _exchange_refresh_token(current, now=now) + rotated = _exchange_with_retry(current, now=now) + except OAuthRefreshError as exc: + if exc.permanent: + _mark_grant_dead(key, current) + logger.error( + "Honcho OAuth grant for host %s is no longer valid (%s); " + "run 'hermes honcho setup' to re-authenticate", host, exc, + ) + else: + logger.warning("Honcho OAuth refresh failed for host %s: %s", host, exc) + return current.access_token, False except Exception as exc: - logger.warning("Honcho OAuth refresh failed for host %s: %s", host, exc) + logger.warning( + "Honcho OAuth refresh failed for host %s: %s", + host, _redact_tokens(str(exc)), + ) return current.access_token, False _persist_credential(path, host, rotated) logger.info("Honcho OAuth token refreshed for host %s", host) return rotated.access_token, True +def force_refresh_token(path: Path, host: str) -> str | None: + """Rotate ``host``'s access token now, ignoring local expiry math. + + Recovery path for a server-side 401 on a token the local clock still + considers valid (clock skew, server-side rotation, revocation). Returns + the new access token, or ``None`` when the host has no OAuth credential, + the grant is dead, or the exchange fails — callers surface the original + auth error in that case. + """ + now = time.time() + key = (str(path), host) + with _refresh_lock, _config_refresh_lock(path): + block = (_read_config(path).get("hosts") or {}).get(host) or {} + cred = OAuthCredential.from_host_block(block) + if cred is None: + _expiry_cache.pop(key, None) + return None + if _grant_is_dead(key, cred): + return None + cached = _expiry_cache.get(key) + # Another thread or process already rotated: adopt the newer on-disk token. + if cached is not None and cred.access_token != cached[1] and not cred.is_expired(now=now): + _expiry_cache[key] = (cred.expires_at, cred.access_token) + return cred.access_token + try: + rotated = _exchange_with_retry(cred, now=now) + except OAuthRefreshError as exc: + if exc.permanent: + _mark_grant_dead(key, cred) + logger.error( + "Honcho OAuth grant for host %s is no longer valid (%s); " + "run 'hermes honcho setup' to re-authenticate", host, exc, + ) + else: + logger.warning("Honcho OAuth forced refresh failed for host %s: %s", host, exc) + return None + except Exception as exc: + logger.warning( + "Honcho OAuth forced refresh failed for host %s: %s", + host, _redact_tokens(str(exc)), + ) + return None + _persist_credential(path, host, rotated) + logger.info("Honcho OAuth token force-refreshed for host %s after an auth failure", host) + return rotated.access_token + + def install_grant( path: Path, host: str, @@ -378,6 +531,7 @@ def install_grant( if apply_config: _deep_merge(raw, granted_config) _expiry_cache[(str(path), host)] = (cred.expires_at, cred.access_token) + _dead_grants.pop((str(path), host), None) hosts = raw.setdefault("hosts", {}) block = hosts.setdefault(host, {}) block["apiKey"] = cred.access_token diff --git a/plugins/memory/honcho/session.py b/plugins/memory/honcho/session.py index d4a53a6e42436..76cf4e94598b4 100644 --- a/plugins/memory/honcho/session.py +++ b/plugins/memory/honcho/session.py @@ -24,6 +24,38 @@ _PEER_ID_HASH_LEN = 8 _PEER_ID_HASH_ESCALATION_LENGTHS = (_PEER_ID_HASH_LEN, 12, 16, 24, 32, 64) +class HonchoAuthError(RuntimeError): + """Auth failure that survived a forced token refresh and one retry. + + Raised instead of swallowed so callers can tell a rejected credential + apart from an empty result; cadence backoff must not count it as empty. + """ + + +# The SDK raises different exception classes per transport, but an auth failure always carries one of these markers. +_AUTH_ERROR_MARKERS = ( + "invalid or expired access token", + "authentication", + "unauthorized", + "401", +) + + +def _is_auth_error(exc: BaseException) -> bool: + status = getattr(exc, "status_code", None) or getattr(exc, "status", None) + if status == 401: + return True + text = str(exc).lower() + return any(marker in text for marker in _AUTH_ERROR_MARKERS) + + +def _auth_error_message(exc: BaseException) -> str: + return ( + f"Honcho rejected our credentials and a forced token refresh did not recover: {exc}. " + "Re-authenticate with 'hermes honcho setup'." + ) + + @dataclass class HonchoSession: """ @@ -105,6 +137,10 @@ class HonchoSessionManager: self._peers_cache: dict[str, Any] = {} self._sessions_cache: dict[str, Any] = {} + # Set when a call still fails auth after a forced token refresh; cleared on the next success. + self._auth_failure: str | None = None + self._auth_notice_emitted = False + # Write frequency state write_frequency = (config.write_frequency if config else "async") self._write_frequency = write_frequency @@ -161,6 +197,59 @@ class HonchoSessionManager: self._honcho = get_honcho_client() return self._honcho + def _record_auth_failure(self, exc: BaseException) -> None: + if self._auth_failure is None: + logger.error( + "Honcho authentication failed and token refresh did not recover; " + "memory sync and recall are paused until the user re-authenticates: %s", + exc, + ) + self._auth_failure = str(exc) + + def _clear_auth_failure(self) -> None: + if self._auth_failure is not None: + logger.info("Honcho authentication recovered; memory sync and recall resumed") + self._auth_failure = None + self._auth_notice_emitted = False + + def pop_auth_notice(self) -> str | None: + """Return the pending auth-failure message once; later calls return None.""" + if self._auth_failure is None or self._auth_notice_emitted: + return None + self._auth_notice_emitted = True + return self._auth_failure + + def _force_reauth(self) -> bool: + """Rotate the OAuth token after a server-side 401 and rebind the client. + + Returns True when a fresh token was obtained and applied. False for + static API keys (no OAuth credential), a dead grant, or a failed + exchange — callers surface the original auth error in that case. + """ + try: + from plugins.memory.honcho import oauth + from plugins.memory.honcho.client import ( + reset_honcho_client, + resolve_config_path, + ) + + host = getattr(self._config, "host", "") or "" + if not host: + return False + token = oauth.force_refresh_token(resolve_config_path(), host) + if not token: + return False + if not oauth.apply_token_to_client(self.honcho, token): + # SDK shape changed: rebuild the client and drop objects holding the old transport. + reset_honcho_client() + with self._cache_lock: + self._peers_cache.clear() + self._sessions_cache.clear() + return True + except Exception: + logger.warning("Honcho post-401 token refresh failed", exc_info=True) + return False + def _get_or_create_peer(self, peer_id: str) -> Any: """ Get or create a Honcho peer. @@ -442,7 +531,19 @@ class HonchoSessionManager: honcho_messages.append(peer.message(msg["content"])) try: - honcho_session.add_messages(honcho_messages) + try: + honcho_session.add_messages(honcho_messages) + except Exception as e: + if not _is_auth_error(e): + raise + logger.warning( + "Honcho message sync hit an auth error; forcing token " + "refresh and retrying once: %s", e, + ) + if not self._force_reauth(): + raise + honcho_session.add_messages(honcho_messages) + self._clear_auth_failure() for msg in new_messages: msg["_synced"] = True logger.debug("Synced %d messages to Honcho for %s", len(honcho_messages), session.key) @@ -450,6 +551,8 @@ class HonchoSessionManager: self._cache[session.key] = session return True except Exception as e: + if _is_auth_error(e): + self._record_auth_failure(e) for msg in new_messages: msg["_synced"] = False logger.error("Failed to sync messages to Honcho: %s", e) @@ -638,6 +741,10 @@ class HonchoSessionManager: Returns: Honcho's synthesized answer, or empty string on failure. + + Raises: + HonchoAuthError: the backend rejected our credentials and a forced + token refresh plus one retry did not recover. """ session = self._cache.get(session_key) if not session: @@ -656,23 +763,43 @@ class HonchoSessionManager: else: level = self._default_reasoning_level() - try: + def _chat_once() -> str: if self._ai_observe_others: # AI peer can observe other peers — use assistant as observer. ai_peer_obj = self._get_or_create_peer(session.assistant_peer_id) if target_peer_id == session.assistant_peer_id: - result = ai_peer_obj.chat(query, reasoning_level=level) or "" - else: - result = ai_peer_obj.chat( - query, - target=target_peer_id, - reasoning_level=level, - ) or "" - else: - # Without cross-observation, each peer queries its own context. - target_peer = self._get_or_create_peer(target_peer_id) - result = target_peer.chat(query, reasoning_level=level) or "" + return ai_peer_obj.chat(query, reasoning_level=level) or "" + return ai_peer_obj.chat( + query, + target=target_peer_id, + reasoning_level=level, + ) or "" + # Without cross-observation, each peer queries its own context. + target_peer = self._get_or_create_peer(target_peer_id) + return target_peer.chat(query, reasoning_level=level) or "" + try: + try: + result = _chat_once() + except Exception as e: + if not _is_auth_error(e): + raise + logger.warning( + "Honcho dialectic query hit an auth error; forcing token " + "refresh and retrying once: %s", e, + ) + if not self._force_reauth(): + self._record_auth_failure(e) + raise HonchoAuthError(_auth_error_message(e)) from e + try: + result = _chat_once() + except Exception as retry_exc: + if _is_auth_error(retry_exc): + self._record_auth_failure(retry_exc) + raise HonchoAuthError(_auth_error_message(retry_exc)) from retry_exc + raise + + self._clear_auth_failure() # Only automatic injection uses the Hermes-side character cap. if ( apply_injection_cap @@ -682,6 +809,8 @@ class HonchoSessionManager: ): result = result[:self._dialectic_max_chars].rsplit(" ", 1)[0] + " …" return result + except HonchoAuthError: + raise except Exception as e: logger.warning("Honcho dialectic query failed: %s", e) return "" diff --git a/tests/honcho_plugin/test_auth_recovery.py b/tests/honcho_plugin/test_auth_recovery.py new file mode 100644 index 0000000000000..15a0e4dd4a396 --- /dev/null +++ b/tests/honcho_plugin/test_auth_recovery.py @@ -0,0 +1,465 @@ +"""Tests for OAuth 401 recovery: prompt exchange retry, invalid_grant handling, +forced refresh + single retry on sync and dialectic, backoff exemption, and +the one-time user-facing notice.""" + +import json +import logging +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from plugins.memory.honcho import HonchoMemoryProvider +from plugins.memory.honcho import oauth +from plugins.memory.honcho.client import HonchoClientConfig +from plugins.memory.honcho.session import ( + HonchoAuthError, + HonchoSession, + HonchoSessionManager, + _is_auth_error, +) + + +def _host_block(refresh="hch-rt-old", expires_at=100): + return { + "apiKey": "hch-at-old", + "oauth": { + "refreshToken": refresh, + "expiresAt": expires_at, + "clientId": "hermes-desktop", + "tokenEndpoint": "http://localhost:8000/oauth/token", + "scope": "write", + "tokenType": "Bearer", + }, + } + + +def _write(path: Path, raw: dict) -> None: + path.write_text(json.dumps(raw), encoding="utf-8") + + +def _rotated_body(n=1): + return { + "access_token": f"hch-at-new{n}", + "refresh_token": f"hch-rt-new{n}", + "expires_in": 3600, + "scope": "write", + "token_type": "Bearer", + } + + +# --------------------------------------------------------------------------- +# oauth: transient vs permanent exchange failures +# --------------------------------------------------------------------------- + + +class TestExchangeRetry: + def test_transient_failure_recovers_on_immediate_retry(self, tmp_path, monkeypatch): + """A timed-out exchange retries right away — the server honors the + replayed refresh token only within its rotation grace window.""" + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + calls = [] + + def flaky(url, data, timeout): + calls.append(data["refresh_token"]) + if len(calls) == 1: + raise TimeoutError("token exchange timed out") + return 200, _rotated_body() + + monkeypatch.setattr(oauth, "_http_post_form_status", flaky) + token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) + + assert token == "hch-at-new1" and refreshed is True + assert calls == ["hch-rt-old", "hch-rt-old"] + saved = json.loads(path.read_text())["hosts"]["hermes"] + assert saved["oauth"]["refreshToken"] == "hch-rt-new1" + + def test_invalid_grant_stops_retries_and_marks_reauth_required(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + calls = [] + + def revoked(url, data, timeout): + calls.append(1) + return 400, {"error": "invalid_grant", "error_description": "grant revoked"} + + monkeypatch.setattr(oauth, "_http_post_form_status", revoked) + token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) + + # Fail-open return, but no retry of a permanently rejected grant. + assert token == "hch-at-old" and refreshed is False + assert len(calls) == 1 + assert oauth.reauth_required(path, "hermes") is True + + # Later refresh attempts skip the endpoint entirely. + token2, refreshed2 = oauth.ensure_fresh_token(path, "hermes", now=2000) + assert token2 == "hch-at-old" and refreshed2 is False + assert len(calls) == 1 + + # The forced (post-401) path refuses a dead grant too. + assert oauth.force_refresh_token(path, "hermes") is None + assert len(calls) == 1 + + def test_relogin_clears_the_dead_grant(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + monkeypatch.setattr( + oauth, "_http_post_form_status", + lambda *a, **k: (400, {"error": "invalid_grant"}), + ) + oauth.ensure_fresh_token(path, "hermes", now=1000) + assert oauth.reauth_required(path, "hermes") is True + + oauth.install_grant( + path, "hermes", + {"access_token": "hch-at-fresh", "refresh_token": "hch-rt-fresh", "expires_in": 3600}, + client_id="hermes-desktop", + token_endpoint="http://localhost:8000/oauth/token", + now=2000, + ) + assert oauth.reauth_required(path, "hermes") is False + token, _ = oauth.ensure_fresh_token(path, "hermes", now=2000) + assert token == "hch-at-fresh" + + def test_error_body_is_logged(self, tmp_path, monkeypatch, caplog): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block()}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + monkeypatch.setattr( + oauth, "_http_post_form_status", + lambda *a, **k: (400, {"error": "invalid_grant", "error_description": "grant revoked"}), + ) + with caplog.at_level(logging.WARNING, logger="plugins.memory.honcho.oauth"): + oauth.ensure_fresh_token(path, "hermes", now=1000) + assert "invalid_grant" in caplog.text + assert "grant revoked" in caplog.text + + def test_redaction_strips_token_values(self): + redacted = oauth._redact_tokens( + "exchange failed for hch-rt-supersecret123 got hch-at-alsosecret456" + ) + assert "supersecret123" not in redacted + assert "alsosecret456" not in redacted + assert "hch-rt-[redacted]" in redacted + assert "hch-at-[redacted]" in redacted + + +class TestForceRefreshToken: + def test_rotates_despite_local_validity(self, tmp_path, monkeypatch): + """A server-side 401 forces a rotation even when the local clock says + the token is still live.""" + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block(expires_at=time.time() + 3600)}}) + monkeypatch.setattr( + oauth, "_http_post_form_status", lambda *a, **k: (200, _rotated_body()) + ) + token = oauth.force_refresh_token(path, "hermes") + assert token == "hch-at-new1" + saved = json.loads(path.read_text())["hosts"]["hermes"] + assert saved["apiKey"] == "hch-at-new1" + + def test_adopts_concurrent_rotation_without_exchange(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + far = time.time() + 7200 + _write(path, {"hosts": {"hermes": _host_block(expires_at=far)}}) + # Seed the expiry cache with the old token. + oauth.ensure_fresh_token(path, "hermes") + # Another process rotated the credential on disk. + rotated = _host_block(refresh="hch-rt-2", expires_at=far) + rotated["apiKey"] = "hch-at-2" + _write(path, {"hosts": {"hermes": rotated}}) + monkeypatch.setattr( + oauth, "_http_post_form_status", + lambda *a, **k: pytest.fail("must adopt the on-disk rotation, not exchange"), + ) + assert oauth.force_refresh_token(path, "hermes") == "hch-at-2" + + def test_transient_failure_returns_none(self, tmp_path, monkeypatch): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": _host_block(expires_at=time.time() + 3600)}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + def boom(*a, **k): + raise ConnectionError("network down") + + monkeypatch.setattr(oauth, "_http_post_form_status", boom) + assert oauth.force_refresh_token(path, "hermes") is None + # Not permanent: a later attempt may exchange again. + assert oauth.reauth_required(path, "hermes") is False + + def test_static_api_key_is_noop(self, tmp_path): + path = tmp_path / "honcho.json" + _write(path, {"hosts": {"hermes": {"apiKey": "hch-v3-static"}}}) + assert oauth.force_refresh_token(path, "hermes") is None + + +# --------------------------------------------------------------------------- +# session: auth error detection +# --------------------------------------------------------------------------- + + +class TestAuthErrorDetection: + def test_matches_honcho_token_message(self): + assert _is_auth_error(Exception("Invalid or expired access token")) + + def test_matches_status_code_attr(self): + exc = Exception("boom") + exc.status_code = 401 + assert _is_auth_error(exc) + + def test_matches_401_text(self): + assert _is_auth_error(Exception("HTTP 401 Unauthorized")) + + def test_ignores_other_errors(self): + assert not _is_auth_error(Exception("connection reset by peer")) + assert not _is_auth_error(Exception("HTTP 500 internal error")) + + +# --------------------------------------------------------------------------- +# session: dialectic 401 recovery +# --------------------------------------------------------------------------- + + +class _FlakyPeer: + """chat() raises an auth error N times, then succeeds.""" + + def __init__(self, failures: int, result: str = "synthesized answer"): + self.failures = failures + self.result = result + self.calls = 0 + + def chat(self, query, **kw): + self.calls += 1 + if self.calls <= self.failures: + raise Exception("Invalid or expired access token") + return self.result + + +def _make_manager(peer, *, reauth_ok=True): + cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True) + mgr = HonchoSessionManager(config=cfg) + session = HonchoSession( + key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s" + ) + mgr._cache["k"] = session + mgr._get_or_create_peer = lambda peer_id: peer + mgr._force_reauth = lambda: reauth_ok + return mgr + + +class TestDialecticAuthRetry: + def test_401_forces_refresh_and_retries_once(self): + peer = _FlakyPeer(failures=1) + mgr = _make_manager(peer) + assert mgr.dialectic_query("k", "who is this user?") == "synthesized answer" + assert peer.calls == 2 # original + one retry + + def test_persistent_401_raises_auth_error(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert peer.calls == 2 # exactly one retry, no loop + + def test_failed_reauth_raises_without_retry(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer, reauth_ok=False) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert peer.calls == 1 # no retry without a fresh token + + def test_non_auth_errors_stay_fail_open(self): + class _BrokenPeer: + def chat(self, *a, **kw): + raise Exception("connection reset by peer") + + mgr = _make_manager(_BrokenPeer()) + assert mgr.dialectic_query("k", "q") == "" + + def test_success_after_failure_clears_auth_state(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer, reauth_ok=False) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert mgr._auth_failure is not None + + peer.failures = 0 + mgr._force_reauth = lambda: True + assert mgr.dialectic_query("k", "q") == "synthesized answer" + assert mgr._auth_failure is None + assert mgr.pop_auth_notice() is None + + +class TestForceReauth: + def test_rotates_and_applies_to_live_client(self, tmp_path, monkeypatch): + from plugins.memory.honcho import client as client_mod + from plugins.memory.honcho import session as session_mod + + fake_client = object() + applied = {} + monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: fake_client) + monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json") + monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h: "hch-at-new") + + def apply(client, token): + applied["client"] = client + applied["token"] = token + return True + + monkeypatch.setattr(oauth, "apply_token_to_client", apply) + + mgr = HonchoSessionManager(config=HonchoClientConfig(host="hermes")) + assert mgr._force_reauth() is True + assert applied == {"client": fake_client, "token": "hch-at-new"} + + def test_returns_false_when_refresh_yields_nothing(self, tmp_path, monkeypatch): + from plugins.memory.honcho import client as client_mod + + monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json") + monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h: None) + mgr = HonchoSessionManager(config=HonchoClientConfig(host="hermes")) + assert mgr._force_reauth() is False + + +# --------------------------------------------------------------------------- +# session: message sync 401 recovery +# --------------------------------------------------------------------------- + + +class _FlakyHonchoSession: + """add_messages() raises an auth error N times, then succeeds.""" + + def __init__(self, failures: int): + self.failures = failures + self.calls = 0 + + def add_messages(self, messages): + self.calls += 1 + if self.calls <= self.failures: + raise Exception("Invalid or expired access token") + + +def _make_sync_manager(flaky_session, *, reauth_ok=True): + cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True) + mgr = HonchoSessionManager(config=cfg) + peer = MagicMock() + peer.message.side_effect = lambda content: content + mgr._get_or_create_peer = lambda peer_id: peer + mgr._sessions_cache["s"] = flaky_session + mgr._force_reauth = lambda: reauth_ok + session = HonchoSession( + key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s" + ) + session.add_message("user", "hello") + session.add_message("assistant", "hi") + return mgr, session + + +class TestSyncAuthRetry: + def test_401_forces_refresh_and_retries_once(self): + flaky = _FlakyHonchoSession(failures=1) + mgr, session = _make_sync_manager(flaky) + assert mgr._flush_session(session) is True + assert flaky.calls == 2 + assert all(m["_synced"] for m in session.messages) + + def test_persistent_401_fails_and_records_auth_failure(self): + flaky = _FlakyHonchoSession(failures=99) + mgr, session = _make_sync_manager(flaky) + assert mgr._flush_session(session) is False + assert flaky.calls == 2 # exactly one retry, no loop + assert not any(m.get("_synced") for m in session.messages) + assert mgr._auth_failure is not None + + def test_failed_reauth_fails_without_retry(self): + flaky = _FlakyHonchoSession(failures=99) + mgr, session = _make_sync_manager(flaky, reauth_ok=False) + assert mgr._flush_session(session) is False + assert flaky.calls == 1 + assert mgr._auth_failure is not None + + def test_later_success_recovers_and_clears_auth_state(self): + flaky = _FlakyHonchoSession(failures=2) + mgr, session = _make_sync_manager(flaky, reauth_ok=False) + assert mgr._flush_session(session) is False + assert mgr._auth_failure is not None + + mgr._force_reauth = lambda: True + assert mgr._flush_session(session) is True + assert all(m["_synced"] for m in session.messages) + assert mgr._auth_failure is None + + +# --------------------------------------------------------------------------- +# one-time user-facing notice +# --------------------------------------------------------------------------- + + +class TestAuthNotice: + def test_manager_emits_notice_exactly_once(self): + peer = _FlakyPeer(failures=99) + mgr = _make_manager(peer, reauth_ok=False) + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + + first = mgr.pop_auth_notice() + assert first and "Invalid or expired access token" in first + assert mgr.pop_auth_notice() is None + + # A second failure inside the same episode does not re-arm the notice. + with pytest.raises(HonchoAuthError): + mgr.dialectic_query("k", "q") + assert mgr.pop_auth_notice() is None + + def test_provider_prefetch_injects_notice_once(self): + class _FakeManager: + def __init__(self): + self.notices = ["Invalid or expired access token"] + + def pop_auth_notice(self): + return self.notices.pop() if self.notices else None + + def pop_context_result(self, session_key): + return {} + + provider = HonchoMemoryProvider() + provider._manager = _FakeManager() + provider._config = SimpleNamespace(timeout=0.01, context_tokens=0) + provider._session_key = "k" + provider._session_initialized = True + provider._recall_mode = "context" + provider._turn_count = 2 + provider._last_dialectic_turn = 0 + provider._base_context_cache = "" + + first = provider.prefetch("what did we decide about the schema?") + assert "hermes honcho setup" in first + assert "paused" in first + + second = provider.prefetch("and the follow-up question?") + assert second == "" + + +# --------------------------------------------------------------------------- +# cadence backoff exemption +# --------------------------------------------------------------------------- + + +class TestBackoffExemption: + def test_auth_error_does_not_widen_backoff(self): + provider = HonchoMemoryProvider() + provider._note_dialectic_failure(HonchoAuthError("still 401 after refresh")) + assert provider._dialectic_empty_streak == 0 + + def test_other_errors_still_widen_backoff(self): + provider = HonchoMemoryProvider() + provider._note_dialectic_failure(RuntimeError("timeout")) + assert provider._dialectic_empty_streak == 1 diff --git a/tests/honcho_plugin/test_oauth.py b/tests/honcho_plugin/test_oauth.py index 58ca9d0b5ba65..bdaebe3ab02a6 100644 --- a/tests/honcho_plugin/test_oauth.py +++ b/tests/honcho_plugin/test_oauth.py @@ -69,7 +69,7 @@ class TestEnsureFreshToken: path = tmp_path / "honcho.json" _write(path, {"hosts": {"hermes": _host_block(expires_at=10_000)}}) monkeypatch.setattr( - oauth, "_http_post_form", + oauth, "_http_post_form_status", lambda *a, **k: pytest.fail("refresh must not be called when fresh"), ) token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=0) @@ -84,7 +84,7 @@ class TestEnsureFreshToken: assert data["grant_type"] == "refresh_token" assert data["refresh_token"] == "hch-rt-old" assert data["client_id"] == "hermes-desktop" - return { + return 200, { "access_token": "hch-at-new", "refresh_token": "hch-rt-new", "expires_in": 3600, @@ -92,7 +92,7 @@ class TestEnsureFreshToken: "token_type": "Bearer", } - monkeypatch.setattr(oauth, "_http_post_form", fake_post) + monkeypatch.setattr(oauth, "_http_post_form_status", fake_post) token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) assert token == "hch-at-new" and refreshed is True @@ -105,14 +105,20 @@ class TestEnsureFreshToken: def test_refresh_failure_fails_open(self, tmp_path, monkeypatch): path = tmp_path / "honcho.json" _write(path, {"hosts": {"hermes": _host_block(expires_at=100)}}) + monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0) + + calls = [] def boom(*a, **k): + calls.append(a) raise RuntimeError("network down") - monkeypatch.setattr(oauth, "_http_post_form", boom) + monkeypatch.setattr(oauth, "_http_post_form_status", boom) token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000) - # Stale token returned, no crash, file untouched. + # Stale token returned, no crash, file untouched. Transient failures + # retry exactly once, then fail open. assert token == "hch-at-old" and refreshed is False + assert len(calls) == 2 assert json.loads(path.read_text())["hosts"]["hermes"]["apiKey"] == "hch-at-old" def test_double_check_uses_disk_when_already_rotated(self, tmp_path, monkeypatch): @@ -123,7 +129,7 @@ class TestEnsureFreshToken: stale_raw = {"hosts": {"hermes": _host_block(refresh="hch-rt-old", expires_at=100)}} stale_raw["hosts"]["hermes"]["apiKey"] = "hch-at-stale" monkeypatch.setattr( - oauth, "_http_post_form", + oauth, "_http_post_form_status", lambda *a, **k: pytest.fail("must not refresh; disk token is fresh"), ) token, refreshed = oauth.ensure_fresh_token(path, "hermes", stale_raw, now=1000)