From 6397776fe897b96ae1bd85caaeded23146c700c5 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:52:51 -0700 Subject: [PATCH] refactor: simplify discord classifier + move attention threshold to config.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Collapse the duplicated discord LoginFailure/PrivilegedIntentsRequired classification (name-match + isinstance blocks repeated the same code/ message tuples) into a single _is() helper — one message per failure. - Replace the user-facing HERMES_RECONNECT_ATTENTION_AFTER_SECONDS env var with agent.reconnect_attention_after in config.yaml (default 7200, 0 disables), bridged internally like gateway_timeout. .env is for secrets. - Use _float_env for robust parsing instead of bare int(os.getenv(...)). - Document terminal classification + needs_attention escalation in website/docs/user-guide/configuration.md. --- gateway/run.py | 12 +++++-- hermes_cli/config_defaults.py | 6 ++++ plugins/platforms/discord/adapter.py | 42 ++++++++---------------- website/docs/user-guide/configuration.md | 14 ++++++++ 4 files changed, 44 insertions(+), 30 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 24497a34e2708..a0002608b0840 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2262,6 +2262,12 @@ if _config_path.exists(): os.environ["HERMES_SESSION_STALL_TIMEOUT"] = str( _agent_cfg["session_stall_timeout"] ) + if "reconnect_attention_after" in _agent_cfg: + # Internal bridge only — config.yaml (agent.reconnect_attention_after) + # is the documented, user-facing setting. + os.environ["HERMES_RECONNECT_ATTENTION_AFTER_SECONDS"] = str( + _agent_cfg["reconnect_attention_after"] + ) if "restart_drain_timeout" in _agent_cfg: os.environ["HERMES_RESTART_DRAIN_TIMEOUT"] = str(_agent_cfg["restart_drain_timeout"]) if "gateway_auto_continue_freshness" in _agent_cfg: @@ -3795,8 +3801,10 @@ _RECONNECT_BACKOFF_CAP = 300 # owners and fleet monitoring can distinguish hour one from week three. # A dead bot token, a revoked Discord intent, or a deterministically crashing # sidecar all present as "retrying" forever without this signal. -_RECONNECT_ATTENTION_AFTER_SECONDS = int( - os.getenv("HERMES_RECONNECT_ATTENTION_AFTER_SECONDS", "7200") +# User-facing setting: agent.reconnect_attention_after in config.yaml +# (bridged to this env var above). 0 disables. +_RECONNECT_ATTENTION_AFTER_SECONDS = _float_env( + "HERMES_RECONNECT_ATTENTION_AFTER_SECONDS", 7200 ) diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 05b621db293af..18ac01785358b 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -227,6 +227,12 @@ DEFAULT_CONFIG = { # from gateway_timeout (which kills the turn) and # gateway_notify_interval ("still working" heartbeats). 0 = disable. "session_stall_timeout": 300, + # Long-lived reconnect-loop escalation (seconds). A platform that has + # been continuously failing/reconnecting for this long gets + # needs_attention flagged in gateway runtime status (visible in + # `hermes status` / fleet monitoring). Retries never stop — this is a + # signal, not a circuit breaker. 0 = disable. + "reconnect_attention_after": 7200, # Freshness window for the gateway auto-continue note (seconds). # After a gateway crash/restart/SIGTERM mid-run, the next user # message gets a "[System note: your previous turn was diff --git a/plugins/platforms/discord/adapter.py b/plugins/platforms/discord/adapter.py index 239a93a7117b4..fdb04dee406a0 100644 --- a/plugins/platforms/discord/adapter.py +++ b/plugins/platforms/discord/adapter.py @@ -1471,10 +1471,19 @@ class DiscordAdapter(BasePlatformAdapter): failure mode the auto-pause removal fixed. The reconnect watcher's NEEDS_ATTENTION escalation covers misclassified permanent failures. """ - name = error.__class__.__name__ - # String-typename fallback keeps this classifiable when discord.py is - # mocked (tests) or the import fails; real isinstance checks follow. - if name == "LoginFailure": + def _is(type_name: str) -> bool: + # Class-name check covers mocked discord.py (tests) and failed + # imports; the isinstance check additionally covers subclasses. + if error.__class__.__name__ == type_name: + return True + try: + import discord as _discord + exc_type = getattr(_discord, type_name, None) + return isinstance(exc_type, type) and isinstance(error, exc_type) + except Exception: + return False + + if _is("LoginFailure"): return ( "discord_auth_error", f"Discord bot token rejected: {error}. The token is invalid or " @@ -1482,7 +1491,7 @@ class DiscordAdapter(BasePlatformAdapter): "and update DISCORD_BOT_TOKEN.", False, ) - if name == "PrivilegedIntentsRequired": + if _is("PrivilegedIntentsRequired"): return ( "discord_intents_required", "Discord privileged intents are not enabled for this bot: " @@ -1491,29 +1500,6 @@ class DiscordAdapter(BasePlatformAdapter): "Discord Developer Portal → Bot → Privileged Gateway Intents.", False, ) - try: - import discord as _discord - login_failure = getattr(_discord, "LoginFailure", None) - intents_required = getattr(_discord, "PrivilegedIntentsRequired", None) - if isinstance(login_failure, type) and isinstance(error, login_failure): - return ( - "discord_auth_error", - f"Discord bot token rejected: {error}. The token is invalid " - "or was revoked — regenerate it in the Discord Developer " - "Portal and update DISCORD_BOT_TOKEN.", - False, - ) - if isinstance(intents_required, type) and isinstance(error, intents_required): - return ( - "discord_intents_required", - "Discord privileged intents are not enabled for this bot: " - f"{error}. Enable 'Message Content Intent' for this " - "application in the Discord Developer Portal → Bot → " - "Privileged Gateway Intents.", - False, - ) - except Exception: - pass return ("discord_connect_error", f"Discord startup failed: {error}", True) def _discord_message_admission( diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index ecb422ee15117..7be3fdbc335cd 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -940,6 +940,20 @@ agent: session_stall_timeout: 300 # seconds; 0 disables the watchdog ``` +## Reconnect Attention Escalation + +When a platform adapter fails to connect (network outage, revoked bot token, broken sidecar), the gateway retries it indefinitely with capped exponential backoff — retries never stop, so a transient outage always self-heals without operator action. The downside is that a *permanent* failure (a revoked Telegram token, missing Discord privileged intents) looks identical to a blip: "retrying", forever. + +Two mechanisms make permanent failures visible: + +- **Terminal classification.** Failures whose exception *type* proves they can never self-heal — rejected/revoked tokens (`telegram_auth_error`, `discord_auth_error`, `email_auth_error`), missing privileged intents (`discord_intents_required`), a Photon sidecar whose dependencies cannot install (`SIDECAR_DEPS_MISSING`) or whose node binary is missing (`SIDECAR_NODE_MISSING`) — are marked fatal instead of entering the retry queue. Classification is strictly type-based; ambiguous errors always keep retrying. +- **Needs-attention escalation.** A platform continuously in the retry queue past `agent.reconnect_attention_after` (default `7200` seconds = 2 hours, `0` disables) gets `needs_attention: true` and a `retrying_since` timestamp in gateway runtime status (`hermes status`), plus a WARNING log. Retries continue unchanged — this is a signal, not a circuit breaker. The flag clears on successful reconnect. + +```yaml +agent: + reconnect_attention_after: 7200 # seconds; 0 disables the escalation flag +``` + ## Gateway Agent Cache The gateway keeps one agent per session so a conversation reuses its cached prompt prefix instead of rebuilding the system prompt every turn. That cached agent also holds the session's full transcript — tool output included, which is tens of megabytes on a session with a hundred tool calls. On a busy multi-platform gateway the cache is therefore the largest single consumer of memory in the process.