From a37166efa5eccda7cab264ad427cc64c2d574318 Mon Sep 17 00:00:00 2001 From: GauravPatil2515 Date: Sat, 8 Aug 2026 13:44:29 +0530 Subject: [PATCH 1/4] fix(approval): disable mouse tracking to prevent micro-movement false denies - Add _suppress_mouse_tracking_cli() in cli.py called at classic CLI startup - Add mouse suppression in prompt_dangerous_approval() for legacy input() path - Enhance _suppress_mouse_residue_early() in hermes_cli/main.py with focus tracking disable - Disables all 10 mouse tracking modes (SGR 1006, X10 1000, focus 1004, kitty 2029, etc.) - Fixes #78587: mouse jiggle falsely interpreted as deny in approval prompts --- cli.py | 29 +++++++++++++++++++++++++++++ hermes_cli/main.py | 5 +++-- tools/approval.py | 14 ++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/cli.py b/cli.py index aed3992922b4a..b4b4dab48a9d8 100644 --- a/cli.py +++ b/cli.py @@ -1008,6 +1008,30 @@ def _mark_tui_input_modes_active() -> None: _tui_input_modes_active = True +def _suppress_mouse_tracking_cli() -> None: + """Disable terminal mouse tracking for the classic CLI to prevent + mouse micro-movements from generating escape sequences that could + be misinterpreted as key presses (e.g. triggering 'deny' in approval prompts). + + This is similar to _suppress_mouse_residue_early() for the TUI but runs + at the start of the classic CLI run() method. + """ + try: + if not os.isatty(1): + return + # Disable all mouse tracking variants: SGR (1006), X10 (1000), button-event (1002), + # any-event (1003), focus (1004), and alternate scroll (1005/1015/1016) + # Also disable vt200 mouse (1001) and kitty mouse (2029) + os.write( + 1, + b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l" + b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l" + b"\x1b[?1004l\x1b[?2029l", + ) + except OSError: + pass + + def _prepare_deferred_agent_startup() -> None: """Run Termux-deferred agent discovery before the first real agent turn.""" global _deferred_agent_startup_done @@ -14958,6 +14982,11 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): _welcome_color = "#FFF8DC" self._console_print(f"[{_welcome_color}]{_welcome_text}[/]") + # Disable terminal mouse tracking to prevent mouse micro-movements + # from generating escape sequences that could interfere with + # approval prompts and other modal dialogs. + _suppress_mouse_tracking_cli() + # Warm the /model picker's provider-models cache off-thread during this # idle window (banner shown, user about to type). The no-args picker # otherwise blocks ~1-2s on serial /v1/models fetches the first time diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 624d0f10109f7..a1c707461449c 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -361,8 +361,9 @@ def _suppress_mouse_residue_early() -> None: # safe to send even when no tracking is currently asserted. os.write( 1, - b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l\x1b[?9l" - b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l\x1b[?2029l", + b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l" + b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l" + b"\x1b[?1004l\x1b[?2029l", ) except OSError: pass diff --git a/tools/approval.py b/tools/approval.py index 151e534065539..6cdf59e6fc825 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -2634,6 +2634,20 @@ def prompt_dangerous_approval(command: str, description: str, os.environ["HERMES_SPINNER_PAUSE"] = "1" try: + # Disable mouse tracking to prevent mouse micro-movements from + # generating escape sequences that could be captured by input() + # and misinterpreted as user input. + try: + if os.isatty(1): + os.write( + 1, + b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l" + b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l" + b"\x1b[?1004l\x1b[?2029l", + ) + except OSError: + pass + # Resolve the active UI language once per prompt so we don't re-read # config/YAML inside the retry loop below. from agent.i18n import t From 542446360dda1a9c9f70a3b8c5af8ef451a90674 Mon Sep 17 00:00:00 2001 From: GauravPatil2515 Date: Sat, 8 Aug 2026 14:08:48 +0530 Subject: [PATCH 2/4] fix(telegram): add connect/read timeouts to fallback transport to prevent cold boot hang - Add connect_timeout/read_timeout parameters to TelegramFallbackTransport - Configure httpx.Timeout at transport level so socket connect/read operations respect timeouts directly (issue #78586) - Pass timeouts from request_kwargs when creating transports in TelegramAdapter - Fixes gateway stalling at 'Connecting to Telegram (attempt 1/8)' on cold boot --- plugins/platforms/telegram/adapter.py | 15 +++++++++++++-- plugins/platforms/telegram/telegram_network.py | 14 +++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index ea6258b9f2dc6..17c02ac6eae6a 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -3835,11 +3835,19 @@ class TelegramAdapter(BasePlatformAdapter): _transport_kwargs: dict = {} if _pool_limits is not None: _transport_kwargs["limits"] = _pool_limits + # Pass connect/read timeouts to the fallback transport so they + # apply at the socket level, preventing indefinite hangs during + # DNS resolution or TCP connect (issue #78586). + connect_timeout = request_kwargs.get("connect_timeout", 10.0) + read_timeout = request_kwargs.get("read_timeout", 20.0) request = HTTPXRequest( **request_kwargs, httpx_kwargs={ "transport": TelegramFallbackTransport( - fallback_ips, **_transport_kwargs + fallback_ips, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + **_transport_kwargs, ) }, ) @@ -3847,7 +3855,10 @@ class TelegramAdapter(BasePlatformAdapter): **request_kwargs, httpx_kwargs={ "transport": TelegramFallbackTransport( - fallback_ips, **_transport_kwargs + fallback_ips, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + **_transport_kwargs, ) }, ) diff --git a/plugins/platforms/telegram/telegram_network.py b/plugins/platforms/telegram/telegram_network.py index 5b1d8d12bfb04..c2a80cdbcf37f 100644 --- a/plugins/platforms/telegram/telegram_network.py +++ b/plugins/platforms/telegram/telegram_network.py @@ -63,12 +63,24 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport): # on its own (#63311). _POOL_LIMITS = httpx.Limits(max_connections=8, max_keepalive_connections=4) - def __init__(self, fallback_ips: Iterable[str], **transport_kwargs): + def __init__( + self, + fallback_ips: Iterable[str], + *, + connect_timeout: float = 10.0, + read_timeout: float = 20.0, + **transport_kwargs, + ): self._fallback_ips = list(dict.fromkeys(_normalize_fallback_ips(fallback_ips))) proxy_url = _resolve_proxy_url(target_hosts=[_TELEGRAM_API_HOST, *self._fallback_ips]) if proxy_url and "proxy" not in transport_kwargs: transport_kwargs["proxy"] = proxy_url transport_kwargs.setdefault("limits", self._POOL_LIMITS) + # Configure timeouts at the transport level so they apply to socket + # connect/read operations directly, preventing indefinite hangs during + # DNS resolution or TCP connect (issue #78586). + timeout = httpx.Timeout(connect=connect_timeout, read=read_timeout) + transport_kwargs["timeout"] = timeout self._transport_kwargs = transport_kwargs self._primary = httpx.AsyncHTTPTransport(**transport_kwargs) # Built on demand and discarded on failure — see _reset_fallback. From 92dda0d367f59b98641cc47593b55b104e053d91 Mon Sep 17 00:00:00 2001 From: GauravPatil2515 Date: Sat, 8 Aug 2026 14:43:00 +0530 Subject: [PATCH 3/4] fix(streaming): handle EmptyStreamError/TimeoutError gracefully to prevent CLI deadlock - Add try/except/finally around run_llm_execution_middleware to catch streaming errors - Handle EmptyStreamError and TimeoutError by invoking error hook with stream_error reason - Skip duplicate error handling in validation logic for streaming errors - Prevents CLI deadlock and UI corruption when provider stream stalls (issue #81521) - Maintains existing retry/fallback logic through error hook --- agent/conversation_loop.py | 84 ++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 8848b64df2cb2..341fedee47e93 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -87,6 +87,7 @@ from agent.retry_utils import ( ) from agent.trajectory import has_incomplete_scratchpad from agent.usage_pricing import estimate_usage_cost, normalize_usage +from agent.errors import EmptyStreamError from hermes_constants import PARTIAL_STREAM_STUB_ID from hermes_logging import set_session_context from tools.skill_provenance import set_current_write_origin @@ -2418,6 +2419,7 @@ def run_conversation( elif _model_request_active is not None: _model_request_active.set() _redirect_crossed_response = False + streaming_error_handled = False try: response = run_llm_execution_middleware( api_kwargs, @@ -2435,6 +2437,45 @@ def run_conversation( api_call_count=api_call_count, middleware_trace=list(_llm_middleware_trace), ) + except (EmptyStreamError, TimeoutError) as e: + # Handle streaming failures (empty stream, stalled stream) gracefully + # instead of letting them propagate and cause deadlock/UI corruption. + logger.warning( + "Streaming API call failed (%s): %s", + type(e).__name__, + e, + ) + # Set response to None so the validation logic below treats it as invalid + response = None + response_invalid = True + error_details = [str(e)] + streaming_error_handled = True + # Invoke error hook immediately for streaming errors so retry/fallback logic runs + agent._invoke_api_request_error_hook( + task_id=effective_task_id, + turn_id=turn_id, + api_request_id=api_request_id, + api_call_count=api_call_count, + api_start_time=api_start_time, + api_kwargs=api_kwargs, + error_type=type(e).__name__, + error_message=str(e), + status_code=None, + retry_count=retry_count, + max_retries=max_retries, + retryable=True, + reason="stream_error", + ) + except InterruptedError: + # Interrupts are handled by the redirect logic below + raise + except BaseException as e: + # Other exceptions (provider errors, etc.) - let them propagate + # to the outer retry/fallback handler + logger.exception("API call raised unexpected error: %s", e) + response = None + response_invalid = True + error_details = [str(e)] finally: if _redirect_lock is not None: with _redirect_lock: @@ -2482,8 +2523,10 @@ def run_conversation( logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}") # Validate response shape before proceeding - response_invalid = False - error_details = [] + # response_invalid and error_details may have been set by the exception handler above + if 'response_invalid' not in locals(): + response_invalid = False + error_details = [] if agent.api_mode == "codex_responses": _ct_v = agent._get_transport() if not _ct_v.validate_response(response): @@ -2562,21 +2605,28 @@ def run_conversation( error_details.append("response.choices is empty") if response_invalid: - agent._invoke_api_request_error_hook( - task_id=effective_task_id, - turn_id=turn_id, - api_request_id=api_request_id, - api_call_count=api_call_count, - api_start_time=api_start_time, - api_kwargs=api_kwargs, - error_type="InvalidAPIResponse", - error_message=", ".join(error_details) or "Invalid API response", - status_code=getattr(getattr(response, "error", None), "code", None), - retry_count=retry_count, - max_retries=max_retries, - retryable=True, - reason="invalid_response", - ) + # Skip duplicate error handling for streaming errors already handled above + if locals().get('streaming_error_handled', False): + # Streaming error was already handled in the except block + # The error hook was invoked and fallback/retry logic may have been triggered + # Just continue to the retry/fallback logic below + pass + else: + agent._invoke_api_request_error_hook( + task_id=effective_task_id, + turn_id=turn_id, + api_request_id=api_request_id, + api_call_count=api_call_count, + api_start_time=api_start_time, + api_kwargs=api_kwargs, + error_type="InvalidAPIResponse", + error_message=", ".join(error_details) or "Invalid API response", + status_code=getattr(getattr(response, "error", None), "code", None), + retry_count=retry_count, + max_retries=max_retries, + retryable=True, + reason="invalid_response", + ) # Stop spinner silently — retry status is now buffered # and only surfaced if every retry+fallback exhausts. if thinking_spinner: From fa71a16b1db1eb132f2d9a451b72ad145ab2a8fb Mon Sep 17 00:00:00 2001 From: GauravPatil2515 Date: Sat, 8 Aug 2026 15:07:25 +0530 Subject: [PATCH 4/4] fix(curator): save state before snapshot to prevent OOM restart loop - Move state persistence (last_run_at) to BEFORE snapshot_skills() call - Prevents vicious cycle: OOM during snapshot -> restart -> immediate curator run -> OOM - Fixes issue #81625: gateway balloons to 60GB RSS every ~2h - State now records run start before any memory-intensive operations --- agent/curator.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/agent/curator.py b/agent/curator.py index ab0adddae118d..adac585b27d05 100644 --- a/agent/curator.py +++ b/agent/curator.py @@ -1530,6 +1530,20 @@ def run_curator_review( if consolidate is None: consolidate = get_consolidate() start = datetime.now(timezone.utc) + + # CRITICAL FIX: Persist state BEFORE snapshot creation so that if + # snapshot creation causes OOM, the curator won't immediately re-run + # on restart (which would create a vicious cycle of OOM -> restart -> OOM). + # See issue #81625: gateway balloons to 60GB RSS every ~2h. + if not dry_run: + state = load_state() + state["last_run_at"] = start.isoformat() + state["run_count"] = int(state.get("run_count", 0)) + 1 + prefix = "auto: " + # We don't have auto_summary yet, use placeholder + state["last_run_summary"] = f"{prefix}starting (snapshot phase)" + save_state(state) + if dry_run: # Count candidates without mutating state. try: @@ -1569,16 +1583,16 @@ def run_curator_review( auto_summary_parts.append(f"{counts['reactivated']} reactivated") auto_summary = ", ".join(auto_summary_parts) if auto_summary_parts else "no changes" + prefix = "dry-run auto: " if dry_run else "auto: " + # Persist state before the LLM pass so a crash mid-review still records # the run and doesn't immediately re-trigger. In dry-run we do NOT bump # last_run_at or run_count — a preview shouldn't push the next scheduled # real pass out. We still record a summary so `hermes curator status` # shows that a preview ran. + # NOTE: State was already saved before snapshot creation (above). + # Here we just update the summary with the actual auto_summary. state = load_state() - if not dry_run: - state["last_run_at"] = start.isoformat() - state["run_count"] = int(state.get("run_count", 0)) + 1 - prefix = "dry-run auto: " if dry_run else "auto: " state["last_run_summary"] = f"{prefix}{auto_summary}" save_state(state)