diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index c05167d7685d0..8ce62a0055f1a 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -91,6 +91,7 @@ from agent.trajectory import has_incomplete_scratchpad # finalizer at turn end. from agent.turn_finalizer import finalize_turn from agent.usage_pricing import estimate_usage_cost, normalize_usage +from agent.errors import EmptyStreamError from hermes_constants import PARTIAL_STREAM_STUB_ID from hermes_logging import set_session_context from tools.skill_provenance import set_current_write_origin @@ -2812,6 +2813,7 @@ def run_conversation( elif _model_request_active is not None: _model_request_active.set() _redirect_crossed_response = False + streaming_error_handled = False try: response = run_llm_execution_middleware( api_kwargs, @@ -2829,6 +2831,45 @@ def run_conversation( api_call_count=api_call_count, middleware_trace=list(_llm_middleware_trace), ) + except (EmptyStreamError, TimeoutError) as e: + # Handle streaming failures (empty stream, stalled stream) gracefully + # instead of letting them propagate and cause deadlock/UI corruption. + logger.warning( + "Streaming API call failed (%s): %s", + type(e).__name__, + e, + ) + # Set response to None so the validation logic below treats it as invalid + response = None + response_invalid = True + error_details = [str(e)] + streaming_error_handled = True + # Invoke error hook immediately for streaming errors so retry/fallback logic runs + agent._invoke_api_request_error_hook( + task_id=effective_task_id, + turn_id=turn_id, + api_request_id=api_request_id, + api_call_count=api_call_count, + api_start_time=api_start_time, + api_kwargs=api_kwargs, + error_type=type(e).__name__, + error_message=str(e), + status_code=None, + retry_count=retry_count, + max_retries=max_retries, + retryable=True, + reason="stream_error", + ) + except InterruptedError: + # Interrupts are handled by the redirect logic below + raise + except BaseException as e: + # Other exceptions (provider errors, etc.) - let them propagate + # to the outer retry/fallback handler + logger.exception("API call raised unexpected error: %s", e) + response = None + response_invalid = True + error_details = [str(e)] finally: if _redirect_lock is not None: with _redirect_lock: @@ -2876,8 +2917,10 @@ def run_conversation( logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}") # Validate response shape before proceeding - response_invalid = False - error_details = [] + # response_invalid and error_details may have been set by the exception handler above + if 'response_invalid' not in locals(): + response_invalid = False + error_details = [] if agent.api_mode == "codex_responses": _ct_v = agent._get_transport() if not _ct_v.validate_response(response): @@ -2956,21 +2999,28 @@ def run_conversation( error_details.append("response.choices is empty") if response_invalid: - agent._invoke_api_request_error_hook( - task_id=effective_task_id, - turn_id=turn_id, - api_request_id=api_request_id, - api_call_count=api_call_count, - api_start_time=api_start_time, - api_kwargs=api_kwargs, - error_type="InvalidAPIResponse", - error_message=", ".join(error_details) or "Invalid API response", - status_code=getattr(getattr(response, "error", None), "code", None), - retry_count=retry_count, - max_retries=max_retries, - retryable=True, - reason="invalid_response", - ) + # Skip duplicate error handling for streaming errors already handled above + if locals().get('streaming_error_handled', False): + # Streaming error was already handled in the except block + # The error hook was invoked and fallback/retry logic may have been triggered + # Just continue to the retry/fallback logic below + pass + else: + agent._invoke_api_request_error_hook( + task_id=effective_task_id, + turn_id=turn_id, + api_request_id=api_request_id, + api_call_count=api_call_count, + api_start_time=api_start_time, + api_kwargs=api_kwargs, + error_type="InvalidAPIResponse", + error_message=", ".join(error_details) or "Invalid API response", + status_code=getattr(getattr(response, "error", None), "code", None), + retry_count=retry_count, + max_retries=max_retries, + retryable=True, + reason="invalid_response", + ) # Stop spinner silently — retry status is now buffered # and only surfaced if every retry+fallback exhausts. if thinking_spinner: diff --git a/agent/curator.py b/agent/curator.py index ab0adddae118d..adac585b27d05 100644 --- a/agent/curator.py +++ b/agent/curator.py @@ -1530,6 +1530,20 @@ def run_curator_review( if consolidate is None: consolidate = get_consolidate() start = datetime.now(timezone.utc) + + # CRITICAL FIX: Persist state BEFORE snapshot creation so that if + # snapshot creation causes OOM, the curator won't immediately re-run + # on restart (which would create a vicious cycle of OOM -> restart -> OOM). + # See issue #81625: gateway balloons to 60GB RSS every ~2h. + if not dry_run: + state = load_state() + state["last_run_at"] = start.isoformat() + state["run_count"] = int(state.get("run_count", 0)) + 1 + prefix = "auto: " + # We don't have auto_summary yet, use placeholder + state["last_run_summary"] = f"{prefix}starting (snapshot phase)" + save_state(state) + if dry_run: # Count candidates without mutating state. try: @@ -1569,16 +1583,16 @@ def run_curator_review( auto_summary_parts.append(f"{counts['reactivated']} reactivated") auto_summary = ", ".join(auto_summary_parts) if auto_summary_parts else "no changes" + prefix = "dry-run auto: " if dry_run else "auto: " + # Persist state before the LLM pass so a crash mid-review still records # the run and doesn't immediately re-trigger. In dry-run we do NOT bump # last_run_at or run_count — a preview shouldn't push the next scheduled # real pass out. We still record a summary so `hermes curator status` # shows that a preview ran. + # NOTE: State was already saved before snapshot creation (above). + # Here we just update the summary with the actual auto_summary. state = load_state() - if not dry_run: - state["last_run_at"] = start.isoformat() - state["run_count"] = int(state.get("run_count", 0)) + 1 - prefix = "dry-run auto: " if dry_run else "auto: " state["last_run_summary"] = f"{prefix}{auto_summary}" save_state(state) diff --git a/cli.py b/cli.py index a64c54af31a1a..6c351d71046f2 100644 --- a/cli.py +++ b/cli.py @@ -997,6 +997,30 @@ def _mark_tui_input_modes_active() -> None: _tui_input_modes_active = True +def _suppress_mouse_tracking_cli() -> None: + """Disable terminal mouse tracking for the classic CLI to prevent + mouse micro-movements from generating escape sequences that could + be misinterpreted as key presses (e.g. triggering 'deny' in approval prompts). + + This is similar to _suppress_mouse_residue_early() for the TUI but runs + at the start of the classic CLI run() method. + """ + try: + if not os.isatty(1): + return + # Disable all mouse tracking variants: SGR (1006), X10 (1000), button-event (1002), + # any-event (1003), focus (1004), and alternate scroll (1005/1015/1016) + # Also disable vt200 mouse (1001) and kitty mouse (2029) + os.write( + 1, + b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l" + b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l" + b"\x1b[?1004l\x1b[?2029l", + ) + except OSError: + pass + + def _prepare_deferred_agent_startup() -> None: """Run Termux-deferred agent discovery before the first real agent turn.""" global _deferred_agent_startup_done @@ -15692,6 +15716,11 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): _welcome_color = "#FFF8DC" self._console_print(f"[{_welcome_color}]{_welcome_text}[/]") + # Disable terminal mouse tracking to prevent mouse micro-movements + # from generating escape sequences that could interfere with + # approval prompts and other modal dialogs. + _suppress_mouse_tracking_cli() + # Warm the /model picker's provider-models cache off-thread during this # idle window (banner shown, user about to type). The no-args picker # otherwise blocks ~1-2s on serial /v1/models fetches the first time diff --git a/hermes_cli/main.py b/hermes_cli/main.py index b58a6c0c18ff4..fa49a3d93948c 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -361,8 +361,9 @@ def _suppress_mouse_residue_early() -> None: # safe to send even when no tracking is currently asserted. os.write( 1, - b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l\x1b[?9l" - b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l\x1b[?2029l", + b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l" + b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l" + b"\x1b[?1004l\x1b[?2029l", ) except OSError: pass diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index f31cd85e6a5ca..ca4aad978e611 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -4142,11 +4142,19 @@ class TelegramAdapter(BasePlatformAdapter): _transport_kwargs: dict = {} if _pool_limits is not None: _transport_kwargs["limits"] = _pool_limits + # Pass connect/read timeouts to the fallback transport so they + # apply at the socket level, preventing indefinite hangs during + # DNS resolution or TCP connect (issue #78586). + connect_timeout = request_kwargs.get("connect_timeout", 10.0) + read_timeout = request_kwargs.get("read_timeout", 20.0) request = HTTPXRequest( **request_kwargs, httpx_kwargs={ "transport": TelegramFallbackTransport( - fallback_ips, **_transport_kwargs + fallback_ips, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + **_transport_kwargs, ) }, ) @@ -4154,7 +4162,10 @@ class TelegramAdapter(BasePlatformAdapter): **request_kwargs, httpx_kwargs={ "transport": TelegramFallbackTransport( - fallback_ips, **_transport_kwargs + fallback_ips, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + **_transport_kwargs, ) }, ) diff --git a/plugins/platforms/telegram/telegram_network.py b/plugins/platforms/telegram/telegram_network.py index aad9361767366..671e69495daab 100644 --- a/plugins/platforms/telegram/telegram_network.py +++ b/plugins/platforms/telegram/telegram_network.py @@ -63,12 +63,24 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport): # on its own (#63311). _POOL_LIMITS = httpx.Limits(max_connections=8, max_keepalive_connections=4) - def __init__(self, fallback_ips: Iterable[str], **transport_kwargs): + def __init__( + self, + fallback_ips: Iterable[str], + *, + connect_timeout: float = 10.0, + read_timeout: float = 20.0, + **transport_kwargs, + ): self._fallback_ips = list(dict.fromkeys(_normalize_fallback_ips(fallback_ips))) proxy_url = _resolve_proxy_url(target_hosts=[_TELEGRAM_API_HOST, *self._fallback_ips]) if proxy_url and "proxy" not in transport_kwargs: transport_kwargs["proxy"] = proxy_url transport_kwargs.setdefault("limits", self._POOL_LIMITS) + # Configure timeouts at the transport level so they apply to socket + # connect/read operations directly, preventing indefinite hangs during + # DNS resolution or TCP connect (issue #78586). + timeout = httpx.Timeout(connect=connect_timeout, read=read_timeout) + transport_kwargs["timeout"] = timeout self._transport_kwargs = transport_kwargs self._primary = httpx.AsyncHTTPTransport(**transport_kwargs) self._primary_lock = asyncio.Lock() diff --git a/tools/approval.py b/tools/approval.py index db0595747cd64..774fc978b4309 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -2922,6 +2922,20 @@ def _prompt_dangerous_approval_inner(command: str, description: str, os.environ["HERMES_SPINNER_PAUSE"] = "1" try: + # Disable mouse tracking to prevent mouse micro-movements from + # generating escape sequences that could be captured by input() + # and misinterpreted as user input. + try: + if os.isatty(1): + os.write( + 1, + b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l" + b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l" + b"\x1b[?1004l\x1b[?2029l", + ) + except OSError: + pass + # Resolve the active UI language once per prompt so we don't re-read # config/YAML inside the retry loop below. from agent.i18n import t