This commit is contained in:
GAURAV PATIL 2026-08-14 09:32:28 -04:00 committed by GitHub
commit dccc3e8fbc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 157 additions and 26 deletions

View File

@ -91,6 +91,7 @@ from agent.trajectory import has_incomplete_scratchpad
# finalizer at turn end.
from agent.turn_finalizer import finalize_turn
from agent.usage_pricing import estimate_usage_cost, normalize_usage
from agent.errors import EmptyStreamError
from hermes_constants import PARTIAL_STREAM_STUB_ID
from hermes_logging import set_session_context
from tools.skill_provenance import set_current_write_origin
@ -2812,6 +2813,7 @@ def run_conversation(
elif _model_request_active is not None:
_model_request_active.set()
_redirect_crossed_response = False
streaming_error_handled = False
try:
response = run_llm_execution_middleware(
api_kwargs,
@ -2829,6 +2831,45 @@ def run_conversation(
api_call_count=api_call_count,
middleware_trace=list(_llm_middleware_trace),
)
except (EmptyStreamError, TimeoutError) as e:
# Handle streaming failures (empty stream, stalled stream) gracefully
# instead of letting them propagate and cause deadlock/UI corruption.
logger.warning(
"Streaming API call failed (%s): %s",
type(e).__name__,
e,
)
# Set response to None so the validation logic below treats it as invalid
response = None
response_invalid = True
error_details = [str(e)]
streaming_error_handled = True
# Invoke error hook immediately for streaming errors so retry/fallback logic runs
agent._invoke_api_request_error_hook(
task_id=effective_task_id,
turn_id=turn_id,
api_request_id=api_request_id,
api_call_count=api_call_count,
api_start_time=api_start_time,
api_kwargs=api_kwargs,
error_type=type(e).__name__,
error_message=str(e),
status_code=None,
retry_count=retry_count,
max_retries=max_retries,
retryable=True,
reason="stream_error",
)
except InterruptedError:
# Interrupts are handled by the redirect logic below
raise
except BaseException as e:
# Other exceptions (provider errors, etc.) - let them propagate
# to the outer retry/fallback handler
logger.exception("API call raised unexpected error: %s", e)
response = None
response_invalid = True
error_details = [str(e)]
finally:
if _redirect_lock is not None:
with _redirect_lock:
@ -2876,8 +2917,10 @@ def run_conversation(
logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}")
# Validate response shape before proceeding
response_invalid = False
error_details = []
# response_invalid and error_details may have been set by the exception handler above
if 'response_invalid' not in locals():
response_invalid = False
error_details = []
if agent.api_mode == "codex_responses":
_ct_v = agent._get_transport()
if not _ct_v.validate_response(response):
@ -2956,21 +2999,28 @@ def run_conversation(
error_details.append("response.choices is empty")
if response_invalid:
agent._invoke_api_request_error_hook(
task_id=effective_task_id,
turn_id=turn_id,
api_request_id=api_request_id,
api_call_count=api_call_count,
api_start_time=api_start_time,
api_kwargs=api_kwargs,
error_type="InvalidAPIResponse",
error_message=", ".join(error_details) or "Invalid API response",
status_code=getattr(getattr(response, "error", None), "code", None),
retry_count=retry_count,
max_retries=max_retries,
retryable=True,
reason="invalid_response",
)
# Skip duplicate error handling for streaming errors already handled above
if locals().get('streaming_error_handled', False):
# Streaming error was already handled in the except block
# The error hook was invoked and fallback/retry logic may have been triggered
# Just continue to the retry/fallback logic below
pass
else:
agent._invoke_api_request_error_hook(
task_id=effective_task_id,
turn_id=turn_id,
api_request_id=api_request_id,
api_call_count=api_call_count,
api_start_time=api_start_time,
api_kwargs=api_kwargs,
error_type="InvalidAPIResponse",
error_message=", ".join(error_details) or "Invalid API response",
status_code=getattr(getattr(response, "error", None), "code", None),
retry_count=retry_count,
max_retries=max_retries,
retryable=True,
reason="invalid_response",
)
# Stop spinner silently — retry status is now buffered
# and only surfaced if every retry+fallback exhausts.
if thinking_spinner:

View File

@ -1530,6 +1530,20 @@ def run_curator_review(
if consolidate is None:
consolidate = get_consolidate()
start = datetime.now(timezone.utc)
# CRITICAL FIX: Persist state BEFORE snapshot creation so that if
# snapshot creation causes OOM, the curator won't immediately re-run
# on restart (which would create a vicious cycle of OOM -> restart -> OOM).
# See issue #81625: gateway balloons to 60GB RSS every ~2h.
if not dry_run:
state = load_state()
state["last_run_at"] = start.isoformat()
state["run_count"] = int(state.get("run_count", 0)) + 1
prefix = "auto: "
# We don't have auto_summary yet, use placeholder
state["last_run_summary"] = f"{prefix}starting (snapshot phase)"
save_state(state)
if dry_run:
# Count candidates without mutating state.
try:
@ -1569,16 +1583,16 @@ def run_curator_review(
auto_summary_parts.append(f"{counts['reactivated']} reactivated")
auto_summary = ", ".join(auto_summary_parts) if auto_summary_parts else "no changes"
prefix = "dry-run auto: " if dry_run else "auto: "
# Persist state before the LLM pass so a crash mid-review still records
# the run and doesn't immediately re-trigger. In dry-run we do NOT bump
# last_run_at or run_count — a preview shouldn't push the next scheduled
# real pass out. We still record a summary so `hermes curator status`
# shows that a preview ran.
# NOTE: State was already saved before snapshot creation (above).
# Here we just update the summary with the actual auto_summary.
state = load_state()
if not dry_run:
state["last_run_at"] = start.isoformat()
state["run_count"] = int(state.get("run_count", 0)) + 1
prefix = "dry-run auto: " if dry_run else "auto: "
state["last_run_summary"] = f"{prefix}{auto_summary}"
save_state(state)

29
cli.py
View File

@ -997,6 +997,30 @@ def _mark_tui_input_modes_active() -> None:
_tui_input_modes_active = True
def _suppress_mouse_tracking_cli() -> None:
"""Disable terminal mouse tracking for the classic CLI to prevent
mouse micro-movements from generating escape sequences that could
be misinterpreted as key presses (e.g. triggering 'deny' in approval prompts).
This is similar to _suppress_mouse_residue_early() for the TUI but runs
at the start of the classic CLI run() method.
"""
try:
if not os.isatty(1):
return
# Disable all mouse tracking variants: SGR (1006), X10 (1000), button-event (1002),
# any-event (1003), focus (1004), and alternate scroll (1005/1015/1016)
# Also disable vt200 mouse (1001) and kitty mouse (2029)
os.write(
1,
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l"
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l"
b"\x1b[?1004l\x1b[?2029l",
)
except OSError:
pass
def _prepare_deferred_agent_startup() -> None:
"""Run Termux-deferred agent discovery before the first real agent turn."""
global _deferred_agent_startup_done
@ -15692,6 +15716,11 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
_welcome_color = "#FFF8DC"
self._console_print(f"[{_welcome_color}]{_welcome_text}[/]")
# Disable terminal mouse tracking to prevent mouse micro-movements
# from generating escape sequences that could interfere with
# approval prompts and other modal dialogs.
_suppress_mouse_tracking_cli()
# Warm the /model picker's provider-models cache off-thread during this
# idle window (banner shown, user about to type). The no-args picker
# otherwise blocks ~1-2s on serial /v1/models fetches the first time

View File

@ -361,8 +361,9 @@ def _suppress_mouse_residue_early() -> None:
# safe to send even when no tracking is currently asserted.
os.write(
1,
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l\x1b[?9l"
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l\x1b[?2029l",
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l"
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l"
b"\x1b[?1004l\x1b[?2029l",
)
except OSError:
pass

View File

@ -4142,11 +4142,19 @@ class TelegramAdapter(BasePlatformAdapter):
_transport_kwargs: dict = {}
if _pool_limits is not None:
_transport_kwargs["limits"] = _pool_limits
# Pass connect/read timeouts to the fallback transport so they
# apply at the socket level, preventing indefinite hangs during
# DNS resolution or TCP connect (issue #78586).
connect_timeout = request_kwargs.get("connect_timeout", 10.0)
read_timeout = request_kwargs.get("read_timeout", 20.0)
request = HTTPXRequest(
**request_kwargs,
httpx_kwargs={
"transport": TelegramFallbackTransport(
fallback_ips, **_transport_kwargs
fallback_ips,
connect_timeout=connect_timeout,
read_timeout=read_timeout,
**_transport_kwargs,
)
},
)
@ -4154,7 +4162,10 @@ class TelegramAdapter(BasePlatformAdapter):
**request_kwargs,
httpx_kwargs={
"transport": TelegramFallbackTransport(
fallback_ips, **_transport_kwargs
fallback_ips,
connect_timeout=connect_timeout,
read_timeout=read_timeout,
**_transport_kwargs,
)
},
)

View File

@ -63,12 +63,24 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
# on its own (#63311).
_POOL_LIMITS = httpx.Limits(max_connections=8, max_keepalive_connections=4)
def __init__(self, fallback_ips: Iterable[str], **transport_kwargs):
def __init__(
self,
fallback_ips: Iterable[str],
*,
connect_timeout: float = 10.0,
read_timeout: float = 20.0,
**transport_kwargs,
):
self._fallback_ips = list(dict.fromkeys(_normalize_fallback_ips(fallback_ips)))
proxy_url = _resolve_proxy_url(target_hosts=[_TELEGRAM_API_HOST, *self._fallback_ips])
if proxy_url and "proxy" not in transport_kwargs:
transport_kwargs["proxy"] = proxy_url
transport_kwargs.setdefault("limits", self._POOL_LIMITS)
# Configure timeouts at the transport level so they apply to socket
# connect/read operations directly, preventing indefinite hangs during
# DNS resolution or TCP connect (issue #78586).
timeout = httpx.Timeout(connect=connect_timeout, read=read_timeout)
transport_kwargs["timeout"] = timeout
self._transport_kwargs = transport_kwargs
self._primary = httpx.AsyncHTTPTransport(**transport_kwargs)
self._primary_lock = asyncio.Lock()

View File

@ -2922,6 +2922,20 @@ def _prompt_dangerous_approval_inner(command: str, description: str,
os.environ["HERMES_SPINNER_PAUSE"] = "1"
try:
# Disable mouse tracking to prevent mouse micro-movements from
# generating escape sequences that could be captured by input()
# and misinterpreted as user input.
try:
if os.isatty(1):
os.write(
1,
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l"
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l"
b"\x1b[?1004l\x1b[?2029l",
)
except OSError:
pass
# Resolve the active UI language once per prompt so we don't re-read
# config/YAML inside the retry loop below.
from agent.i18n import t