Merge fa71a16b1d into c896c09c42
This commit is contained in:
commit
dccc3e8fbc
|
|
@ -91,6 +91,7 @@ from agent.trajectory import has_incomplete_scratchpad
|
|||
# finalizer at turn end.
|
||||
from agent.turn_finalizer import finalize_turn
|
||||
from agent.usage_pricing import estimate_usage_cost, normalize_usage
|
||||
from agent.errors import EmptyStreamError
|
||||
from hermes_constants import PARTIAL_STREAM_STUB_ID
|
||||
from hermes_logging import set_session_context
|
||||
from tools.skill_provenance import set_current_write_origin
|
||||
|
|
@ -2812,6 +2813,7 @@ def run_conversation(
|
|||
elif _model_request_active is not None:
|
||||
_model_request_active.set()
|
||||
_redirect_crossed_response = False
|
||||
streaming_error_handled = False
|
||||
try:
|
||||
response = run_llm_execution_middleware(
|
||||
api_kwargs,
|
||||
|
|
@ -2829,6 +2831,45 @@ def run_conversation(
|
|||
api_call_count=api_call_count,
|
||||
middleware_trace=list(_llm_middleware_trace),
|
||||
)
|
||||
except (EmptyStreamError, TimeoutError) as e:
|
||||
# Handle streaming failures (empty stream, stalled stream) gracefully
|
||||
# instead of letting them propagate and cause deadlock/UI corruption.
|
||||
logger.warning(
|
||||
"Streaming API call failed (%s): %s",
|
||||
type(e).__name__,
|
||||
e,
|
||||
)
|
||||
# Set response to None so the validation logic below treats it as invalid
|
||||
response = None
|
||||
response_invalid = True
|
||||
error_details = [str(e)]
|
||||
streaming_error_handled = True
|
||||
# Invoke error hook immediately for streaming errors so retry/fallback logic runs
|
||||
agent._invoke_api_request_error_hook(
|
||||
task_id=effective_task_id,
|
||||
turn_id=turn_id,
|
||||
api_request_id=api_request_id,
|
||||
api_call_count=api_call_count,
|
||||
api_start_time=api_start_time,
|
||||
api_kwargs=api_kwargs,
|
||||
error_type=type(e).__name__,
|
||||
error_message=str(e),
|
||||
status_code=None,
|
||||
retry_count=retry_count,
|
||||
max_retries=max_retries,
|
||||
retryable=True,
|
||||
reason="stream_error",
|
||||
)
|
||||
except InterruptedError:
|
||||
# Interrupts are handled by the redirect logic below
|
||||
raise
|
||||
except BaseException as e:
|
||||
# Other exceptions (provider errors, etc.) - let them propagate
|
||||
# to the outer retry/fallback handler
|
||||
logger.exception("API call raised unexpected error: %s", e)
|
||||
response = None
|
||||
response_invalid = True
|
||||
error_details = [str(e)]
|
||||
finally:
|
||||
if _redirect_lock is not None:
|
||||
with _redirect_lock:
|
||||
|
|
@ -2876,8 +2917,10 @@ def run_conversation(
|
|||
logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}")
|
||||
|
||||
# Validate response shape before proceeding
|
||||
response_invalid = False
|
||||
error_details = []
|
||||
# response_invalid and error_details may have been set by the exception handler above
|
||||
if 'response_invalid' not in locals():
|
||||
response_invalid = False
|
||||
error_details = []
|
||||
if agent.api_mode == "codex_responses":
|
||||
_ct_v = agent._get_transport()
|
||||
if not _ct_v.validate_response(response):
|
||||
|
|
@ -2956,21 +2999,28 @@ def run_conversation(
|
|||
error_details.append("response.choices is empty")
|
||||
|
||||
if response_invalid:
|
||||
agent._invoke_api_request_error_hook(
|
||||
task_id=effective_task_id,
|
||||
turn_id=turn_id,
|
||||
api_request_id=api_request_id,
|
||||
api_call_count=api_call_count,
|
||||
api_start_time=api_start_time,
|
||||
api_kwargs=api_kwargs,
|
||||
error_type="InvalidAPIResponse",
|
||||
error_message=", ".join(error_details) or "Invalid API response",
|
||||
status_code=getattr(getattr(response, "error", None), "code", None),
|
||||
retry_count=retry_count,
|
||||
max_retries=max_retries,
|
||||
retryable=True,
|
||||
reason="invalid_response",
|
||||
)
|
||||
# Skip duplicate error handling for streaming errors already handled above
|
||||
if locals().get('streaming_error_handled', False):
|
||||
# Streaming error was already handled in the except block
|
||||
# The error hook was invoked and fallback/retry logic may have been triggered
|
||||
# Just continue to the retry/fallback logic below
|
||||
pass
|
||||
else:
|
||||
agent._invoke_api_request_error_hook(
|
||||
task_id=effective_task_id,
|
||||
turn_id=turn_id,
|
||||
api_request_id=api_request_id,
|
||||
api_call_count=api_call_count,
|
||||
api_start_time=api_start_time,
|
||||
api_kwargs=api_kwargs,
|
||||
error_type="InvalidAPIResponse",
|
||||
error_message=", ".join(error_details) or "Invalid API response",
|
||||
status_code=getattr(getattr(response, "error", None), "code", None),
|
||||
retry_count=retry_count,
|
||||
max_retries=max_retries,
|
||||
retryable=True,
|
||||
reason="invalid_response",
|
||||
)
|
||||
# Stop spinner silently — retry status is now buffered
|
||||
# and only surfaced if every retry+fallback exhausts.
|
||||
if thinking_spinner:
|
||||
|
|
|
|||
|
|
@ -1530,6 +1530,20 @@ def run_curator_review(
|
|||
if consolidate is None:
|
||||
consolidate = get_consolidate()
|
||||
start = datetime.now(timezone.utc)
|
||||
|
||||
# CRITICAL FIX: Persist state BEFORE snapshot creation so that if
|
||||
# snapshot creation causes OOM, the curator won't immediately re-run
|
||||
# on restart (which would create a vicious cycle of OOM -> restart -> OOM).
|
||||
# See issue #81625: gateway balloons to 60GB RSS every ~2h.
|
||||
if not dry_run:
|
||||
state = load_state()
|
||||
state["last_run_at"] = start.isoformat()
|
||||
state["run_count"] = int(state.get("run_count", 0)) + 1
|
||||
prefix = "auto: "
|
||||
# We don't have auto_summary yet, use placeholder
|
||||
state["last_run_summary"] = f"{prefix}starting (snapshot phase)"
|
||||
save_state(state)
|
||||
|
||||
if dry_run:
|
||||
# Count candidates without mutating state.
|
||||
try:
|
||||
|
|
@ -1569,16 +1583,16 @@ def run_curator_review(
|
|||
auto_summary_parts.append(f"{counts['reactivated']} reactivated")
|
||||
auto_summary = ", ".join(auto_summary_parts) if auto_summary_parts else "no changes"
|
||||
|
||||
prefix = "dry-run auto: " if dry_run else "auto: "
|
||||
|
||||
# Persist state before the LLM pass so a crash mid-review still records
|
||||
# the run and doesn't immediately re-trigger. In dry-run we do NOT bump
|
||||
# last_run_at or run_count — a preview shouldn't push the next scheduled
|
||||
# real pass out. We still record a summary so `hermes curator status`
|
||||
# shows that a preview ran.
|
||||
# NOTE: State was already saved before snapshot creation (above).
|
||||
# Here we just update the summary with the actual auto_summary.
|
||||
state = load_state()
|
||||
if not dry_run:
|
||||
state["last_run_at"] = start.isoformat()
|
||||
state["run_count"] = int(state.get("run_count", 0)) + 1
|
||||
prefix = "dry-run auto: " if dry_run else "auto: "
|
||||
state["last_run_summary"] = f"{prefix}{auto_summary}"
|
||||
save_state(state)
|
||||
|
||||
|
|
|
|||
29
cli.py
29
cli.py
|
|
@ -997,6 +997,30 @@ def _mark_tui_input_modes_active() -> None:
|
|||
_tui_input_modes_active = True
|
||||
|
||||
|
||||
def _suppress_mouse_tracking_cli() -> None:
|
||||
"""Disable terminal mouse tracking for the classic CLI to prevent
|
||||
mouse micro-movements from generating escape sequences that could
|
||||
be misinterpreted as key presses (e.g. triggering 'deny' in approval prompts).
|
||||
|
||||
This is similar to _suppress_mouse_residue_early() for the TUI but runs
|
||||
at the start of the classic CLI run() method.
|
||||
"""
|
||||
try:
|
||||
if not os.isatty(1):
|
||||
return
|
||||
# Disable all mouse tracking variants: SGR (1006), X10 (1000), button-event (1002),
|
||||
# any-event (1003), focus (1004), and alternate scroll (1005/1015/1016)
|
||||
# Also disable vt200 mouse (1001) and kitty mouse (2029)
|
||||
os.write(
|
||||
1,
|
||||
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l"
|
||||
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l"
|
||||
b"\x1b[?1004l\x1b[?2029l",
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _prepare_deferred_agent_startup() -> None:
|
||||
"""Run Termux-deferred agent discovery before the first real agent turn."""
|
||||
global _deferred_agent_startup_done
|
||||
|
|
@ -15692,6 +15716,11 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
|||
_welcome_color = "#FFF8DC"
|
||||
self._console_print(f"[{_welcome_color}]{_welcome_text}[/]")
|
||||
|
||||
# Disable terminal mouse tracking to prevent mouse micro-movements
|
||||
# from generating escape sequences that could interfere with
|
||||
# approval prompts and other modal dialogs.
|
||||
_suppress_mouse_tracking_cli()
|
||||
|
||||
# Warm the /model picker's provider-models cache off-thread during this
|
||||
# idle window (banner shown, user about to type). The no-args picker
|
||||
# otherwise blocks ~1-2s on serial /v1/models fetches the first time
|
||||
|
|
|
|||
|
|
@ -361,8 +361,9 @@ def _suppress_mouse_residue_early() -> None:
|
|||
# safe to send even when no tracking is currently asserted.
|
||||
os.write(
|
||||
1,
|
||||
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l\x1b[?9l"
|
||||
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l\x1b[?2029l",
|
||||
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l"
|
||||
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l"
|
||||
b"\x1b[?1004l\x1b[?2029l",
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -4142,11 +4142,19 @@ class TelegramAdapter(BasePlatformAdapter):
|
|||
_transport_kwargs: dict = {}
|
||||
if _pool_limits is not None:
|
||||
_transport_kwargs["limits"] = _pool_limits
|
||||
# Pass connect/read timeouts to the fallback transport so they
|
||||
# apply at the socket level, preventing indefinite hangs during
|
||||
# DNS resolution or TCP connect (issue #78586).
|
||||
connect_timeout = request_kwargs.get("connect_timeout", 10.0)
|
||||
read_timeout = request_kwargs.get("read_timeout", 20.0)
|
||||
request = HTTPXRequest(
|
||||
**request_kwargs,
|
||||
httpx_kwargs={
|
||||
"transport": TelegramFallbackTransport(
|
||||
fallback_ips, **_transport_kwargs
|
||||
fallback_ips,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
**_transport_kwargs,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
|
@ -4154,7 +4162,10 @@ class TelegramAdapter(BasePlatformAdapter):
|
|||
**request_kwargs,
|
||||
httpx_kwargs={
|
||||
"transport": TelegramFallbackTransport(
|
||||
fallback_ips, **_transport_kwargs
|
||||
fallback_ips,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
**_transport_kwargs,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -63,12 +63,24 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
|
|||
# on its own (#63311).
|
||||
_POOL_LIMITS = httpx.Limits(max_connections=8, max_keepalive_connections=4)
|
||||
|
||||
def __init__(self, fallback_ips: Iterable[str], **transport_kwargs):
|
||||
def __init__(
|
||||
self,
|
||||
fallback_ips: Iterable[str],
|
||||
*,
|
||||
connect_timeout: float = 10.0,
|
||||
read_timeout: float = 20.0,
|
||||
**transport_kwargs,
|
||||
):
|
||||
self._fallback_ips = list(dict.fromkeys(_normalize_fallback_ips(fallback_ips)))
|
||||
proxy_url = _resolve_proxy_url(target_hosts=[_TELEGRAM_API_HOST, *self._fallback_ips])
|
||||
if proxy_url and "proxy" not in transport_kwargs:
|
||||
transport_kwargs["proxy"] = proxy_url
|
||||
transport_kwargs.setdefault("limits", self._POOL_LIMITS)
|
||||
# Configure timeouts at the transport level so they apply to socket
|
||||
# connect/read operations directly, preventing indefinite hangs during
|
||||
# DNS resolution or TCP connect (issue #78586).
|
||||
timeout = httpx.Timeout(connect=connect_timeout, read=read_timeout)
|
||||
transport_kwargs["timeout"] = timeout
|
||||
self._transport_kwargs = transport_kwargs
|
||||
self._primary = httpx.AsyncHTTPTransport(**transport_kwargs)
|
||||
self._primary_lock = asyncio.Lock()
|
||||
|
|
|
|||
|
|
@ -2922,6 +2922,20 @@ def _prompt_dangerous_approval_inner(command: str, description: str,
|
|||
|
||||
os.environ["HERMES_SPINNER_PAUSE"] = "1"
|
||||
try:
|
||||
# Disable mouse tracking to prevent mouse micro-movements from
|
||||
# generating escape sequences that could be captured by input()
|
||||
# and misinterpreted as user input.
|
||||
try:
|
||||
if os.isatty(1):
|
||||
os.write(
|
||||
1,
|
||||
b"\x1b[?1006l\x1b[?1005l\x1b[?1015l\x1b[?1016l"
|
||||
b"\x1b[?1003l\x1b[?1002l\x1b[?1001l\x1b[?1000l"
|
||||
b"\x1b[?1004l\x1b[?2029l",
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Resolve the active UI language once per prompt so we don't re-read
|
||||
# config/YAML inside the retry loop below.
|
||||
from agent.i18n import t
|
||||
|
|
|
|||
Loading…
Reference in New Issue