diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 86f3f5099262d..f064d9451a1de 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -647,6 +647,13 @@ def finalize_turn( } if agent._tool_guardrail_halt_decision is not None: result["guardrail"] = agent._tool_guardrail_halt_decision.to_metadata() + # Persistence failures already set failed=True + an explanation in + # final_response; also stamp `error` so gateway surfaces status="error" + # (and desktop can toast disk-full) instead of a quiet complete frame. + if failed and str(_turn_exit_reason) == "session_persistence_failed": + result["error"] = final_response or ( + "session storage could not be written — free disk space and try again" + ) # Surface any post-loop cleanup failures so the caller can distinguish a # clean turn from one whose trajectory/session/resource teardown raised # (the response is still returned either way — #8049). diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts index 7897c437d49ee..c83320f7c2ccf 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts @@ -34,7 +34,7 @@ import { setChangeEventsAvailable } from '@/store/live-sync' import { dispatchNativeNotification } from '@/store/native-notifications' -import { notify } from '@/store/notifications' +import { isDiskFullErrorMessage, notify, notifyError } from '@/store/notifications' import { requestDesktopOnboarding, requestDesktopOnboardingForCredentialWarning } from '@/store/onboarding' import { revealDesktopPane } from '@/store/pane-focus' import { flashPetActivity, markPetUnread, setPetActivity } from '@/store/pet' @@ -1161,6 +1161,8 @@ export function useGatewayEventHandler(deps: GatewayEventDeps) { if (looksLikeProviderSetup) { requestDesktopOnboarding(errorMessage) + } else if (isDiskFullErrorMessage(errorMessage)) { + notifyError(new Error(errorMessage), translateNow('notifications.errors.diskFull')) } else { // Toast globally, not just when the failing thread is focused: a // turn-ending error (e.g. out of funds) blocks every thread, so the diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/index.ts b/apps/desktop/src/app/session/hooks/use-message-stream/index.ts index 7aa89a8cf14f8..61469c0df9c85 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/index.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/index.ts @@ -22,6 +22,7 @@ import { } from '@/lib/generated-images' import { parseTodos } from '@/lib/todos' import { dispatchNativeNotification } from '@/store/native-notifications' +import { isDiskFullErrorMessage, notifyError } from '@/store/notifications' import { broadcastSessionsChanged } from '@/store/session-sync' import { upsertSubagent } from '@/store/subagents' import { setSessionTodos } from '@/store/todos' @@ -599,6 +600,16 @@ export function useMessageStream({ } }) + // Persistence / mid-turn disk-full failures land as a terminal frame with + // an error string, not a rejected prompt.submit. Toast them here so a + // full disk never looks like a silent no-reply. Only fire on actual + // failure signals — never on a healthy reply that happens to say + // "disk full". + const diskFullSignal = failure?.error || (failure ? text : '') + if (diskFullSignal && isDiskFullErrorMessage(diskFullSignal)) { + notifyError(new Error(diskFullSignal), translateNow('notifications.errors.diskFull')) + } + scheduleSessionsRefresh() if (compactedTurnRef.current.delete(sessionId)) { diff --git a/apps/desktop/src/i18n/ar.ts b/apps/desktop/src/i18n/ar.ts index 0f37ba6d104eb..67e9aab5a4ec2 100644 --- a/apps/desktop/src/i18n/ar.ts +++ b/apps/desktop/src/i18n/ar.ts @@ -117,6 +117,7 @@ export const ar = defineLocale({ errors: { elevenLabsNeedsKey: 'يتطلب ElevenLabs STT المفتاح ELEVENLABS_API_KEY.', elevenLabsRejectedKey: 'رفض ElevenLabs مفتاح API (401).', + diskFull: 'القرص ممتلئ — حرّر مساحة ثم أعد المحاولة.', methodNotAllowed: 'رفضت خلفية سطح المكتب هذا الطلب (405 Method Not Allowed). جرب إعادة تشغيل Hermes Desktop.', microphonePermission: 'تم رفض إذن الميكروفون.', openaiRejectedApiKey: 'رفض OpenAI مفتاح API.', diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index 6733eb49a5d05..fe1da62d0fe3f 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -132,6 +132,7 @@ export const en: Translations = { errors: { elevenLabsNeedsKey: 'ElevenLabs STT needs ELEVENLABS_API_KEY.', elevenLabsRejectedKey: 'ElevenLabs rejected the API key (401).', + diskFull: 'Disk full — free some space, then try again.', gatewayAuthFailed: 'Gateway authentication failed — check your API_SERVER_KEY.', methodNotAllowed: 'The desktop backend rejected that request (405 Method Not Allowed). Try restarting Hermes Desktop.', diff --git a/apps/desktop/src/i18n/ja.ts b/apps/desktop/src/i18n/ja.ts index fab8e071bf78b..4a31419cce32d 100644 --- a/apps/desktop/src/i18n/ja.ts +++ b/apps/desktop/src/i18n/ja.ts @@ -133,6 +133,7 @@ export const ja = defineLocale({ errors: { elevenLabsNeedsKey: 'ElevenLabs STT には ELEVENLABS_API_KEY が必要です。', elevenLabsRejectedKey: 'ElevenLabs が API キーを拒否しました (401)。', + diskFull: 'ディスク容量不足です — 空きを作ってからもう一度お試しください。', gatewayAuthFailed: 'ゲートウェイ認証に失敗しました — API_SERVER_KEY を確認してください。', methodNotAllowed: 'デスクトップバックエンドがそのリクエストを拒否しました (405 Method Not Allowed)。Hermes Desktop を再起動してください。', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 88f3ffdf95a5c..303bd421c3139 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -174,6 +174,7 @@ export interface Translations { errors: { elevenLabsNeedsKey: string elevenLabsRejectedKey: string + diskFull: string gatewayAuthFailed: string methodNotAllowed: string microphonePermission: string diff --git a/apps/desktop/src/i18n/zh-hant.ts b/apps/desktop/src/i18n/zh-hant.ts index 8a808dffec85d..2b93bd09bea74 100644 --- a/apps/desktop/src/i18n/zh-hant.ts +++ b/apps/desktop/src/i18n/zh-hant.ts @@ -129,6 +129,7 @@ export const zhHant = defineLocale({ errors: { elevenLabsNeedsKey: 'ElevenLabs STT 需要 ELEVENLABS_API_KEY。', elevenLabsRejectedKey: 'ElevenLabs 拒絕了該 API 金鑰 (401)。', + diskFull: '磁碟已滿 — 請騰出一些空間後再試。', gatewayAuthFailed: '閘道認證失敗 — 請檢查你的 API_SERVER_KEY。', methodNotAllowed: '桌面後端拒絕了該請求 (405 Method Not Allowed)。請嘗試重新啟動 Hermes Desktop。', microphonePermission: '麥克風權限已被拒絕。', diff --git a/apps/desktop/src/i18n/zh.ts b/apps/desktop/src/i18n/zh.ts index c34d811344e3b..4f206a2b2319a 100644 --- a/apps/desktop/src/i18n/zh.ts +++ b/apps/desktop/src/i18n/zh.ts @@ -129,6 +129,7 @@ export const zh: Translations = { errors: { elevenLabsNeedsKey: 'ElevenLabs STT 需要 ELEVENLABS_API_KEY。', elevenLabsRejectedKey: 'ElevenLabs 拒绝了该 API key (401)。', + diskFull: '磁盘已满 — 请腾出一些空间后再试。', gatewayAuthFailed: '网关认证失败 — 请检查你的 API_SERVER_KEY。', methodNotAllowed: '桌面后端拒绝了该请求 (405 Method Not Allowed)。请尝试重启 Hermes Desktop。', microphonePermission: '麦克风权限已被拒绝。', diff --git a/apps/desktop/src/store/notifications.test.ts b/apps/desktop/src/store/notifications.test.ts index 49cb0b1024f61..e11ed82b4ca88 100644 --- a/apps/desktop/src/store/notifications.test.ts +++ b/apps/desktop/src/store/notifications.test.ts @@ -1,6 +1,6 @@ import { beforeEach, expect, test } from 'vitest' -import { $notifications, clearNotifications, notifyError } from './notifications' +import { $notifications, clearNotifications, isDiskFullErrorMessage, notifyError } from './notifications' beforeEach(() => { clearNotifications() @@ -32,3 +32,26 @@ test('provider invalid_api_key error still maps to the OpenAI summary', () => { expect(lastMessage()).toMatch(/OpenAI rejected the API key/i) }) + +test('disk-full / ENOSPC errors toast a free-space message', () => { + expect(isDiskFullErrorMessage('OSError: [Errno 28] No space left on device')).toBe(true) + expect(isDiskFullErrorMessage('sqlite3.OperationalError: database or disk is full')).toBe(true) + expect(isDiskFullErrorMessage('disk full: session storage could not be written — free some disk space')).toBe(true) + expect(isDiskFullErrorMessage('This is often a full disk — free some space')).toBe(true) + expect(isDiskFullErrorMessage('session storage could not be written: permission denied')).toBe(false) + expect(isDiskFullErrorMessage('network timeout')).toBe(false) + + notifyError(new Error('OSError: [Errno 28] No space left on device: state.db'), 'Prompt failed') + + expect(lastMessage()).toMatch(/Disk full/i) + expect(lastMessage()).toMatch(/free some space/i) +}) + +test('session storage write failure is treated as disk-full class', () => { + notifyError( + new Error('disk full: session storage could not be written — free some disk space and try again'), + 'Prompt failed' + ) + + expect(lastMessage()).toMatch(/Disk full/i) +}) diff --git a/apps/desktop/src/store/notifications.ts b/apps/desktop/src/store/notifications.ts index b0f8bb752c47a..3709b8fe6fe1e 100644 --- a/apps/desktop/src/store/notifications.ts +++ b/apps/desktop/src/store/notifications.ts @@ -76,7 +76,27 @@ function cleanErrorText(value: string) { return value.replace(/^Error:\s*/, '').trim() } +/** True when an error string is a disk-full / ENOSPC / SQLITE_FULL failure. */ +export function isDiskFullErrorMessage(message: string): boolean { + return ( + /no space left on device/i.test(message) || + /not enough space/i.test(message) || + /database or disk is full/i.test(message) || + /\bENOSPC\b/i.test(message) || + /disk full/i.test(message) || + /full disk/i.test(message) + ) +} + const ERROR_SUMMARIES: { test: (msg: string) => boolean; summarize: (msg: string) => string }[] = [ + { + // Disk full / ENOSPC — session DB write, backend crash, or any path that + // bubbles "no space left" / SQLITE_FULL through notifyError. Match before + // generic length truncation so the user gets a clear "free space" toast + // instead of a silent send or a raw errno dump. + test: isDiskFullErrorMessage, + summarize: () => translateNow('notifications.errors.diskFull') + }, { test: msg => /['"]code['"]\s*:\s*['"]gateway_auth_failed['"]/i.test(msg), summarize: () => translateNow('notifications.errors.gatewayAuthFailed') diff --git a/hermes_state.py b/hermes_state.py index 077d2aa512e20..0c1f362c5c8da 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -16,6 +16,7 @@ Key design decisions: import asyncio import atexit +import errno import json import logging import os @@ -939,6 +940,36 @@ def is_malformed_db_error(exc: BaseException) -> bool: return any(marker in str(exc).lower() for marker in _MALFORMED_SCHEMA_MARKERS) +# Markers that mean the host filesystem cannot accept another write. Kept as +# plain substrings so OSError, sqlite3.OperationalError, and wrapped RPC +# error strings all match the same helper. +_DISK_FULL_MARKERS = ( + "no space left on device", + "not enough space", + "database or disk is full", # SQLITE_FULL + "disk full", + "full disk", + "enospc", +) + + +def is_disk_full_error(exc: BaseException | str | None) -> bool: + """True when *exc* (or a stringified error) is a disk-full / ENOSPC failure. + + Covers: + * ``OSError`` with ``errno.ENOSPC`` + * SQLite ``OperationalError: database or disk is full`` (SQLITE_FULL) + * Plain English / errno strings that survive RPC wrapping + """ + if exc is None: + return False + if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.ENOSPC: + return True + text = exc if isinstance(exc, str) else str(exc) + lowered = text.lower() + return any(marker in lowered for marker in _DISK_FULL_MARKERS) + + def _claim_repair_attempt(db_path: Path) -> bool: """Claim the one-shot repair attempt for *db_path* in this process. diff --git a/run_agent.py b/run_agent.py index e574b28ff23cc..c4241a83ed751 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3530,8 +3530,8 @@ class AIAgent: prefix + "the turn was stopped because session storage could not be " "written (the transcript would have been lost on restart). " - "Check disk space / permissions for the state DB, then send " - "your message again." + "This is often a full disk — free some space (or fix state.db " + "permissions), then send your message again." ) # Unknown/diagnostic-only reasons (e.g. "unknown", guardrail_halt # which already surfaces its own message) — don't second-guess. diff --git a/tests/state/test_disk_full_error.py b/tests/state/test_disk_full_error.py new file mode 100644 index 0000000000000..ac4e11738c212 --- /dev/null +++ b/tests/state/test_disk_full_error.py @@ -0,0 +1,30 @@ +"""is_disk_full_error classifies ENOSPC / SQLITE_FULL failures.""" + +from __future__ import annotations + +import errno +import sqlite3 + +from hermes_state import is_disk_full_error + + +def test_enospc_oserror(): + assert is_disk_full_error(OSError(errno.ENOSPC, "No space left on device")) is True + + +def test_sqlite_full_operational_error(): + assert is_disk_full_error(sqlite3.OperationalError("database or disk is full")) is True + + +def test_string_markers(): + assert is_disk_full_error("disk full: session storage could not be written") is True + assert is_disk_full_error("ENOSPC writing state.db") is True + assert is_disk_full_error("This is often a full disk — free some space") is True + + +def test_unrelated_errors(): + assert is_disk_full_error(None) is False + assert is_disk_full_error(OSError(errno.EACCES, "Permission denied")) is False + assert is_disk_full_error(RuntimeError("network timeout")) is False + assert is_disk_full_error("session not found") is False + assert is_disk_full_error("session storage could not be written: permission denied") is False diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 098b6aa4c2274..8983070bbd0e1 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -233,10 +233,32 @@ def _(rid, params: dict) -> dict: ) # Persist the DB row lazily, now that the user has actually sent a message. - _ensure_session_db_row(session) - # A branch becomes real here: copy its parent's transcript into the row so it - # resumes with full context (the agent won't persist the seed itself). - _persist_branch_seed(session) + # Disk-full must fail the RPC (not stream silently): desktop maps the error + # string to a "disk full" toast so the user knows why the send vanished. + try: + _ensure_session_db_row(session) + # A branch becomes real here: copy its parent's transcript into the row so it + # resumes with full context (the agent won't persist the seed itself). + _persist_branch_seed(session) + except Exception as exc: + from hermes_state import is_disk_full_error + + with session["history_lock"]: + session["running"] = False + session["last_active"] = time.time() + _clear_inflight_turn(session) + if is_disk_full_error(exc): + return _err( + rid, + 5070, + "disk full: session storage could not be written — free some disk space and try again", + ) + logger.warning("prompt.submit: session persist failed: %s", exc, exc_info=True) + return _err( + rid, + 5071, + f"session storage could not be written: {exc}", + ) _start_agent_build(sid, session) def run_after_agent_ready() -> None: diff --git a/tui_gateway/server.py b/tui_gateway/server.py index b399992a20be4..f991716706859 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -2518,7 +2518,14 @@ def _ensure_session_db_row(session: dict) -> None: # means the launch/default profile (matches run_agent's convention). profile_name=Path(profile_home).name if profile_home else None, ) - except Exception: + except Exception as exc: + # Disk-full is not a soft failure: if we swallow it here, prompt.submit + # returns {"status":"streaming"} and the user's message vanishes with + # no toast. Re-raise so the submit handler can return a real RPC error. + from hermes_state import is_disk_full_error + + if is_disk_full_error(exc): + raise logger.debug("failed to persist desktop session row", exc_info=True) finally: if close_db: @@ -2561,7 +2568,11 @@ def _persist_branch_seed(session: dict) -> None: timestamp=msg.get("timestamp"), ) session["_branch_seed_persisted"] = True - except Exception: + except Exception as exc: + from hermes_state import is_disk_full_error + + if is_disk_full_error(exc): + raise logger.debug("branch seed persist failed", exc_info=True)