diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 485549fa4a811..b0e47eee60901 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -4780,38 +4780,85 @@ def _clear_bytecode_cache(root: Path) -> int: return removed -_UPDATE_RUNTIME_RELOAD_MODULES = ( - "hermes_constants", - "tools.environments.local", - "tools.lazy_deps", +# Update pipeline extracted to hermes_cli/update_cmd.py (main.py decomposition, +# mechanical move). Every moved name is re-exported here so the argparse wiring +# and existing test monkeypatches (hermes_cli.main._cmd_update_impl, +# hermes_cli.main._run_pre_update_backup, ...) keep resolving unchanged. +from hermes_cli.update_cmd import ( # noqa: F401 + _add_upstream_remote, + _atomic_replace_dir, + _capture_head_sha, + _cmd_update_check, + _cmd_update_impl, + _cold_start_windows_gateway_after_update, + _count_commits_between, + _detect_venv_python_processes, + _discard_lockfile_churn, + _discard_stashed_changes, + _ensure_acp_launcher, + _ensure_fhs_path_guard, + _ensure_uv_for_termux, + _finish_dashboard_update_cleanup, + _for_each_systemd_gateway_unit, + _format_concurrent_instances_message, + _format_time_ago, + _format_venv_python_holders_message, + _gateway_prompt, + _get_origin_url, + _has_upstream_remote, + _install_psutil_android_compat, + _invalidate_update_cache, + _is_android_python, + _is_fork, + _log_only_write, + _mark_skip_upstream_prompt, + _npm_bin_exists, + _npm_lockfile_changed, + _npm_manifest_paths, + _npm_manifests_digest, + _pause_windows_gateways_for_update, + _print_curator_first_run_notice, + _print_curator_recent_run_notice, + _print_fts_optimize_available_notice, + _print_stash_cleanup_guidance, + _record_npm_lockfile_hash, + _refresh_active_lazy_features, + _refresh_active_memory_provider_dependencies, + _refresh_windows_gateway_launchers, + _reload_updated_runtime_modules, + _resolve_pre_update_backup_mode, + _resolve_stash_selector, + _restore_stashed_changes, + _resume_windows_gateways_after_update, + _run_logged_subprocess, + _run_pre_update_backup, + _should_skip_upstream_prompt, + _stash_apply_failed_only_on_existing_untracked, + _stash_local_changes_if_needed, + _sync_fork_with_upstream, + _sync_with_upstream_if_needed, + _update_node_dependencies, + _update_via_zip, + _upgrade_pip_before_lazy_refresh, + _validate_critical_files_syntax, + _venv_core_imports_healthy, + _wait_for_windows_update_gateway_exit, + _warn_incomplete_gateway_fleet_restart, + _web_build_toolchain_ready, + _web_toolchain_roots, + _write_lazy_refresh_incomplete_marker, + _write_marker_file, + _write_update_incomplete_marker, + _write_update_planned_stop_marker, + _UPDATE_RUNTIME_RELOAD_MODULES, + _UPDATE_CRITICAL_FILES, + OFFICIAL_REPO_URLS, + OFFICIAL_REPO_URL, + SKIP_UPSTREAM_PROMPT_FILE, + _PRE_UPDATE_SNAPSHOT_KEEP, + _PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, ) - -def _reload_updated_runtime_modules() -> None: - """Reload update-sensitive modules after the checkout changes in-place. - - ``hermes update`` keeps running in the pre-pull Python process. After a - large update, modules already present in ``sys.modules`` can still expose - old symbols even though their source files on disk are new. Refresh the - small module set used by lazy-backend refresh before that step imports - newly-updated code paths. - """ - try: - import importlib - - importlib.invalidate_caches() - for module_name in _UPDATE_RUNTIME_RELOAD_MODULES: - module = sys.modules.get(module_name) - if module is None: - continue - try: - importlib.reload(module) - except Exception as exc: - logger.debug("Could not reload updated module %s: %s", module_name, exc) - except Exception as exc: - logger.debug("Could not refresh update runtime modules: %s", exc) - - # Stamp file recording the checkout fingerprint the bytecode cache was last # validated against. Lives next to the checkout (NOT in HERMES_HOME) because # __pycache__ is per-checkout state shared by every profile. @@ -4882,132 +4929,6 @@ def _sweep_stale_bytecode_if_checkout_changed() -> None: logger.debug("Stale-bytecode launch sweep failed: %s", exc) -# Critical files that Hermes must be able to import immediately after an -# update/install. Most are imported on every CLI startup; ``web_server.py`` -# is the desktop/dashboard backend path that a fresh Windows install launches -# right away. If any of these fail to parse after a pull, the user can be -# left with a bricked CLI or desktop backend. The post-pull syntax guard -# validates these and auto-rolls-back on failure. -_UPDATE_CRITICAL_FILES = ( - "hermes_cli/main.py", - "hermes_cli/config.py", - "hermes_cli/__init__.py", - "hermes_cli/web_server.py", - "cli.py", - "run_agent.py", - "model_tools.py", - "toolsets.py", - "hermes_constants.py", -) - - -def _capture_head_sha(git_cmd, cwd) -> str | None: - """Return the current HEAD SHA, or None if it can't be resolved.""" - try: - result = subprocess.run( - git_cmd + ["rev-parse", "HEAD"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=True, - ) - return result.stdout.strip() or None - except (subprocess.CalledProcessError, OSError): - return None - - -def _validate_critical_files_syntax(root) -> tuple[bool, str | None, str | None]: - """Compile each file in ``_UPDATE_CRITICAL_FILES`` to catch SyntaxErrors. - - These are the files imported on every ``hermes`` startup; if any of them - has a syntax error (orphan merge-conflict markers, bad ref to a name - that no longer exists, etc.) the CLI can't bootstrap at all. We validate - them after a successful ``git pull`` so we can auto-roll-back instead of - leaving the user with a bricked install. - - The compiled ``.pyc`` is written to a temp directory rather than the - source tree's ``__pycache__/`` so we don't race with concurrent test - workers that walk the same dir, and so we don't leave a stale pyc - behind in production if the next interpreter run picks a different - Python version. The pyc is discarded on function return either way — - we only care about the compile-or-not signal. - - Returns ``(ok, failing_path, error_message)``. ``ok=True`` means every - file parsed cleanly. - """ - import py_compile - import tempfile - - root = Path(root) - with tempfile.TemporaryDirectory(prefix="hermes-syntax-check-") as tmpdir: - for relpath in _UPDATE_CRITICAL_FILES: - path = root / relpath - if not path.exists(): - # Missing file is suspicious but not necessarily fatal — a future - # refactor may legitimately remove one of these. Skip and move on. - continue - # Mirror the relative path under the tmpdir so two different - # files with the same basename don't collide on the cfile name. - cfile = Path(tmpdir) / (relpath.replace("/", "__") + "c") - try: - py_compile.compile(str(path), cfile=str(cfile), doraise=True) - except py_compile.PyCompileError as exc: - return False, str(path), str(exc) - except OSError as exc: - return False, str(path), f"could not read: {exc}" - return True, None, None - - -def _gateway_prompt(prompt_text: str, default: str = "", timeout: float = 300.0) -> str: - """File-based IPC prompt for gateway mode. - - Writes a prompt marker file so the gateway can forward the question to the - user, then polls for a response file. Falls back to *default* on timeout. - - Used by ``hermes update --gateway`` so interactive prompts (stash restore, - config migration) are forwarded to the messenger instead of being silently - skipped. - """ - import json as _json - import uuid as _uuid - from hermes_constants import get_hermes_home - - home = get_hermes_home() - prompt_path = home / ".update_prompt.json" - response_path = home / ".update_response" - - # Clean any stale response file - response_path.unlink(missing_ok=True) - - payload = { - "prompt": prompt_text, - "default": default, - "id": str(_uuid.uuid4()), - } - tmp = prompt_path.with_suffix(".tmp") - tmp.write_text(_json.dumps(payload), encoding="utf-8") - tmp.replace(prompt_path) - - # Poll for response - deadline = _time.monotonic() + timeout - while _time.monotonic() < deadline: - if response_path.exists(): - try: - answer = response_path.read_text(encoding="utf-8").strip() - response_path.unlink(missing_ok=True) - prompt_path.unlink(missing_ok=True) - return answer if answer else default - except (OSError, ValueError): - pass - _time.sleep(0.5) - - # Timeout — clean up and use default - prompt_path.unlink(missing_ok=True) - response_path.unlink(missing_ok=True) - print(f" (no response after {int(timeout)}s, using default: {default!r})") - return default - - def _web_ui_build_needed(web_dir: Path) -> bool: """Return True if the web UI dist is missing or its source content changed. @@ -5348,41 +5269,6 @@ def _run_npm_install_deterministic( ) -def _npm_bin_exists(bin_dir: Path, name: str) -> bool: - """True when an npm bin shim for *name* exists (POSIX or Windows).""" - return any( - (bin_dir / candidate).exists() - for candidate in (name, f"{name}.cmd", f"{name}.ps1", f"{name}.exe") - ) - - -def _web_build_toolchain_ready(*roots: Path) -> bool: - """True when ``tsc`` and ``vite`` shims are reachable from any of *roots*. - - Callers must pass every root the build would search; checking only one - reports a healthy tree as broken. - """ - bin_dirs = [ - bin_dir - for bin_dir in (root / "node_modules" / ".bin" for root in roots) - if bin_dir.is_dir() - ] - return bool(bin_dirs) and all( - any(_npm_bin_exists(bin_dir, tool) for bin_dir in bin_dirs) - for tool in ("tsc", "vite") - ) - - -def _web_toolchain_roots(web_dir: Path) -> tuple[Path, ...]: - """Roots whose ``node_modules/.bin`` can satisfy the web build. - - ``npm run build`` prepends ``node_modules/.bin`` for the package and each - of its ancestors, so shims hoisted to the workspace root and shims nested - under a package that owns its lockfile (#42973) are equally valid. - """ - return (web_dir, web_dir.parent) - - def _missing_web_build_tool(output: str) -> str | None: """Return the build tool a failed ``npm run build`` could not resolve. @@ -7200,247 +7086,6 @@ def _dashboard_probe_host(host: str | None) -> str: return normalized -def _print_curator_first_run_notice() -> None: - """Print a short heads-up about the skill curator after `hermes update`. - - Only fires when the curator is enabled AND has no recorded run yet, which - is exactly the window where the gateway ticker used to fire Curator - against a fresh skill library immediately after an update. We defer the - first real pass by one ``interval_hours``; this notice tells the user how - to preview or disable before then. Silent on steady state. - """ - try: - from agent import curator - except Exception: - return - try: - if not curator.is_enabled(): - return - state = curator.load_state() - except Exception: - return - if state.get("last_run_at"): - # Curator has run before (real or already seeded) — no notice needed. - return - try: - hours = curator.get_interval_hours() - except Exception: - hours = 24 * 7 - days = max(1, hours // 24) - print() - print("ℹ Skill curator") - print( - f" Background skill maintenance is enabled. First pass is deferred " - f"~{days}d after installation; only agent-created skills are in " - f"scope and nothing is ever auto-deleted (archive is recoverable)." - ) - print(" Preview now: hermes curator run --dry-run") - print(" Pause it: hermes curator pause") - print( - " Docs: https://hermes-agent.nousresearch.com/docs/user-guide/features/curator" - ) - - -def _print_fts_optimize_available_notice() -> None: - """Advertise the opt-in v23 search-index optimization after `hermes update`. - - Only fires when the current profile's state.db is still on the legacy - (pre-v23) inline FTS layout. Leads with the reclaimable-space figure and - points at the exact command. Honors ``sessions.fts_optimize_notice``: - ``advise`` (default) prints an advisory notice, ``require`` prints a - firmer required-upgrade notice, ``off`` suppresses it. Silent for - fresh/already-optimized installs. - """ - mode = "advise" - try: - from hermes_cli.config import load_config - - mode = str( - ((load_config() or {}).get("sessions") or {}).get( - "fts_optimize_notice", "advise" - ) - ).strip().lower() - except Exception: - mode = "advise" - if mode == "off": - return - - try: - from hermes_constants import get_hermes_home - from hermes_state import SessionDB - except Exception: - return - db_path = get_hermes_home() / "state.db" - if not db_path.exists(): - return - try: - size_gb = db_path.stat().st_size / (1024 ** 3) - except OSError: - return - # Skip the notice for trivially small DBs — the win isn't worth the nag. - if size_gb < 0.5: - return - db = None - interrupted = False - try: - db = SessionDB(db_path=db_path, read_only=True) - # read_only opens skip schema init, so probe the layout directly. - row = db._conn.execute( - "SELECT sql FROM sqlite_master " - "WHERE type = 'table' AND name = 'messages_fts'" - ).fetchone() - # An interrupted `optimize-storage` run: the table is already the - # v23 shape, but backfill markers / demoted trash tables remain. - # Offer the command again — re-running resumes and finishes it. - interrupted = bool( - db._conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone() - or db._conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' " - "AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1" - ).fetchone() - or db._conn.execute( - "SELECT 1 FROM state_meta WHERE key IN " - "('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1" - ).fetchone() - ) - except Exception: - return - finally: - if db is not None: - try: - db.close() - except Exception: - pass - sql = (row[0] if row else "") or "" - if not sql or ("tool_name" in sql and not interrupted): - # v23 layout already present (fresh/optimized) — nothing to offer. - return - - if interrupted: - print() - print("◆ Session database optimization incomplete") - print( - " A previous `hermes sessions optimize-storage` run was " - "interrupted. Search still works; re-run the command to resume " - "and finish reclaiming disk:" - ) - print(" hermes sessions optimize-storage") - return - - # Concrete size framing — lead with the savings the user cares about. - est_reclaim = size_gb * 0.6 - print() - if mode == "require": - print("◆ Session database upgrade required") - print( - f" Your search index uses the OLD storage layout and should be " - f"upgraded. The new layout typically frees ~60% of state.db " - f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is " - f"required for continued optimal operation." - ) - else: - print("◆ Reclaim ~60% of your session database disk") - print( - f" Your search index uses the old storage layout. Upgrading it " - f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB " - f"of your current {size_gb:.1f} GB." - ) - print(" Run when convenient: hermes sessions optimize-storage") - print( - " It runs in the foreground with a progress bar, is safe to " - "interrupt/re-run, and never changes your conversations." - ) - - -def _print_curator_recent_run_notice() -> None: - """Print the most recent curator run summary, exactly once. - - The curator runs in the background (gateway tick + CLI session start), - so users learn about skill consolidations only by stumbling into a - rename. ``hermes update`` is a high-attention surface — surface the - most recent run's rename map here, once. - - Show-once: state stamps ``last_run_summary_shown_at`` after printing. - Subsequent ``hermes update`` invocations skip the block until a newer - curator run lands. Silent when the curator has never run, when the - most recent summary has already been shown, or when the summary has - no rename information to display (no archives). - """ - try: - from agent import curator - except Exception: - return - try: - state = curator.load_state() - except Exception: - return - - last_run_at = state.get("last_run_at") - if not last_run_at: - return # no curator run yet — first-run notice handles this case - - if state.get("last_run_summary_shown_at") == last_run_at: - return # already shown for this run - - summary = state.get("last_run_summary") or "" - if not summary: - return - - # Only print when there's something interesting to show — i.e. the - # rename map block was appended (multi-line summary). A bare "auto: - # no changes; llm: no change" doesn't warrant interrupting the - # update flow. - if "\n" not in summary: - # Still stamp it shown so we don't reconsider it on every update. - try: - state["last_run_summary_shown_at"] = last_run_at - curator.save_state(state) - except Exception: - pass - return - - # Format the timestamp as "Xh ago" for readability. - when = _format_time_ago(last_run_at) - print() - print(f"ℹ Skill curator — last run {when}") - for line in summary.splitlines(): - print(f" {line}") - print( - " (This message shows once per curator run. " - "View anytime: hermes curator status)" - ) - - # Stamp shown so we don't repeat on the next update. - try: - state["last_run_summary_shown_at"] = last_run_at - curator.save_state(state) - except Exception: - pass - - -def _format_time_ago(iso_ts: str) -> str: - """Render an ISO timestamp as `Xh ago` / `Xd ago` / `Xm ago`. Best effort.""" - try: - from datetime import datetime, timezone - ts = datetime.fromisoformat(iso_ts.replace("Z", "+00:00")) - if ts.tzinfo is None: - ts = ts.replace(tzinfo=timezone.utc) - delta = datetime.now(timezone.utc) - ts - secs = int(delta.total_seconds()) - if secs < 60: - return "just now" - if secs < 3600: - return f"{secs // 60}m ago" - if secs < 86400: - return f"{secs // 3600}h ago" - return f"{secs // 86400}d ago" - except Exception: - return "recently" - - _DASHBOARD_SYSTEMD_UNIT = "hermes-dashboard.service" @@ -7943,947 +7588,15 @@ def _kill_stale_dashboard_processes( } -def _finish_dashboard_update_cleanup(node_failures: list[str]) -> None: - """Refresh managed dashboards or stop stale manual ones after an update.""" - if node_failures: - print() - print(" ℹ Leaving running dashboard process(es) untouched because the") - print(" Node.js dependency refresh did not complete.") - return - - stop_result = _kill_stale_dashboard_processes(restart_managed=True) - if not stop_result.get("unrecovered"): - return - - print() - print( - "⚠ A web dashboard/serve process was stopped during update and could " - "not be auto-restarted." - ) - print(" Re-launch it when you want the web UI back:") - print(" hermes dashboard --port ") - - # Back-compat alias: some tests and any external callers may import the old # warn-only name. The new behaviour (kill stale processes) replaces it. _warn_stale_dashboard_processes = _kill_stale_dashboard_processes -def _atomic_replace_dir(src: str, dst: str) -> None: - """Replace directory *dst* with *src* without leaving *dst* half-deleted. - - The naive ``rmtree(dst); copytree(src, dst)`` has a destructive window: if - the copy fails partway (common on the Windows ZIP-update path, which only - runs because file I/O is already flaky on that machine), the old directory - is already gone and nothing replaced it — the install is left with a - deleted tree (issue #49145, where ``ui-tui/`` vanished and broke the TUI). - - Instead, stage the new copy into a sibling temp dir first; only once that - fully succeeds do we swap it in. A failure during staging raises with the - original *dst* still intact. - """ - staging = f"{dst}.hermes-update-staging" - backup = f"{dst}.hermes-update-old" - # Clear any leftovers from a previously-interrupted update. - for leftover in (staging, backup): - if os.path.exists(leftover): - shutil.rmtree(leftover, ignore_errors=True) - - # 1. Stage the new copy. If this fails, dst is untouched. - shutil.copytree(src, staging) - # 2. Swap: move the live dir aside, move staging into place. Both moves are - # same-filesystem renames; if the second fails we restore the backup. - if os.path.exists(dst): - os.rename(dst, backup) - try: - os.rename(staging, dst) - except OSError: - if os.path.exists(backup) and not os.path.exists(dst): - os.rename(backup, dst) # roll back to the original - raise - # 3. New dir is in place; drop the old one (best-effort — never fatal). - if os.path.exists(backup): - shutil.rmtree(backup, ignore_errors=True) - - -def _update_via_zip(args): - """Update Hermes Agent by downloading a ZIP archive. - - Used on Windows when git file I/O is broken (antivirus, NTFS filter - drivers causing 'Invalid argument' errors on file creation). - """ - import tempfile - import zipfile - from urllib.request import urlretrieve - - # The ZIP fallback exists for Windows git-file-I/O breakage. It pulls a - # static archive from GitHub, which is fine for the default "main" - # channel but would silently ignore --branch and update from main even - # if the user asked for something else — exactly the silent-divergence - # bug --branch was added to prevent. Refuse to proceed in that case - # rather than lie. - branch = _resolve_update_branch(args) - if branch != "main": - print( - f"✗ --branch={branch} is not supported on the Windows ZIP-fallback " - "update path." - ) - print( - " This path runs when git file I/O is broken on the system. " - "Either resolve the git-side breakage (typically an antivirus " - "or NTFS filter holding files open) and rerun `hermes update " - f"--branch {branch}`, or update against main with `hermes update`." - ) - sys.exit(1) - zip_url = ( - f"https://github.com/NousResearch/hermes-agent/archive/refs/heads/{branch}.zip" - ) - - print("→ Downloading latest version...") - tmp_dir = tempfile.mkdtemp(prefix="hermes-update-") - try: - zip_path = os.path.join(tmp_dir, f"hermes-agent-{branch}.zip") - urlretrieve(zip_url, zip_path) - - print("→ Extracting...") - import stat as _stat - with zipfile.ZipFile(zip_path, "r") as zf: - # Validate paths to prevent zip-slip (path traversal) AND reject - # symlink members. A GitHub source ZIP for hermes-agent itself - # should never contain symlinks — they'd point outside the - # extracted tree and let an attacker who can compromise the - # update mirror plant arbitrary files via the update path. - tmp_dir_real = os.path.realpath(tmp_dir) - for member in zf.infolist(): - member_path = os.path.realpath(os.path.join(tmp_dir, member.filename)) - if ( - not member_path.startswith(tmp_dir_real + os.sep) - and member_path != tmp_dir_real - ): - raise ValueError( - f"Zip-slip detected: {member.filename} escapes extraction directory" - ) - # Unix mode lives in the upper 16 bits of external_attr; - # mask to the file-type bits. - mode = (member.external_attr >> 16) & 0o170000 - if _stat.S_ISLNK(mode): - raise ValueError( - f"ZIP contains unsupported symlink member: {member.filename}" - ) - zf.extractall(tmp_dir) - - # GitHub ZIPs extract to hermes-agent-/ - extracted = os.path.join(tmp_dir, f"hermes-agent-{branch}") - if not os.path.isdir(extracted): - # Try to find it - for d in os.listdir(tmp_dir): - candidate = os.path.join(tmp_dir, d) - if os.path.isdir(candidate) and d != "__MACOSX": - extracted = candidate - break - - # Copy updated files over existing installation, preserving venv/node_modules/.git - preserve = {"venv", "node_modules", ".git", ".env"} - update_count = 0 - for item in os.listdir(extracted): - if item in preserve: - continue - src = os.path.join(extracted, item) - dst = os.path.join(str(PROJECT_ROOT), item) - if os.path.isdir(src): - # Atomic-ish replace: never leave dst half-deleted if the copy - # fails partway (the failure mode behind #49145 on Windows). - _atomic_replace_dir(src, dst) - else: - shutil.copy2(src, dst) - update_count += 1 - - print(f"✓ Updated {update_count} items from ZIP") - - except Exception as e: - print(f"✗ ZIP update failed: {e}") - sys.exit(1) - finally: - shutil.rmtree(tmp_dir, ignore_errors=True) - - # Clear stale bytecode after ZIP extraction - removed = _clear_bytecode_cache(PROJECT_ROOT) - if removed: - print( - f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" - ) - _record_bytecode_fingerprint() - - # Reinstall Python dependencies. Prefer .[all], but if one optional extra - # breaks on this machine, keep base deps and reinstall the remaining extras - # individually so update does not silently strip working capabilities. - print("→ Updating Python dependencies...") - - from hermes_cli.managed_uv import ensure_uv, update_managed_uv - - # Keep managed uv current — runs `uv self update` if we already have one. - update_managed_uv() - - uv_bin = ensure_uv() - - pip_cmd = [sys.executable, "-m", "pip"] - if not uv_bin: - uv_bin = _ensure_uv_for_termux(pip_cmd) - if uv_bin: - uv_env = {**os.environ, "VIRTUAL_ENV": str(PROJECT_ROOT / "venv")} - if _is_termux_env(uv_env): - uv_env.pop("PYTHONPATH", None) - uv_env.pop("PYTHONHOME", None) - _install_python_dependencies_with_optional_fallback([uv_bin, "pip"], env=uv_env) - else: - # Use sys.executable to explicitly call the venv's pip module, - # avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu. - # Some environments lose pip inside the venv; bootstrap it back with - # ensurepip before trying the editable install. - try: - subprocess.run( - pip_cmd + ["--version"], - cwd=PROJECT_ROOT, - check=True, - capture_output=True, - ) - except subprocess.CalledProcessError: - subprocess.run( - [sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], - cwd=PROJECT_ROOT, - check=True, - ) - _install_python_dependencies_with_optional_fallback(pip_cmd) - - # ZIP path parity: heal the active memory provider's bridge packages - # after the dependency reinstall, same as the git-pull path (#53272, - # #70636). - _refresh_active_memory_provider_dependencies() - - node_failures = _update_node_dependencies() - _build_web_ui(PROJECT_ROOT / "web") - - # Sync skills - try: - from tools.skills_sync import sync_skills - - print("→ Syncing bundled skills...") - result = sync_skills(quiet=True) - if result["copied"]: - print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}") - if result.get("updated"): - print( - f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}" - ) - if result.get("user_modified"): - print(f" ~ {len(result['user_modified'])} user-modified (kept)") - print( - " → see them: hermes skills list-modified " - "(diff/reset to resume updates)" - ) - if result.get("cleaned"): - print(f" − {len(result['cleaned'])} removed from manifest") - if result.get("relocated"): - print( - f" → {len(result['relocated'])} moved to new upstream paths: " - f"{', '.join(result['relocated'])}" - ) - if not result["copied"] and not result.get("updated"): - print(" ✓ Skills are up to date") - except Exception: - pass - - # Seed the model-catalog disk cache from the freshly-unpacked checkout - # (same rationale as the git-pull path in _cmd_update_impl). Non-fatal. - try: - from hermes_cli.model_catalog import seed_cache_from_checkout - - if seed_cache_from_checkout(PROJECT_ROOT): - print(" ✓ Model catalog cache refreshed from checkout") - except Exception as e: - logger.debug("Model catalog seed during zip update failed: %s", e) - - # ── Post-update state.db integrity guard (#68474) ───────────────── - # Same as the git-pull path: verify state.db survived the ZIP update - # and auto-restore from the most recent pre-update snapshot if needed. - try: - from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity - - _state_path = get_hermes_home() / "state.db" - if _state_path.exists(): - _state_ok = verify_sqlite_integrity( - _state_path, check_header=True, run_pragma=True - ) - if not _state_ok.get("valid"): - print() - print( - "⚠ state.db is corrupted after update: " - + _state_ok.get("message", "unknown error") - ) - _snap_root = _quick_snapshot_root(get_hermes_home()) - if _snap_root.exists(): - _snap_dirs = sorted( - (d for d in _snap_root.iterdir() if d.is_dir()), - reverse=True, - ) - for _snap_dir in _snap_dirs: - _snap_state = _snap_dir / "state.db" - if _snap_state.exists(): - _snap_ok = verify_sqlite_integrity( - _snap_state, check_header=True, run_pragma=True - ) - if _snap_ok.get("valid"): - try: - import shutil as _shutil - - _shutil.copy2(_snap_state, _state_path) - _restored_ok = verify_sqlite_integrity( - _state_path, - check_header=True, - run_pragma=True, - ) - if _restored_ok.get("valid"): - print( - " ✓ Auto-restored from snapshot " - f"{_snap_dir.name}" - ) - else: - print( - " ✗ Auto-restore FAILED — restored " - "copy also failed integrity" - ) - break - except OSError as _exc: - print( - f" ✗ Auto-restore file copy failed: {_exc}" - ) - break - except Exception as exc: - logger.debug( - "Post-update state.db integrity check (zip path) failed: %s", exc - ) - - print() - if node_failures: - print( - "⚠ Update partially complete — Node.js dependencies for " - f"{', '.join(node_failures)} did not refresh." - ) - print(" Code and Python deps are updated, but the dashboard/TUI may") - print(" be in a mixed state until the Node deps are rebuilt.") - else: - print("✓ Update complete!") - try: - _print_curator_first_run_notice() - except Exception as e: - logger.debug("Curator first-run notice failed: %s", e) - try: - _print_curator_recent_run_notice() - except Exception as e: - logger.debug("Curator recent-run notice failed: %s", e) - # Don't stop a working dashboard when the Node refresh failed — see the - # git-update path for rationale (#30271). - _finish_dashboard_update_cleanup(node_failures) - - -def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]: - status = subprocess.run( - git_cmd + ["status", "--porcelain"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=True, - ) - if not status.stdout.strip(): - return None - - # If the index has unmerged entries (e.g. from an interrupted merge/rebase), - # git stash will fail with "needs merge / could not write index". Clear the - # conflict state with `git reset` so the stash can proceed. Working-tree - # changes are preserved; only the index conflict markers are dropped. - unmerged = subprocess.run( - git_cmd + ["ls-files", "--unmerged"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if unmerged.stdout.strip(): - print("→ Clearing unmerged index entries from a previous conflict...") - subprocess.run(git_cmd + ["reset"], cwd=cwd, capture_output=True) - - from datetime import datetime, timezone - - stash_name = datetime.now(timezone.utc).strftime( - "hermes-update-autostash-%Y%m%d-%H%M%S" - ) - print("→ Local changes detected — stashing before update...") - prev_stash = subprocess.run( - git_cmd + ["rev-parse", "--verify", "refs/stash"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ).stdout.strip() - push = subprocess.run( - git_cmd + ["stash", "push", "--include-untracked", "-m", stash_name], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if push.stdout.strip(): - print(push.stdout.strip()) - stash_probe = subprocess.run( - git_cmd + ["rev-parse", "--verify", "refs/stash"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - stash_ref = stash_probe.stdout.strip() - stash_created = ( - stash_probe.returncode == 0 and bool(stash_ref) and stash_ref != prev_stash - ) - - if push.returncode != 0: - if stash_created: - # git stash push exits non-zero when it saved everything but could - # not delete some swept untracked files from the working tree - # (e.g. a root-owned directory: "warning: failed to remove ...: - # Permission denied"). The stash entry is complete — the changes - # are safe — so this is not a failure. Leave the undeletable - # files in place and continue the update. - if push.stderr.strip(): - print(push.stderr.strip()) - print( - " ⚠ Some untracked files could not be removed from the " - "working tree (permission denied)." - ) - print( - " They were still saved to the stash and were left in " - "place — the update will continue." - ) - # A partially-failed stash push also aborts its working-tree - # cleanup for TRACKED modifications — they are saved in the stash - # but still dirty the tree, which would break the checkout/pull - # that follows. Safe to reset: everything is in the stash entry. - subprocess.run( - git_cmd + ["reset", "--hard", "HEAD"], - cwd=cwd, - capture_output=True, - ) - else: - # No stash entry was created: the changes were NOT saved. This - # is a real failure — bail out before the update touches HEAD. - print("✗ Could not stash local changes — update aborted.") - if push.stderr.strip(): - print(f" {push.stderr.strip().splitlines()[0]}") - print( - " Commit, stash, or clean up your local changes manually, " - "then re-run `hermes update`." - ) - raise subprocess.CalledProcessError( - push.returncode, push.args, output=push.stdout, stderr=push.stderr - ) - - return stash_ref - - -def _resolve_stash_selector( - git_cmd: list[str], cwd: Path, stash_ref: str -) -> Optional[str]: - stash_list = subprocess.run( - git_cmd + ["stash", "list", "--format=%gd %H"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=True, - ) - for line in stash_list.stdout.splitlines(): - selector, _, commit = line.partition(" ") - if commit.strip() == stash_ref: - return selector.strip() - return None - - -def _print_stash_cleanup_guidance( - stash_ref: str, stash_selector: Optional[str] = None -) -> None: - print( - " Check `git status` first so you don't accidentally reapply the same change twice." - ) - print(" Find the saved entry with: git stash list --format='%gd %H %s'") - if stash_selector: - print(f" Remove it with: git stash drop {stash_selector}") - else: - print( - f" Look for commit {stash_ref}, then drop its selector with: git stash drop stash@{{N}}" - ) - - -def _stash_apply_failed_only_on_existing_untracked(stderr: str) -> bool: - """True when a ``git stash apply`` failure is ONLY about untracked files - that already exist in the working tree. - - This is the tail end of the permission-denied autostash class: ``git stash - push --include-untracked`` swept undeletable files (e.g. a root-owned - ``packaging/`` directory) into the stash but could not remove them from - disk. On restore, git applies all tracked changes, then refuses to - overwrite those still-present files (``already exists, no checkout`` / - ``could not restore untracked files from stash``) and exits non-zero even - though nothing was lost. Any other error line (e.g. ``would be - overwritten by merge`` / ``Aborting``) means the tracked apply itself - failed and this returns False. - """ - lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()] - if not lines: - return False - saw_untracked_error = False - for ln in lines: - if "already exists, no checkout" in ln: - saw_untracked_error = True - elif "could not restore untracked files from stash" in ln: - saw_untracked_error = True - elif ln.startswith(("warning:", "hint:")): - continue - else: - return False - return saw_untracked_error - - -def _restore_stashed_changes( - git_cmd: list[str], - cwd: Path, - stash_ref: str, - prompt_user: bool = False, - input_fn=None, -) -> bool: - if prompt_user: - print() - print("⚠ Local changes were stashed before updating.") - print( - " Restoring them may reapply local customizations onto the updated codebase." - ) - print(" Review the result afterward if Hermes behaves unexpectedly.") - print("Restore local changes now? [Y/n]") - if input_fn is not None: - response = input_fn("Restore local changes now? [Y/n]", "y") - else: - response = input().strip().lower() - if response not in {"", "y", "yes"}: - print("Skipped restoring local changes.") - print("Your changes are still preserved in git stash.") - print(f"Restore manually with: git stash apply {stash_ref}") - return False - - print("→ Restoring local changes...") - restore = subprocess.run( - git_cmd + ["stash", "apply", stash_ref], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - - # Check for unmerged (conflicted) files — can happen even when returncode is 0 - unmerged = subprocess.run( - git_cmd + ["diff", "--name-only", "--diff-filter=U"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - has_conflicts = bool(unmerged.stdout.strip()) - - if restore.returncode != 0 and not has_conflicts and ( - _stash_apply_failed_only_on_existing_untracked(restore.stderr) - ): - # Permission-denied autostash tail end: the tracked changes applied - # cleanly; the only "failure" is untracked files that never left the - # working tree (git could not delete them at stash time, so it now - # refuses to overwrite them). Their content was never touched — - # nothing is lost. Treat as restored. - print( - " ⚠ Some stashed untracked files already exist in the working " - "tree and were kept as-is." - ) - elif restore.returncode != 0 or has_conflicts: - print("✗ Update pulled new code, but restoring local changes hit conflicts.") - if restore.stdout.strip(): - print(restore.stdout.strip()) - if restore.stderr.strip(): - print(restore.stderr.strip()) - - # Show which files conflicted - conflicted_files = unmerged.stdout.strip() - if conflicted_files: - print("\nConflicted files:") - for f in conflicted_files.splitlines(): - print(f" • {f}") - - print("\nYour stashed changes are preserved — nothing is lost.") - print(f" Stash ref: {stash_ref}") - - # Always reset to clean state — leaving conflict markers in source - # files makes hermes completely unrunnable (SyntaxError on import). - # The user's changes are safe in the stash for manual recovery. - subprocess.run( - git_cmd + ["reset", "--hard", "HEAD"], - cwd=cwd, - capture_output=True, - ) - print("Working tree reset to clean state.") - print(f"Restore your changes later with: git stash apply {stash_ref}") - # Don't sys.exit — the code update itself succeeded, only the stash - # restore had conflicts. Let cmd_update continue with pip install, - # skill sync, and gateway restart. - return False - - stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) - if stash_selector is None: - print( - "⚠ Local changes were restored, but Hermes couldn't find the stash entry to drop." - ) - print( - " The stash was left in place. You can remove it manually after checking the result." - ) - _print_stash_cleanup_guidance(stash_ref) - else: - drop = subprocess.run( - git_cmd + ["stash", "drop", stash_selector], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if drop.returncode != 0: - print( - "⚠ Local changes were restored, but Hermes couldn't drop the saved stash entry." - ) - if drop.stdout.strip(): - print(drop.stdout.strip()) - if drop.stderr.strip(): - print(drop.stderr.strip()) - print( - " The stash was left in place. You can remove it manually after checking the result." - ) - _print_stash_cleanup_guidance(stash_ref, stash_selector) - - print("⚠ Local changes were restored on top of the updated codebase.") - print(" Review `git diff` / `git status` if Hermes behaves unexpectedly.") - return True - - -def _discard_stashed_changes( - git_cmd: list[str], - cwd: Path, - stash_ref: str, -) -> bool: - """Throw away a stash created before an update, without applying it. - - Used only on a NON-interactive update when the user has set - ``updates.non_interactive_local_changes: discard`` — i.e. they've opted out - of keeping local source edits on this machine. Drops the stash entry - instead of re-applying it, so the working tree stays clean at the freshly - pulled HEAD. Unlike ``git reset --hard`` + ``git clean -fd``, this only - affects what was stashed (tracked changes + the untracked files we - explicitly captured) — ignored paths like node_modules/venv/build outputs - are never touched, since they were never stashed. - - Returns True if the stash was dropped, False on a git failure (in which - case the stash is left in place for safety). - """ - stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) - if stash_selector is None: - print( - "⚠ Configured to discard local changes on non-interactive update, " - "but Hermes couldn't find the stash entry to drop." - ) - _print_stash_cleanup_guidance(stash_ref) - return False - - drop = subprocess.run( - git_cmd + ["stash", "drop", stash_selector], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if drop.returncode != 0: - print( - "⚠ Configured to discard local changes, but Hermes couldn't drop " - "the saved stash entry." - ) - if drop.stderr.strip(): - print(f" {drop.stderr.strip().splitlines()[0]}") - _print_stash_cleanup_guidance(stash_ref, stash_selector) - return False - - print("→ Discarded local source changes (updates.non_interactive_local_changes=discard).") - return True - - # ========================================================================= # Fork detection and upstream management for `hermes update` # ========================================================================= -OFFICIAL_REPO_URLS = { - "https://github.com/NousResearch/hermes-agent.git", - "git@github.com:NousResearch/hermes-agent.git", - "https://github.com/NousResearch/hermes-agent", - "git@github.com:NousResearch/hermes-agent", -} -OFFICIAL_REPO_URL = "https://github.com/NousResearch/hermes-agent.git" -SKIP_UPSTREAM_PROMPT_FILE = ".skip_upstream_prompt" - - -def _get_origin_url(git_cmd: list[str], cwd: Path) -> Optional[str]: - """Get the URL of the origin remote, or None if not set.""" - try: - result = subprocess.run( - git_cmd + ["remote", "get-url", "origin"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if result.returncode == 0: - return result.stdout.strip() - except Exception: - pass - return None - - -def _is_fork(origin_url: Optional[str]) -> bool: - """Check if the origin remote points to a fork (not the official repo).""" - if not origin_url: - return False - # Normalize URL for comparison (strip trailing .git if present) - normalized = origin_url.rstrip("/") - if normalized.endswith(".git"): - normalized = normalized[:-4] - for official in OFFICIAL_REPO_URLS: - official_normalized = official.rstrip("/") - if official_normalized.endswith(".git"): - official_normalized = official_normalized[:-4] - if normalized == official_normalized: - return False - return True - - -def _has_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: - """Check if an 'upstream' remote already exists.""" - try: - result = subprocess.run( - git_cmd + ["remote", "get-url", "upstream"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - return result.returncode == 0 - except Exception: - return False - - -def _add_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: - """Add the official repo as the 'upstream' remote. Returns True on success.""" - try: - result = subprocess.run( - git_cmd + ["remote", "add", "upstream", OFFICIAL_REPO_URL], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - return result.returncode == 0 - except Exception: - return False - - -def _count_commits_between(git_cmd: list[str], cwd: Path, base: str, head: str) -> int: - """Count commits on `head` that are not on `base`. Returns -1 on error.""" - try: - result = subprocess.run( - git_cmd + ["rev-list", "--count", f"{base}..{head}"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if result.returncode == 0: - return int(result.stdout.strip()) - except Exception: - pass - return -1 - - -def _should_skip_upstream_prompt() -> bool: - """Check if user previously declined to add upstream.""" - from hermes_constants import get_hermes_home - - return (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).exists() - - -def _mark_skip_upstream_prompt(): - """Create marker file to skip future upstream prompts.""" - try: - from hermes_constants import get_hermes_home - - (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).touch() - except Exception: - pass - - -def _sync_fork_with_upstream(git_cmd: list[str], cwd: Path) -> bool: - """Attempt to push updated main to origin (sync fork). - - Returns True if push succeeded, False otherwise. - """ - try: - result = subprocess.run( - git_cmd + ["push", "origin", "main", "--force-with-lease"], - cwd=cwd, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - return result.returncode == 0 - except Exception: - return False - - -def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path) -> None: - """Check if fork is behind upstream and sync if safe. - - This implements the fork upstream sync logic: - - If upstream remote doesn't exist, ask user if they want to add it - - Compare origin/main with upstream/main - - If origin/main is strictly behind upstream/main, pull from upstream - - Try to sync fork back to origin if possible - """ - has_upstream = _has_upstream_remote(git_cmd, cwd) - - if not has_upstream: - # Check if user previously declined - if _should_skip_upstream_prompt(): - return - - # Ask user if they want to add upstream - print() - print("ℹ Your fork is not tracking the official Hermes repository.") - print(" This means you may miss updates from NousResearch/hermes-agent.") - print() - try: - response = ( - input("Add official repo as 'upstream' remote? [Y/n]: ").strip().lower() - ) - except (EOFError, KeyboardInterrupt): - print() - response = "n" - - if response in {"", "y", "yes"}: - print("→ Adding upstream remote...") - if _add_upstream_remote(git_cmd, cwd): - print( - " ✓ Added upstream: https://github.com/NousResearch/hermes-agent.git" - ) - has_upstream = True - else: - print(" ✗ Failed to add upstream remote. Skipping upstream sync.") - return - else: - print( - " Skipped. Run 'git remote add upstream https://github.com/NousResearch/hermes-agent.git' to add later." - ) - _mark_skip_upstream_prompt() - return - - # Fetch upstream main only. This sync compares upstream/main with - # origin/main, so there's no reason to pull every upstream ref — and a bare - # fetch drags in thousands of auto-generated branches. - print() - print("→ Fetching upstream...") - try: - subprocess.run( - git_cmd + ["fetch", "upstream", "main", "--quiet"], - cwd=cwd, - capture_output=True, - check=True, - ) - except subprocess.CalledProcessError: - print(" ✗ Failed to fetch upstream. Skipping upstream sync.") - return - - # Compare origin/main with upstream/main - origin_ahead = _count_commits_between(git_cmd, cwd, "upstream/main", "origin/main") - upstream_ahead = _count_commits_between( - git_cmd, cwd, "origin/main", "upstream/main" - ) - - if origin_ahead < 0 or upstream_ahead < 0: - print(" ✗ Could not compare branches. Skipping upstream sync.") - return - - # If origin/main has commits not on upstream, don't trample - if origin_ahead > 0: - print() - print(f"ℹ Your fork has {origin_ahead} commit(s) not on upstream.") - print(" Skipping upstream sync to preserve your changes.") - print(" If you want to merge upstream changes, run:") - print(" git pull upstream main") - return - - # If upstream is not ahead, fork is up to date - if upstream_ahead == 0: - print(" ✓ Fork is up to date with upstream") - return - - # origin/main is strictly behind upstream/main (can fast-forward) - print() - print(f"→ Fork is {upstream_ahead} commit(s) behind upstream") - print("→ Pulling from upstream...") - - try: - subprocess.run( - git_cmd + ["pull", "--ff-only", "upstream", "main"], - cwd=cwd, - check=True, - ) - except subprocess.CalledProcessError: - print( - " ✗ Failed to pull from upstream. You may need to resolve conflicts manually." - ) - return - - print(" ✓ Updated from upstream") - - # Try to sync fork back to origin - print("→ Syncing fork...") - if _sync_fork_with_upstream(git_cmd, cwd): - print(" ✓ Fork synced with upstream") - else: - print( - " ℹ Got updates from upstream but couldn't push to fork (no write access?)" - ) - print(" Your local repo is updated, but your fork on GitHub may be behind.") - - -def _invalidate_update_cache(): - """Delete the update-check cache for ALL profiles so no banner - reports a stale "commits behind" count after a successful update. - - The git repo is shared across profiles — when one profile runs - ``hermes update``, every profile is now current. - """ - homes = [] - # Default profile home (Docker-aware — uses /opt/data in Docker) - from hermes_constants import get_default_hermes_root - - default_home = get_default_hermes_root() - homes.append(default_home) - # Named profiles under /profiles/ - profiles_root = default_home / "profiles" - if profiles_root.is_dir(): - for entry in profiles_root.iterdir(): - if entry.is_dir(): - homes.append(entry) - for home in homes: - try: - cache_file = home / ".update_check" - if cache_file.exists(): - cache_file.unlink() - except Exception: - pass - def _load_installable_optional_extras(group: str = "all") -> list[str]: """Return optional extras referenced by a dependency group. @@ -8932,16 +7645,6 @@ def _lazy_refresh_marker_path() -> Path: return PROJECT_ROOT / ".lazy-refresh-incomplete" -def _write_marker_file(path: Path, *, label: str) -> None: - """Drop an update-recovery breadcrumb. Never raises.""" - try: - path.write_text( - f"started={_time.time()}\npid={os.getpid()}\n", encoding="utf-8" - ) - except OSError as exc: - logger.debug("Could not write %s marker: %s", label, exc) - - def _clear_marker_file(path: Path, *, label: str) -> None: """Remove an update-recovery breadcrumb. Never raises.""" try: @@ -8952,21 +7655,11 @@ def _clear_marker_file(path: Path, *, label: str) -> None: logger.debug("Could not clear %s marker: %s", label, exc) -def _write_update_incomplete_marker() -> None: - """Drop the interrupted core-install breadcrumb. Never raises.""" - _write_marker_file(_update_marker_path(), label="update-incomplete") - - def _clear_update_incomplete_marker() -> None: """Remove the interrupted core-install breadcrumb. Never raises.""" _clear_marker_file(_update_marker_path(), label="update-incomplete") -def _write_lazy_refresh_incomplete_marker() -> None: - """Drop the interrupted lazy-refresh breadcrumb. Never raises.""" - _write_marker_file(_lazy_refresh_marker_path(), label="lazy-refresh-incomplete") - - def _clear_lazy_refresh_incomplete_marker() -> None: """Remove the interrupted lazy-refresh breadcrumb. Never raises.""" _clear_marker_file(_lazy_refresh_marker_path(), label="lazy-refresh-incomplete") @@ -9416,32 +8109,6 @@ def _detect_concurrent_hermes_instances( return matches -def _format_concurrent_instances_message( - matches: list[tuple[int, str]], scripts_dir: Path -) -> str: - """Build a human-readable explanation + remediation hint for the user.""" - shim = scripts_dir / "hermes.exe" - lines = ["✗ Another hermes.exe is running:"] - for pid, name in matches: - lines.append(f" PID {pid} {name}") - lines.append("") - lines.append(f" Updating now would fail to overwrite {shim} because") - lines.append(" Windows blocks REPLACE on a running executable.") - lines.append("") - lines.append(" Close Hermes Desktop, exit any open `hermes` REPLs, and") - lines.append(" stop the gateway (`hermes gateway stop`) before retrying.") - lines.append("") - if matches: - pid_args = " ".join(f"/PID {pid}" for pid, _ in matches) - lines.append(" If you've already closed everything and these PIDs are") - lines.append(" stale, terminate them directly, then retry the update:") - lines.append(f" taskkill {pid_args} /F") - lines.append("") - lines.append(" Override with `hermes update --force` if you've already") - lines.append(" confirmed those processes will not write to the venv.") - return "\n".join(lines) - - def _quarantine_running_hermes_exe( scripts_dir: Path, *, max_attempts: int = 4 ) -> list[tuple[Path, Path]]: @@ -9718,26 +8385,6 @@ def _lazy_refresh_repair_specs(packages: list[str]) -> list[str]: return [name_to_spec.get(pkg.lower(), pkg) for pkg in packages] -def _upgrade_pip_before_lazy_refresh( - install_cmd_prefix: list[str], - *, - env: dict[str, str] | None = None, -) -> None: - """Upgrade pip before lazy-backend refreshes. - - Older pip (e.g. 24.0 on Python 3.11) can fail setuptools-backed source - builds during lazy installs and leave a partially-written venv (#57828). - Never raises. - """ - try: - _run_package_only_install( - install_cmd_prefix + ["install", "--upgrade", "pip"], - env=env, - ) - except subprocess.CalledProcessError as exc: - logger.debug("pip upgrade before lazy refresh failed: %s", exc) - - def _detect_broken_lazy_refresh_imports( install_cmd_prefix: list[str], *, @@ -9883,160 +8530,6 @@ def _repair_venv_via_import_probes( return "failed" -def _refresh_active_lazy_features( - install_cmd_prefix: list[str] | None = None, - *, - env: dict[str, str] | None = None, -) -> bool: - """Refresh lazy-installed backends after a code update. - - When pyproject.toml's ``[all]`` extra was slimmed down (May 2026), most - optional backends moved to ``tools/lazy_deps.py`` and only install on - first use. ``hermes update`` runs ``uv pip install -e .[all]`` which - leaves those packages untouched — so if we bump a pin in - :data:`LAZY_DEPS` (CVE response, transitive bug fix), users who already - activated the backend keep the stale version forever. - - This function asks lazy_deps which features the user has previously - activated and reinstalls them under the current pins. Features the - user never enabled stay quiet — no churn for cold backends. - - Returns True when the venv is safe to use (refresh succeeded, or no - active lazy backends, or post-failure import repair succeeded). Returns - False when a failed lazy install left broken core imports that automatic - repair could not fix (#57828). - - Never raises. A failure here must not block the rest of the update. - """ - try: - from tools import lazy_deps - except Exception as exc: - logger.debug("Lazy refresh skipped (import failed): %s", exc) - return True - - try: - active = lazy_deps.active_features() - except Exception as exc: - logger.debug("Lazy refresh skipped (active_features failed): %s", exc) - return True - - if not active: - return True - - print() - print(f"→ Refreshing {len(active)} active lazy backend(s)...") - - unexpected_failure = False - try: - results = lazy_deps.refresh_active_features(prompt=False) - except Exception as exc: - # refresh_active_features is documented as never-raise, but defend - # the update flow against future regressions. - print(f" ⚠ Lazy refresh failed unexpectedly: {exc}") - results = {} - unexpected_failure = True - - refreshed = [f for f, s in results.items() if s == "refreshed"] - current = [f for f, s in results.items() if s == "current"] - failed = [(f, s) for f, s in results.items() if s.startswith("failed:")] - skipped = [(f, s) for f, s in results.items() if s.startswith("skipped:")] - - if refreshed: - print(f" ↑ {len(refreshed)} refreshed: {', '.join(refreshed)}") - if current: - print(f" ✓ {len(current)} already current") - if skipped: - # Most common reason: security.allow_lazy_installs=false. Show one - # line so the user knows why; not an error. - names = ", ".join(f for f, _ in skipped) - reason = skipped[0][1].split(": ", 1)[-1] - print(f" · {len(skipped)} skipped ({reason}): {names}") - - if not failed and not unexpected_failure: - return True - - for feature, status in failed: - reason = status.split(": ", 1)[-1] - # Clip noisy pip stderr to keep update output legible. - if len(reason) > 200: - reason = reason[:200] + "..." - print(f" ⚠ {feature} failed to refresh: {reason}") - - if install_cmd_prefix is None: - print(" ⚠ Lazy refresh failed; rerun `hermes update` once resolved.") - return False - - # Immediate import-based recovery — metadata-only verifiers miss the case - # where DISTRIBUTION-INFO remains but import files were wiped (#57828). - # Unavailable probes are indeterminate, not healthy — keep the lazy marker. - status = _repair_venv_via_import_probes(install_cmd_prefix, env=env) - if status == "repaired": - print( - " Lazy backend(s) keep their previous version until refresh succeeds." - ) - return True - if status == "healthy": - print( - " Lazy backend(s) keep their previous version; probed packages look intact." - ) - print(" Rerun `hermes update` once the upstream issue is resolved.") - return True - if status == "indeterminate": - print( - " ⚠ Leaving `.lazy-refresh-incomplete` until import probes can confirm health." - ) - return False - - -def _refresh_active_memory_provider_dependencies() -> None: - """Refresh pip dependencies for the configured external memory provider. - - Memory-provider bridge packages are declared in each provider's - ``plugin.yaml`` (plus mode-dependent extras like Hindsight's - ``hindsight-all``), NOT in Hermes' editable-install extras or - ``LAZY_DEPS`` alone — so the core dependency reinstall above can strip - or downgrade them (#53272 mem0ai, #70636 hindsight-embed). Re-run the - provider's declared install for the ACTIVE provider only, after the - core install and lazy refresh, so the last write to any shared package - is the one the active provider needs. - - Never raises. A failure here must not block the rest of the update. - """ - try: - from hermes_cli.config import load_config - - cfg = load_config() - except Exception as exc: - logger.debug("Memory provider refresh skipped (config load failed): %s", exc) - return - - provider = "" - if isinstance(cfg, dict): - memory_cfg = cfg.get("memory") - if isinstance(memory_cfg, dict): - if memory_cfg.get("enabled") is False: - return - provider = str(memory_cfg.get("provider") or "").strip() - - # "default" / empty is the built-in file-backed store — no pip deps. - if not provider or provider in {"default", "builtin", "none"}: - return - - try: - from hermes_cli.memory_setup import _install_dependencies - except Exception as exc: - logger.debug("Memory provider refresh skipped (import failed): %s", exc) - return - - print() - print(f"→ Refreshing active memory provider dependencies ({provider})...") - - try: - _install_dependencies(provider, force=True) - except Exception as exc: - print(f" ⚠ {provider} dependencies failed to refresh: {exc}") - - def _install_python_dependencies_with_optional_fallback( install_cmd_prefix: list[str], *, @@ -10408,176 +8901,6 @@ def _is_termux_env(env: dict[str, str] | None = None) -> bool: return _is_termux_startup_environment(env) -def _is_android_python() -> bool: - return sys.platform == "android" - - -def _install_psutil_android_compat( - install_cmd_prefix: list[str], - *, - env: dict[str, str] | None = None, -) -> None: - """Install psutil on Android by patching upstream platform detection. - - psutil's setup currently gates Linux sources behind - ``sys.platform.startswith('linux')``. On Termux Python reports - ``sys.platform == 'android'``, so setup aborts with - "platform android is not supported" despite compiling fine when using the - Linux source path. - - We patch only the extracted build tree used for this install attempt; - nothing is persisted in the repository. - - Stopgap: remove this once https://github.com/giampaolo/psutil/pull/2762 - merges and ships in a release. The standalone installer script uses the - same shared helper and should be removed together. - """ - import tempfile - import urllib.request - from hermes_cli.psutil_android import PSUTIL_URL, prepare_patched_psutil_sdist - - with tempfile.TemporaryDirectory() as tmp: - tmp_path = Path(tmp) - archive = tmp_path / "psutil.tar.gz" - urllib.request.urlretrieve(PSUTIL_URL, archive) - src_root = prepare_patched_psutil_sdist(archive, tmp_path) - - _run_install_with_heartbeat( - install_cmd_prefix + ["install", "--no-build-isolation", str(src_root)], - env=env, - ) - - -def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None: - """Best-effort uv bootstrap on Termux for faster update installs. - - The normal path (``ensure_uv()`` in managed_uv) installs the managed - standalone uv into ``$HERMES_HOME/bin/uv``, but on Termux the official - installer may not work (glibc vs bionic). Prefer a uv already on PATH - (e.g. ``pkg install uv``); only if there is none do we fall back to a - wheel-only ``pip install uv`` so we never source-build the Rust crate. - """ - from hermes_cli.managed_uv import resolve_uv - - existing = resolve_uv() - if existing: - return existing - if not _is_termux_env(): - return None - # A Termux-packaged uv lands on PATH but not in the managed bin dir, so - # resolve_uv() misses it. Use it before pip, which has no Android wheel and - # would otherwise build uv from source on a low-memory device. - system_uv = shutil.which("uv") - if system_uv: - return system_uv - try: - print(" → Termux detected: trying to install uv for faster dependency updates...") - result = subprocess.run( - pip_cmd + ["install", "uv", "--only-binary", ":all:"], - cwd=PROJECT_ROOT, - check=False, - ) - if result.returncode != 0: - return None - except Exception: - pass - # After pip install, check managed path first, then PATH - return resolve_uv() or shutil.which("uv") - - -def _npm_manifest_paths() -> tuple[Path, ...]: - """Manifests whose changes must defeat the update-skip. - - The lockfile alone is NOT a sufficient key: on a local checkout a dev - can edit package.json (root or a workspace) without running npm — the - lockfile is then unchanged but `hermes update` is exactly the step - expected to sync node_modules (via the `npm install` fallback in - _run_npm_install_deterministic). - - The workspace list is pulled from the root package.json's `workspaces` - globs (npm's own source of truth) rather than hardcoded, so adding a - workspace can never silently escape the skip key. The root install - (step 1, --workspaces=false) still hoists shared deps for EVERY - workspace — desktop included — so all of them belong in the key, not - just the ones step 2 installs. Falls back to hashing just root - manifests if package.json is unreadable (never skips more than main - would have installed). - """ - root_pkg = PROJECT_ROOT / "package.json" - paths = [PROJECT_ROOT / "package-lock.json", root_pkg] - try: - workspaces = json.loads(root_pkg.read_text(encoding="utf-8")).get( - "workspaces", [] - ) - if isinstance(workspaces, dict): # legacy {"packages": [...]} form - workspaces = workspaces.get("packages", []) - for pattern in workspaces: - for match in sorted(PROJECT_ROOT.glob(str(pattern))): - manifest = match / "package.json" - if manifest.is_file(): - paths.append(manifest) - except (OSError, json.JSONDecodeError, TypeError): - pass - return tuple(paths) - - -def _npm_manifests_digest() -> str | None: - """Combined sha256 over the lockfile + all workspace package.json files. - - Returns None when the lockfile is missing (never skip then). - """ - if not (PROJECT_ROOT / "package-lock.json").exists(): - return None - h = hashlib.sha256() - for p in _npm_manifest_paths(): - h.update(str(p.relative_to(PROJECT_ROOT)).encode()) - try: - h.update(p.read_bytes()) - except OSError: - h.update(b"") - return h.hexdigest() - - -def _npm_lockfile_changed(hermes_root: Path) -> bool: - current = _npm_manifests_digest() - if current is None: - return True - # Also check that node_modules exists; a matching hash with missing - # node_modules means the cache was recorded by another checkout. - if not (PROJECT_ROOT / "node_modules").is_dir(): - return True - # A matching lockfile hash over a tree whose web build toolchain never - # landed must NOT skip the reinstall — otherwise every later `hermes - # update` keeps rebuilding against a half-installed tree and serving a - # stale dist. - web_dir = PROJECT_ROOT / "web" - if (web_dir / "package.json").is_file() and not _web_build_toolchain_ready( - *_web_toolchain_roots(web_dir) - ): - return True - try: - # Key the cache by PROJECT_ROOT so parallel worktrees don't collide. - cache_key = hashlib.sha256(str(PROJECT_ROOT).encode()).hexdigest()[:12] - cache_file = hermes_root / f".npm_lock_hash_{cache_key}" - if not cache_file.exists(): - return True - return cache_file.read_text(encoding="utf-8").strip() != current - except OSError: - return True - - -def _record_npm_lockfile_hash(hermes_root: Path) -> None: - digest = _npm_manifests_digest() - if digest is None: - return - try: - cache_key = hashlib.sha256(str(PROJECT_ROOT).encode()).hexdigest()[:12] - cache_file = hermes_root / f".npm_lock_hash_{cache_key}" - cache_file.write_text(digest, encoding="utf-8") - except OSError: - logger.debug("Could not write npm lockfile hash cache") - - def _is_windows_npm_path(npm_path: str) -> bool: """Return True if ``npm_path`` points at a Windows npm shim. @@ -10635,113 +8958,6 @@ def _resolve_node_runtime_npm() -> str | None: return None -def _update_node_dependencies() -> list[str]: - """Refresh Node deps in the repo root and update workspaces. - - Returns the list of labels whose npm install failed (empty on success), - so the caller can treat a Node refresh failure as a partial update rather - than silently reporting ``Update complete!`` (#30271). - """ - if not (PROJECT_ROOT / "package.json").exists(): - return [] - - npm = _resolve_node_runtime_npm() - if not npm: - # If the only npm reachable inside this WSL shell is the Windows one, - # flag it loudly: silently skipping leaves ui-tui deps stale while the - # rest of the update proceeds, and running it would corrupt the tree. - from hermes_constants import is_wsl - - path_npm = shutil.which("npm") - if is_wsl() and path_npm and _is_windows_npm_path(path_npm): - print("→ Updating Node.js dependencies...") - print(" ⚠ Skipped: only a Windows npm is reachable from this WSL shell.") - print(" Install Node.js inside the WSL distro (nvm, or your distro's") - print(" package manager), then re-run `hermes update`.") - failed = ["repo root"] - if any( - (PROJECT_ROOT / workspace / "package.json").exists() - for workspace in ("ui-tui", "web") - ): - failed.append("ui-tui, web workspaces") - return failed - return [] - - from hermes_constants import get_default_hermes_root - - # This cache describes PROJECT_ROOT/node_modules, which is shared by every - # Hermes profile using this checkout. Keep one per-checkout cache under the - # shared Hermes root rather than rerunning npm once per named profile. - shared_hermes_root = get_default_hermes_root() - if not _npm_lockfile_changed(shared_hermes_root): - logger.info("npm lockfile unchanged, skipping npm install") - return [] - - # With a single workspace lockfile the root install would cover ALL - # workspaces — but apps/desktop pulls in Electron as a devDependency, - # and its postinstall downloads a ~200MB binary. Most users don't - # need desktop during `hermes update`, so we install root-only first - # then add just the workspaces the CLI/TUI/web build actually requires. - # Desktop deps are installed on demand by the desktop launcher - # (see _desktop_build_needed). - print("→ Updating Node.js dependencies...") - - def _partial_update_failure(*labels: str) -> list[str]: - print() - print(" ⚠ Node.js dependency refresh did not complete cleanly; the") - print(" installation may be in a mixed state (updated code, stale Node") - print(" deps). Fix npm and re-run `hermes update`.") - return list(labels) - - extra_args = ["--no-fund", "--no-audit", "--progress=false"] - - from hermes_constants import with_hermes_node_path - - nixos_env = with_hermes_node_path(_nixos_build_env()) - - # Step 1: root install (no workspace recursion). - # NOTE: capture_output=False here is deliberate (#18840) — optional - # postinstall scripts (e.g. @askjo/camofox-browser's browser-binary fetch) - # print download progress, and capturing it makes a long download look - # hung. The chatty npm-deprecation noise during `hermes update` comes from - # the *desktop* build, not this step; that one is captured to update.log. - root_args = [*extra_args, "--workspaces=false"] - root_result = _run_npm_install_deterministic( - npm, - PROJECT_ROOT, - extra_args=tuple(root_args), - capture_output=False, - env=nixos_env, - ) - if root_result.returncode != 0: - print(" ⚠ npm install failed in repo root") - stderr = (root_result.stderr or "").strip() if root_result.stderr else "" - if stderr: - print(f" {stderr.splitlines()[-1]}") - return _partial_update_failure("repo root") - - # Step 2: install only the workspaces update needs (ui-tui, web). - # --workspace selects specific workspaces; the rest (desktop) are skipped. - ws_args = [*extra_args, "--workspace", "ui-tui", "--workspace", "web"] - ws_result = _run_npm_install_deterministic( - npm, - PROJECT_ROOT, - extra_args=tuple(ws_args), - capture_output=False, - env=nixos_env, - ) - if ws_result.returncode == 0: - _record_npm_lockfile_hash(shared_hermes_root) - print(" ✓ repo root + ui-tui, web workspaces (desktop skipped)") - return [] - - print(" ⚠ npm workspace install failed") - stderr = (ws_result.stderr or "").strip() if ws_result.stderr else "" - if stderr: - print(f" {stderr.splitlines()[-1]}") - return _partial_update_failure("ui-tui, web workspaces") - - class _UpdateOutputStream: """Stream wrapper used during ``hermes update`` to survive terminal loss. @@ -10896,50 +9112,6 @@ def _install_hangup_protection(gateway_mode: bool = False): return state -def _log_only_write(text: str) -> None: - """Write ``text`` to ``~/.hermes/logs/update.log`` only, never the terminal. - - During ``hermes update`` ``sys.stdout`` is an ``_UpdateOutputStream`` that - mirrors to both the terminal and ``update.log``. Loud, low-signal - subprocess output (npm installs, the Electron/vite build, the cua-driver - installer's "Next steps" wall) should be captured and tucked into the log - so failures stay debuggable, without flooding the user's terminal. This - reaches past the mirroring stream straight to the underlying log handle. - """ - if not text: - return - stream = sys.stdout - log_file = getattr(stream, "_log", None) - if log_file is None: - return - try: - log_file.write(text if text.endswith("\n") else text + "\n") - log_file.flush() - except Exception: - pass - - -def _run_logged_subprocess(cmd, *, cwd=None, env=None): - """Run ``cmd`` capturing combined output into update.log (not the terminal). - - Returns the ``CompletedProcess`` (with ``stdout`` populated) so the caller - can decide whether to surface the captured output on failure. - """ - result = subprocess.run( - cmd, - cwd=cwd, - env=env, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - encoding="utf-8", - errors="replace", - ) - _log_only_write(result.stdout or "") - return result - - def _finalize_update_output(state): """Restore stdio and close the update.log handle opened by ``_install_hangup_protection``.""" if not state: @@ -10973,314 +9145,6 @@ def _resolve_update_branch(args) -> str: return (getattr(args, "branch", None) or "main").strip() or "main" -def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): - """Implement ``hermes update --check``: fetch and report without installing. - - ``branch`` selects which branch the check compares against. Default is - "main"; callers can pass another branch to ask "are there new commits - on origin/?" without performing the update. - - ``branch_explicit`` is True iff the caller passed --branch on the CLI. - Installs that can't honor non-default branches (e.g. Docker) surface a - one-line notice instead of silently dropping the flag. - """ - from hermes_cli.config import detect_install_method, recommended_update_command_for_method - method = detect_install_method(PROJECT_ROOT) - if method == "docker": - # Docker can't ``git fetch`` from within the container. Surface the - # same long-form ``docker pull`` guidance ``hermes update`` (apply - # path) uses — telling the user to "reinstall via curl" or that - # ".git is missing" would point them at the wrong remediation. - from hermes_cli.config import format_docker_update_message - print(format_docker_update_message()) - sys.exit(1) - - if method in {"nix", "nixos"}: - print(recommended_update_command_for_method(method)) - sys.exit(1) - - git_dir = PROJECT_ROOT / ".git" - if not git_dir.exists(): - print("✗ Not a git repository — cannot check for updates.") - sys.exit(1) - - git_cmd = ["git"] - if sys.platform == "win32": - git_cmd = ["git", "-c", "windows.appendAtomically=false"] - - # Fetch only the branch we compare against; prefer upstream as the canonical - # reference. A bare `git fetch ` pulls every ref, and this repo has - # thousands of auto-generated branches, so scope the fetch to . - # Note: upstream/ may not exist for non-main branches (a fork's - # bb/gui has no upstream counterpart), so when the caller picks a - # non-default branch we skip the upstream probe and use origin directly. - # Installer checkouts are shallow (`git clone --depth 1`). A plain - # `git fetch` would unshallow the repo (dragging in the whole history — - # the exact cost the shallow clone avoided) and the rev-list count below - # would then report a huge bogus "behind" number. Detect shallow up front: - # fetch with --depth 1 to preserve the boundary and report presence-only. - is_shallow = ( - subprocess.run( - git_cmd + ["rev-parse", "--is-shallow-repository"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ).stdout.strip() - == "true" - ) - depth_args = ["--depth", "1"] if is_shallow else [] - - if branch == "main": - # Probe locally (~6 ms) whether an 'upstream' remote exists at all - # before spending a network fetch on it. Non-fork installs have no - # 'upstream' remote, and the old flow burned a failed network attempt - # (~0.3-1 s) on every --check before falling back to origin. - has_upstream_remote = ( - subprocess.run( - git_cmd + ["remote", "get-url", "upstream"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ).returncode - == 0 - ) - fetch_result = None - if has_upstream_remote: - print("→ Fetching from upstream...") - fetch_result = subprocess.run( - git_cmd + ["fetch"] + depth_args + ["upstream", branch], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if fetch_result is not None and fetch_result.returncode == 0: - upstream_exists = True - compare_branch = f"upstream/{branch}" - else: - # No upstream remote, or the upstream fetch failed — use origin. - print("→ Fetching from origin...") - fetch_result = subprocess.run( - git_cmd + ["fetch"] + depth_args + ["origin", branch], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - upstream_exists = False - compare_branch = f"origin/{branch}" - else: - # Non-default branch: compare against origin/ directly. - print("→ Fetching from origin...") - fetch_result = subprocess.run( - git_cmd + ["fetch"] + depth_args + ["origin", branch], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - upstream_exists = False - compare_branch = f"origin/{branch}" - - if fetch_result.returncode != 0: - stderr = fetch_result.stderr.strip() - if "Could not resolve host" in stderr or "unable to access" in stderr: - print("✗ Network error — cannot reach the remote repository.") - elif "Authentication failed" in stderr or "could not read Username" in stderr: - print("✗ Authentication failed — check your git credentials or SSH key.") - else: - print("✗ Failed to fetch.") - if stderr: - print(f" {stderr.splitlines()[0]}") - sys.exit(1) - - # Verify the compare ref actually exists before asking rev-list about it. - # Without this, `git rev-list HEAD..origin/ --count` exits 128 and - # (with check=True) raises CalledProcessError, surfacing a Python - # traceback. Friendlier to detect-and-report. - verify_result = subprocess.run( - git_cmd + ["rev-parse", "--verify", "--quiet", compare_branch], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if verify_result.returncode != 0: - print(f"✗ Branch '{branch}' not found on {compare_branch.split('/', 1)[0]}.") - sys.exit(1) - - if is_shallow: - # No history to count across the shallow boundary. Compare tip SHAs and - # report presence-only (mirrors the banner's _check_via_local_git). - head_sha = subprocess.run( - git_cmd + ["rev-parse", "HEAD"], - cwd=PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", - ).stdout.strip() - target_sha = subprocess.run( - git_cmd + ["rev-parse", compare_branch], - cwd=PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", - ).stdout.strip() - if head_sha and target_sha and head_sha == target_sha: - print("✓ Already up to date.") - else: - print(f"⚕ Update available (behind {compare_branch}).") - from hermes_cli.config import recommended_update_command - - print(f" Run '{recommended_update_command()}' to install.") - return - - rev_result = subprocess.run( - git_cmd + ["rev-list", f"HEAD..{compare_branch}", "--count"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=True, - ) - behind = int(rev_result.stdout.strip()) - - if behind == 0: - print("✓ Already up to date.") - else: - commits_word = "commit" if behind == 1 else "commits" - print(f"⚕ Update available: {behind} {commits_word} behind {compare_branch}.") - from hermes_cli.config import recommended_update_command - - print(f" Run '{recommended_update_command()}' to install.") - - -def _ensure_fhs_path_guard() -> None: - """Ensure /usr/local/bin is on PATH for RHEL-family root non-login shells. - - Mirrors the post-symlink probe added to ``scripts/install.sh`` so that - existing FHS-layout root installs on RHEL/CentOS/Rocky/Alma 8+ get - repaired on ``hermes update`` without requiring a reinstall. The - installer's assumption that ``/usr/local/bin`` is on PATH for every - standard shell breaks on those distros in non-login interactive shells - (su, sudo -s, tmux panes, some web terminals): /etc/bashrc doesn't - add /usr/local/bin and /root/.bash_profile doesn't either. Symptom: - ``hermes`` prints ``command not found`` even though the symlink lives - at /usr/local/bin/hermes. - - Silent no-op on: non-Linux, non-root, non-FHS installs, and any system - where ``bash -i -c 'command -v hermes'`` already resolves. Idempotent. - """ - if sys.platform != "linux": - return - try: - if os.geteuid() != 0: # windows-footgun: ok — Linux FHS helper, guarded by sys.platform == "linux" above + AttributeError catch - return - except AttributeError: - return - # Only act when this is actually an FHS-layout install (command link at - # /usr/local/bin/hermes, code at /usr/local/lib/hermes-agent). - fhs_link = Path("/usr/local/bin/hermes") - if not fhs_link.is_symlink() and not fhs_link.exists(): - return - - # Probe a fresh non-login interactive bash the way the user will use it. - # ``bash -i -c`` sources ~/.bashrc but NOT ~/.bash_profile or /etc/profile, - # which is the exact scenario where RHEL root loses /usr/local/bin. - home = os.environ.get("HOME") or "/root" - try: - probe = subprocess.run( - [ - "env", - "-i", - f"HOME={home}", - f"TERM={os.environ.get('TERM', 'dumb')}", - "bash", - "-i", - "-c", - "command -v hermes", - ], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=10, - ) - except (FileNotFoundError, subprocess.TimeoutExpired): - return # no bash or probe hung — don't block update on this - if probe.returncode == 0: - return # already on PATH, nothing to do - - path_line = 'export PATH="/usr/local/bin:$PATH"' - path_comment = ( - "# Hermes Agent — ensure /usr/local/bin is on PATH " "(RHEL non-login shells)" - ) - wrote_any = False - for candidate in (".bashrc", ".bash_profile"): - cfg = Path(home) / candidate - if not cfg.is_file(): - continue - try: - existing = cfg.read_text(errors="replace", encoding="utf-8") - except OSError: - continue - # Idempotency: skip if any uncommented PATH= line already references - # /usr/local/bin. Mirrors the grep pattern used by install.sh. - already_guarded = any( - "/usr/local/bin" in line - and "PATH" in line - and not line.lstrip().startswith("#") - for line in existing.splitlines() - ) - if already_guarded: - continue - try: - with cfg.open("a", encoding="utf-8") as f: - f.write("\n" + path_comment + "\n" + path_line + "\n") - except OSError as e: - print(f" ⚠ Could not update {cfg}: {e}") - continue - print(f" ✓ Added /usr/local/bin to PATH in {cfg}") - wrote_any = True - if wrote_any: - print(" (reload your shell or run 'source ~/.bashrc' to pick it up)") - - -def _ensure_acp_launcher() -> None: - """Self-heal: install a ``hermes-acp`` launcher next to the ``hermes`` one. - - Mirrors the launcher block in ``scripts/install.sh`` so existing installs - gain the ACP command on ``hermes update`` without a reinstall. ACP hosts - (Zed, JetBrains, Buzz Desktop) spawn the agent by resolving the - ``hermes-acp`` command name against the login-shell PATH; the console - script of that name lives inside the install's venv, which is not on that - PATH, so those hosts report Hermes as not installed even when it is. - - The shim simply delegates to the sibling ``hermes`` launcher with the - ``acp`` subcommand, which makes it correct for every install layout - (venv wrapper, FHS symlink, pipx/pip console script) without having to - reconstruct interpreter/entrypoint paths. - - No-op on Windows (install.ps1 puts ``venv\\Scripts`` on the user PATH, so - ``hermes-acp.exe`` already resolves) and wherever a ``hermes-acp`` is - already present next to the ``hermes`` command. Unwritable directories - (e.g. ``/usr/local/bin`` as non-root) are skipped silently. Idempotent. - """ - if sys.platform == "win32": - return - for bin_dir in (Path.home() / ".local" / "bin", Path("/usr/local/bin")): - hermes_cmd = bin_dir / "hermes" - acp_cmd = bin_dir / "hermes-acp" - try: - if not (hermes_cmd.is_file() or hermes_cmd.is_symlink()): - continue - # Already present — a console script (pip/pipx install), an - # earlier shim, or a symlink. is_symlink() catches broken - # symlinks that exists() would miss; never follow-and-overwrite - # (the #21454 failure mode). - if acp_cmd.exists() or acp_cmd.is_symlink(): - continue - shim = ( - "#!/usr/bin/env bash\n" - "# Hermes Agent — ACP launcher (written by `hermes update`).\n" - "# ACP hosts (Zed, JetBrains, Buzz) resolve the agent by this\n" - "# command name on the login-shell PATH.\n" - f'exec "{hermes_cmd}" acp "$@"\n' - ) - acp_cmd.write_text(shim, encoding="utf-8") - acp_cmd.chmod(acp_cmd.stat().st_mode | 0o755) - except OSError: - continue - print(f" ✓ Installed hermes-acp launcher → {acp_cmd}") - - def _size_delta_label(saved_mb: float) -> str: """Human label for a before/after database size delta, in MB. @@ -11294,849 +9158,6 @@ def _size_delta_label(saved_mb: float) -> str: return f"grew by {-saved_mb:.1f} MB" -_PRE_UPDATE_SNAPSHOT_KEEP = 1 - -# Per-file size cap for the pre-update quick snapshot. Anything larger is -# skipped with a warning: the snapshot exists to protect small, hard-to- -# regenerate state (pairing JSONs, cron jobs, config, auth) — not to copy a -# multi-GB state.db on every update (observed: a 24 GB state.db added ~60s -# of wall time and silently ate 24 GB of disk per update). -_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE = 1 << 30 # 1 GiB - - -def _resolve_pre_update_backup_mode(args) -> str: - """Resolve the pre-update backup mode: ``"off"``, ``"quick"``, or ``"full"``. - - CLI flags win over config; ``--no-backup`` beats ``--backup`` when both - are set. Config accepts the mode strings plus legacy booleans: - ``true`` → ``full`` (the old zip behavior), ``false`` → ``off`` - (an explicit opt-out now disables the quick snapshot too — previously - it ran unconditionally, ignoring the user's setting). A missing key - defaults to ``quick``. - """ - if getattr(args, "no_backup", False): - return "off" - if getattr(args, "backup", False): - return "full" - - try: - from hermes_cli.config import load_config - - cfg = load_config() - except Exception as exc: - logging.getLogger(__name__).debug( - "Could not load config for pre-update backup: %s", exc - ) - cfg = {} - - updates_cfg = cfg.get("updates", {}) if isinstance(cfg, dict) else {} - raw = updates_cfg.get("pre_update_backup", "quick") - - if raw is True: - return "full" - if raw is False: - return "off" - mode = str(raw).strip().lower() - if mode in ("off", "false", "none", "disabled"): - return "off" - if mode in ("full", "zip", "true"): - return "full" - if mode == "quick": - return "quick" - logging.getLogger(__name__).warning( - "Unknown updates.pre_update_backup value %r — using 'quick'", raw - ) - return "quick" - - -def _run_pre_update_backup(args) -> Optional[str]: - """Run the pre-update safety backup and return the quick-snapshot id. - - Single consolidated mechanism gated on ``updates.pre_update_backup``: - - - ``off`` — nothing runs. Explicit user opt-out is honored fully. - - ``quick`` (default) — a state snapshot of critical small files - (pairing JSONs, cron jobs, config, auth; see ``_QUICK_STATE_FILES``) - under ``state-snapshots/``. Files over 1 GiB are skipped with a - warning so a bloated state.db can never stall the update - (issues #15733, #34600 are the reason this safety net exists). - - ``full`` — the quick snapshot PLUS a full zip of HERMES_HOME under - ``backups/`` (restorable via ``hermes import``; the #48200 wrong-path - wipe is the reason this level exists). - - ``--backup`` forces ``full`` for one run; ``--no-backup`` forces ``off``. - Never raises — a backup failure should not block the update itself. - - Returns the quick-snapshot id (used by the post-update cron-jobs - restore safety net), or ``None`` when mode is ``off`` or the snapshot - failed. - """ - mode = _resolve_pre_update_backup_mode(args) - - if mode == "off": - if getattr(args, "no_backup", False): - print("◆ Pre-update backup: skipped (--no-backup)") - print() - # Config-level off is silent — the user opted out; don't spam them - # on every update. - return None - - snapshot_id = None - try: - from hermes_cli.backup import ( - _quick_snapshot_root, - create_quick_snapshot, - verify_sqlite_integrity, - ) - - # NOTE: this function later does `from hermes_constants import - # get_hermes_home`, which makes the name function-local — the - # module-level import is shadowed and unbound here. Alias explicitly. - from hermes_cli.config import get_hermes_home as _get_home - - snapshot_id = create_quick_snapshot( - label="pre-update", - keep=_PRE_UPDATE_SNAPSHOT_KEEP, - max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, - ) - - # After the snapshot, verify the source state.db is still intact. - # The snapshot was taken via _safe_copy_db (read-only SQLite backup - # API), but a concurrent process (antivirus, force-killed gateway - # releasing file handles, Windows filter driver) can corrupt the live - # file at any point. A silent zeroing at this point would proceed with - # the update and exit code 0 — exactly the #68474 symptom. - if snapshot_id: - _src_path = _get_home() / "state.db" - if _src_path.exists(): - _integrity = verify_sqlite_integrity( - _src_path, - check_header=True, - run_pragma=True, - max_bytes=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, - ) - if not _integrity.get("valid"): - _msg = _integrity.get("message", "unknown error") - print( - f" ⚠ state.db integrity check FAILED after snapshot: {_msg}" - ) - # Check if the snapshot itself is valid. - _snap_root = _quick_snapshot_root(_get_home()) - _snap_state = _snap_root / snapshot_id / "state.db" - if _snap_state.exists(): - _snap_ok = verify_sqlite_integrity( - _snap_state, check_header=True, run_pragma=True - ) - if _snap_ok.get("valid"): - print( - " ✓ Snapshot copy is valid — continuing update." - ) - print( - " If state.db is lost after update it will be auto-restored." - ) - else: - print( - " ✗ Snapshot copy ALSO failed integrity — " - "the source was already corrupted before the backup." - ) - else: - print( - " ⚠ Snapshot does not contain state.db (was skipped or too large)." - ) - print() - if snapshot_id: - print(f"◆ Pre-update snapshot: {snapshot_id}") - except Exception as exc: - # Never let a snapshot failure block an update. - logging.getLogger(__name__).debug("Pre-update snapshot failed: %s", exc) - - if mode != "full": - if snapshot_id: - print() - return snapshot_id - - try: - from hermes_cli.backup import create_pre_update_backup - except Exception as exc: - print( - f"⚠ Pre-update backup: could not load backup module ({exc}); continuing update." - ) - print() - return snapshot_id - - try: - from hermes_cli.config import load_config - - _keep = (load_config() or {}).get("updates", {}).get("backup_keep", 5) - except Exception: - _keep = 5 - - print("◆ Creating pre-update backup...") - t0 = _time.monotonic() - try: - out_path = create_pre_update_backup(keep=int(_keep)) - except Exception as exc: # defensive — helper already swallows, but just in case - print(f" ⚠ Backup failed: {exc}") - print(" Continuing with update.") - print() - return snapshot_id - - elapsed = _time.monotonic() - t0 - - if out_path is None: - print(" ⚠ Backup skipped (no files found or write failed); continuing update.") - print() - return snapshot_id - - try: - size_bytes = out_path.stat().st_size - except OSError: - size_bytes = 0 - - # Human-readable size - size_str = f"{size_bytes} B" - for unit in ("KB", "MB", "GB"): - if size_bytes < 1024: - break - size_bytes /= 1024 - size_str = f"{size_bytes:.1f} {unit}" - - # Render path using display_hermes_home so the user sees ~/.hermes/... - try: - from hermes_constants import get_hermes_home, display_hermes_home - - home = get_hermes_home() - try: - display_path = f"{display_hermes_home()}/{out_path.relative_to(home)}" - except ValueError: - display_path = str(out_path) - except Exception: - display_path = str(out_path) - - print(f" Saved: {display_path} ({size_str}, {elapsed:.1f}s)") - print(f" Restore: hermes import {out_path}") - print(" Disable: set updates.pre_update_backup: quick (or off) in config.yaml") - print() - return snapshot_id - - -def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool: - """Write a planned-stop marker into a specific profile home.""" - try: - from datetime import timezone - - from gateway.status import _get_process_start_time - from utils import atomic_json_write - - record = { - "target_pid": pid, - "target_start_time": _get_process_start_time(pid), - "stopper_pid": os.getpid(), - "written_at": datetime.now(timezone.utc).isoformat(), - } - atomic_json_write( - Path(profile_path) / ".gateway-planned-stop.json", - record, - indent=None, - separators=(",", ":"), - ) - return True - except (OSError, PermissionError): - return False - - -def _wait_for_windows_update_gateway_exit( - pids: list[int], *, timeout: float -) -> set[int]: - """Wait for the given gateway PIDs to exit, returning survivors.""" - if not pids: - return set() - - from gateway.status import _pid_exists - - remaining = set(pids) - deadline = _time.monotonic() + max(timeout, 0.0) - while remaining and _time.monotonic() < deadline: - for pid in list(remaining): - try: - if not _pid_exists(pid): - remaining.discard(pid) - except Exception: - remaining.discard(pid) - if remaining: - _time.sleep(0.25) - - survivors: set[int] = set() - for pid in remaining: - try: - if _pid_exists(pid): - survivors.add(pid) - except Exception: - pass - return survivors - - -def _venv_core_imports_healthy() -> tuple[bool, str]: - """Probe the project venv for the core imports the backend needs to boot. - - Runs a tiny import check inside the venv interpreter (NOT this process — - ``hermes update`` may be driven by a different Python). Catches the - half-updated-venv state: git checkout current but a dependency sync that - failed or was killed partway (e.g. Windows access-denied on a loaded - .pyd), leaving imports like ``fastapi``'s new transitive deps missing. - Without this probe, ``hermes update`` on a current checkout prints - "Already up to date!" and returns without ever re-syncing dependencies — - the user's install stays broken no matter how many times they update - (ryanc's incident, July 2026). - - Returns ``(healthy, detail)``. Never raises; unknown states report - healthy so a probe failure can't force needless reinstalls. - """ - venv_dir = PROJECT_ROOT / "venv" - python_name = "python.exe" if _is_windows() else "python" - bin_dir = "Scripts" if _is_windows() else "bin" - venv_python = venv_dir / bin_dir / python_name - if not venv_python.exists(): - # No venv interpreter at all. In a dev checkout that's normal (the - # dev may run hermes from any interpreter), so report healthy to - # avoid forcing reinstalls. But on a MANAGED install (the Windows - # installer / desktop bootstrap stamps `.hermes-bootstrap-complete`, - # and an interrupted update leaves `.update-incomplete`), the venv - # IS the install — its absence means a repair got interrupted after - # the old venv was moved aside, and "Already up to date!" would - # gaslight the user while nothing can run. - managed_markers = ( - PROJECT_ROOT / ".hermes-bootstrap-complete", - _update_marker_path(), - ) - if any(m.exists() for m in managed_markers): - return False, f"venv python missing ({venv_python})" - return True, "" - - # Core web/serve imports plus their newest transitive deps. Import (not - # just metadata) — a package can have intact dist-info but a missing - # module after an interrupted uninstall/install cycle. - check = ( - "import importlib\n" - "mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n" - "missing = []\n" - "for m in mods:\n" - " try: importlib.import_module(m)\n" - " except Exception as e: missing.append(f'{m}: {e}')\n" - "print('\\n'.join(missing))\n" - ) - try: - result = subprocess.run( - [str(venv_python), "-c", check], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=60, - cwd=PROJECT_ROOT, - ) - except Exception as exc: - logger.debug("venv health probe failed to run: %s", exc) - return True, "" - - missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] - if result.returncode != 0 and not missing: - # Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy. - detail = (result.stderr or "").strip().splitlines() - return False, detail[0] if detail else "venv python failed to run" - if missing: - return False, "; ".join(missing[:4]) - return True, "" - - -def _detect_venv_python_processes( - *, exclude_pids: set[int] | None = None -) -> list[tuple[int, str, str]]: - """Find live processes running from the project venv's interpreter. - - The hermes.exe shim guard misses the biggest lock-holder class on - Windows: the Desktop app's backend (``python.exe -m hermes_cli.main - serve``) and anything else running straight off ``venv\\Scripts\\python - (w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a - dependency sync mid-update dies with access-denied and strands the venv - half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026). - - Killing them from here is pointless — the Desktop app supervises its - backend and respawns it within seconds — so the caller should refuse and - tell the user to close the app instead. Returns ``(pid, name, cmdline)`` - tuples; empty off-Windows / without psutil / when nothing matches. The - calling process and its ancestors are always excluded (a CLI ``hermes - update`` itself runs from the venv python). Never raises. - """ - if not _is_windows(): - return [] - try: - import psutil - except Exception: - return [] - - venv_dir = PROJECT_ROOT / "venv" - try: - venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep - except OSError: - venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep - try: - root_prefix = str(PROJECT_ROOT.resolve()).lower().rstrip(os.sep) + os.sep - except OSError: - root_prefix = str(PROJECT_ROOT).lower().rstrip(os.sep) + os.sep - - skip: set[int] = set(exclude_pids or set()) - skip.add(os.getpid()) - try: - for anc in psutil.Process().parents(): - skip.add(int(anc.pid)) - except Exception: - pass - - matches: list[tuple[int, str, str]] = [] - try: - proc_iter = psutil.process_iter(["pid", "exe", "name", "cmdline", "cwd"]) - except Exception: - return [] - for proc in proc_iter: - try: - info = proc.info - except Exception: - continue - pid = info.get("pid") - exe = info.get("exe") - if not exe or pid is None or int(pid) in skip: - continue - try: - exe_norm = str(Path(exe).resolve()).lower() - except (OSError, ValueError): - exe_norm = str(exe).lower() - cmdline_raw = " ".join(info.get("cmdline") or []) - cmdline_low = cmdline_raw.lower() - cwd_low = str(info.get("cwd") or "").lower().rstrip(os.sep) + os.sep - - # Primary match: the executable itself lives under this venv - # (venv\Scripts\python(w).exe — the desktop backend / gateway case). - is_holder = exe_norm.startswith(venv_prefix) - # Fallback: uv/base-interpreter trampolines run a python whose exe is - # OUTSIDE the venv but which still imports from it and holds its .pyd - # files. Catch those by what they're running: a cmdline that references - # this venv's path, or a `-m hermes_cli.main ...` invocation tied to - # this install (install root in the cmdline or as the working dir). - if not is_holder and venv_prefix in cmdline_low: - is_holder = True - if not is_holder and "hermes_cli.main" in cmdline_low: - if root_prefix in cmdline_low or cwd_low.startswith(root_prefix): - is_holder = True - if not is_holder: - continue - name = info.get("name") or Path(exe).name - matches.append((int(pid), str(name), cmdline_raw[:120])) - return matches - - -def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str: - """Explain which venv processes block the update and how to clear them.""" - lines = [ - "✗ Other Hermes processes are running from this install's venv:", - ] - for pid, name, cmdline in matches[:6]: - hint = "" - low = cmdline.lower() - if "serve" in low or "dashboard" in low: - hint = " ← Hermes Desktop backend (close the desktop app)" - elif "gateway" in low: - hint = " ← gateway" - lines.append(f" PID {pid} {name} {cmdline}{hint}") - if len(matches) > 6: - lines.append(f" ... and {len(matches) - 6} more") - lines.append("") - lines.append( - " On Windows these keep native extension files (.pyd) locked, so the" - ) - lines.append( - " dependency update would fail partway and leave a broken install." - ) - lines.append( - " Close the Hermes desktop app / other Hermes terminals, then re-run:" - ) - lines.append(" hermes update") - lines.append(" (or use `hermes update --force-venv` to proceed anyway at your own risk)") - return "\n".join(lines) - - -def _pause_windows_gateways_for_update() -> dict | None: - """Stop running Windows gateways before mutating the checkout or venv. - - Windows scheduled/startup gateways run through pythonw.exe, so the generic - hermes.exe concurrent-instance guard does not see them. They still import - from the checkout and can keep files locked while ``git`` or ``uv`` updates - the install. Stop only PIDs that the gateway discovery code identifies. - """ - if not _is_windows(): - return None - - try: - from gateway.status import terminate_pid - from hermes_cli.gateway import ( - _capture_gateway_argv, - _get_restart_drain_timeout, - find_gateway_pids, - find_profile_gateway_processes, - ) - except Exception as exc: - logger.debug("Could not prepare Windows gateway pause for update: %s", exc) - return None - - try: - running_pids = list(dict.fromkeys(find_gateway_pids(all_profiles=True))) - except Exception as exc: - logger.debug("Could not discover Windows gateway PIDs before update: %s", exc) - return None - if not running_pids: - # No gateway is running right now, but the user may have installed an - # autostart entry (Scheduled Task or Startup-folder login item) — that - # is an explicit "I want a gateway" signal. A gateway that died between - # updates (e.g. the spawning terminal/TUI closed, taking its child with - # it) would otherwise never come back: the autostart entry only fires on - # the next login, and the update flow's resume path only relaunched - # gateways that were running when the update began. Cold-start one after - # the update so an installed gateway is actually up post-update. Users - # who run gateway-less (no autostart entry) get nothing forced on them. - try: - from hermes_cli import gateway_windows - - if gateway_windows.is_installed(): - return { - "resume_needed": True, - "profiles": {}, - "unmapped_pids": [], - "unmapped": [], - "cold_start_if_installed": True, - } - except Exception as exc: - logger.debug( - "Could not check Windows gateway autostart state before update: %s", - exc, - ) - return None - - profile_processes = {} - try: - profile_processes = { - proc.pid: proc for proc in find_profile_gateway_processes() - } - except Exception as exc: - logger.debug("Could not map Windows gateway PIDs to profiles: %s", exc) - - profiles: dict[str, int] = {} - mapped_pids = [] - for pid in running_pids: - proc = profile_processes.get(pid) - if proc is None: - continue - profiles[str(proc.profile)] = int(pid) - mapped_pids.append(int(pid)) - _write_update_planned_stop_marker(Path(proc.path), int(pid)) - - print("→ Stopping Windows gateway process(es) before updating Hermes...") - try: - drain_timeout = max(float(_get_restart_drain_timeout()), 1.0) - except Exception: - drain_timeout = 10.0 - survivors = _wait_for_windows_update_gateway_exit( - mapped_pids, - timeout=drain_timeout, - ) - unmapped_pids = [pid for pid in running_pids if pid not in profile_processes] - - # Snapshot each unmapped gateway's command line *before* we force-kill it, - # so ``_resume_windows_gateways_after_update`` can respawn it by replaying - # its own argv. Unmapped gateways are ones with no profile→PID-file mapping - # — e.g. a Windows Scheduled Task running ``pythonw.exe -m hermes_cli.main - # gateway run``. Without this snapshot they were force-killed and never - # restarted (the "Restart manually after update" dead-end from #50090). - unmapped: list[dict] = [] - for pid in unmapped_pids: - argv = None - try: - argv = _capture_gateway_argv(int(pid)) - except Exception as exc: - logger.debug("Could not capture argv for unmapped gateway %s: %s", pid, exc) - unmapped.append({"pid": int(pid), "argv": argv}) - - force_killed = [] - for pid in sorted(set(survivors).union(unmapped_pids)): - try: - terminate_pid(int(pid), force=True) - force_killed.append(int(pid)) - except (ProcessLookupError, PermissionError, OSError): - pass - - if profiles: - print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}") - if force_killed: - print(f" → Force-stopped {len(force_killed)} gateway process(es)") - - if unmapped_pids: - respawnable = sum(1 for u in unmapped if u.get("argv")) - print( - f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping" - ) - if respawnable < len(unmapped_pids): - # Some had no recoverable command line (psutil missing, access - # denied, already gone): those still need a manual restart. - print(" Restart manually after update: hermes gateway run") - - return { - "resume_needed": True, - "profiles": profiles, - "unmapped_pids": unmapped_pids, - "unmapped": unmapped, - } - - -def _cold_start_windows_gateway_after_update() -> None: - """Start a fresh detached gateway after update when one is installed but down. - - Invoked from ``_resume_windows_gateways_after_update`` for the - ``cold_start_if_installed`` case: no gateway was running when the update - began, but an autostart entry (Scheduled Task / Startup-folder login item) - is installed, signalling the user wants a gateway. Unlike the relaunch - paths — which watch an old PID and respawn once it exits — this is a direct - fresh spawn via the same hidden-console + breakaway path that - ``hermes gateway start`` uses (``gateway_windows._spawn_detached``). - - Best-effort and idempotent: re-checks that nothing is running first so a - concurrent start (e.g. the autostart entry firing) can't produce a - duplicate gateway. - """ - if not _is_windows(): - return - try: - from hermes_cli import gateway_windows - from hermes_cli.gateway import find_gateway_pids - except Exception as exc: - logger.debug("Could not load Windows gateway cold-start helpers: %s", exc) - return - - # Re-check liveness right before spawning — between pause and resume the - # autostart entry may have already brought a gateway up, or a leftover - # process may have re-registered. Don't double-start. - try: - if list(find_gateway_pids(all_profiles=True)): - return - except Exception as exc: - logger.debug("Could not re-check gateway liveness before cold-start: %s", exc) - return - - try: - pid = gateway_windows._spawn_detached() - except Exception as exc: - logger.debug("Could not cold-start Windows gateway after update: %s", exc) - return - - if pid: - print() - print(f" ✓ Starting Windows gateway after update (PID {pid})") - - - -def _for_each_systemd_gateway_unit( - list_units_stdout: str, - *, - process_unit, - on_unit_timeout, -) -> None: - """Process each ``hermes-gateway*.service`` from ``systemctl list-units``. - - ``subprocess.TimeoutExpired`` raised by ``process_unit`` is isolated to - that unit via ``on_unit_timeout`` so one wedged systemctl call cannot - abort the rest of the fleet (#68523). - """ - for line in (list_units_stdout or "").strip().splitlines(): - parts = line.split() - if not parts: - continue - unit = parts[0] - if not unit.endswith(".service"): - continue - # list-units is already pattern-filtered, but keep the name gate so a - # stray non-gateway line cannot enter the restart path. - if not unit.startswith("hermes-gateway"): - continue - svc_name = unit.removesuffix(".service") - try: - process_unit(svc_name) - except subprocess.TimeoutExpired as exc: - on_unit_timeout(svc_name, exc) - - -def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None: - """Print an explicit incomplete-update warning for unrestarted units.""" - if not failed_units: - return - # Preserve discovery order while de-duplicating. - seen = set() - ordered = [] - for name in failed_units: - if name in seen: - continue - seen.add(name) - ordered.append(name) - print() - print("⚠ Update incomplete — some gateway units were not restarted:") - for name in ordered: - print(f" - {name}") - print(" Skipped units may still be running pre-update code (mixed") - print(" sys.modules). Restart them manually, then verify:") - print(" hermes gateway status") - print(" systemctl --user restart # user-scope") - print(" sudo systemctl restart # system-scope") - - -def _refresh_windows_gateway_launchers() -> None: - """Regenerate installed Windows gateway launcher scripts after update. - - The Scheduled Task / Startup-folder launchers (``gateway.cmd`` + - ``gateway.vbs``) are persistence artifacts written once at install time — - ``hermes update`` never touched them, so installs created before the - hidden-console rework (aa2ae36c3f) kept launching the gateway through - ``pythonw.exe`` forever: every descendant spawn flashed a conhost - (#54220/#56747) and, since #70344, the console-less gateway died at - startup with ``RuntimeError: sys.stderr is None`` (#71671). - - The task's /TR points at a stable script path, so rewriting the files in - place retargets the task without any schtasks call (no UAC needed). - ``_write_task_script`` is idempotent and renders from current code, so - this is a no-op for modern installs. Best-effort: a failed refresh must - never fail the update. - """ - if not _is_windows(): - return - try: - from hermes_cli import gateway_windows - - if not gateway_windows.is_installed(): - return - gateway_windows._write_task_script() - print(" ✓ Refreshed Windows gateway launcher scripts") - except Exception as exc: - logger.debug("Could not refresh Windows gateway launchers after update: %s", exc) - - -def _resume_windows_gateways_after_update(token: dict | None) -> None: - """Restart Windows profile gateways previously paused for update.""" - if not token or not token.get("resume_needed"): - return - token["resume_needed"] = False - if not _is_windows(): - return - - # Regenerate the persisted launcher scripts before respawning anything, - # so a legacy pythonw-era Scheduled Task / Startup entry comes back on - # the current hidden-console design at the next login too. - _refresh_windows_gateway_launchers() - - profiles = token.get("profiles") or {} - unmapped = token.get("unmapped") or [] - cold_start = bool(token.get("cold_start_if_installed")) - if not profiles and not any(u.get("argv") for u in unmapped): - if cold_start: - _cold_start_windows_gateway_after_update() - return - - try: - from hermes_cli.gateway import ( - launch_detached_gateway_restart_by_cmdline, - launch_detached_profile_gateway_restart, - ) - except Exception as exc: - logger.debug("Could not load Windows gateway restart helper: %s", exc) - return - - relaunched = [] - for profile, old_pid in sorted(profiles.items()): - try: - if launch_detached_profile_gateway_restart(str(profile), int(old_pid)): - relaunched.append(str(profile)) - except Exception as exc: - logger.debug( - "Could not restart Windows gateway profile %s after update: %s", - profile, - exc, - ) - - # Respawn unmapped gateways (no profile→PID-file mapping, e.g. a Scheduled - # Task) by replaying the argv we snapshotted before force-killing them. - unmapped_relaunched = 0 - for entry in unmapped: - argv = entry.get("argv") - old_pid = entry.get("pid") - if not argv or not old_pid: - continue - try: - if launch_detached_gateway_restart_by_cmdline(int(old_pid), list(argv)): - unmapped_relaunched += 1 - except Exception as exc: - logger.debug( - "Could not restart unmapped Windows gateway (pid %s) after update: %s", - old_pid, - exc, - ) - - if relaunched: - print() - print(f" ✓ Restarting Windows gateway profile(s): {', '.join(relaunched)}") - if unmapped_relaunched: - if not relaunched: - print() - print( - f" ✓ Restarting {unmapped_relaunched} unmapped Windows gateway process(es)" - ) - - -def _discard_lockfile_churn(git_cmd, repo_root): - """Restore tracked ``package-lock.json`` files that npm dirtied locally. - - npm rewrites lockfiles non-deterministically at install/build time. On a - managed install those diffs are never intentional, so we discard them so - ``hermes update`` sees a clean tree instead of autostashing every run. - Best-effort; only ever touches files named ``package-lock.json``. - """ - try: - diff = subprocess.run( - git_cmd + ["diff", "--name-only"], - cwd=repo_root, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if diff.returncode != 0: - return - dirty_package_dirs = { - Path(line.strip()).parent - for line in diff.stdout.splitlines() - if line.strip().endswith("package.json") - } - dirty = [ - line.strip() - for line in diff.stdout.splitlines() - if line.strip().endswith("package-lock.json") - and Path(line.strip()).parent not in dirty_package_dirs - ] - if not dirty: - return - subprocess.run( - git_cmd + ["checkout", "--", *dirty], - cwd=repo_root, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=False, - ) - print(f"→ Discarded npm lockfile churn ({len(dirty)} file(s))") - except Exception: - # Never let lockfile cleanup block an update. - pass - - def cmd_update(args): """Update Hermes Agent to the latest version. @@ -12193,1856 +9214,6 @@ def cmd_update(args): _finalize_update_output(_update_io_state) -def _cmd_update_impl(args, gateway_mode: bool): - """Body of ``cmd_update`` — kept separate so the wrapper can always - restore stdio even on ``sys.exit``.""" - # In gateway mode, use file-based IPC for prompts instead of stdin - gw_input_fn = ( - (lambda prompt, default="": _gateway_prompt(prompt, default)) - if gateway_mode - else None - ) - assume_yes = bool(getattr(args, "yes", False)) - - # Whether this update is running without a human at the keyboard. - # Interactive terminal updates always stash-and-ask (unchanged behavior); - # only non-interactive updates (desktop/chat app, gateway, `--yes`) consult - # the `updates.non_interactive_local_changes` config setting to decide - # whether to auto-restore stashed local source changes or throw them away. - _non_interactive_update = ( - gateway_mode - or assume_yes - or not (sys.stdin.isatty() and sys.stdout.isatty()) - ) - discard_local_changes = False - if _non_interactive_update: - try: - from hermes_cli.config import load_config - - _update_cfg = (load_config() or {}).get("updates", {}) - if isinstance(_update_cfg, dict): - _mode = str(_update_cfg.get("non_interactive_local_changes", "stash")).lower() - discard_local_changes = _mode == "discard" - except Exception as exc: - # Never let a config read failure change the safe default. - logger.debug("Could not read updates.non_interactive_local_changes: %s", exc) - discard_local_changes = False - - print("⚕ Updating Hermes Agent...") - print() - - # On Windows, abort early if another hermes.exe is holding the venv shim - # open. Continuing would result in a string of WinError 32 warnings and - # then either a deferred-rename leftover or a failed git-pull fast path - # that silently falls back to the slower ZIP route. See issue #26670. - if _is_windows() and not getattr(args, "force", False): - scripts_dir = _venv_scripts_dir() - if scripts_dir is not None: - concurrent = _detect_concurrent_hermes_instances(scripts_dir) - if concurrent: - print(_format_concurrent_instances_message(concurrent, scripts_dir)) - sys.exit(2) - - # Pre-update backup — runs before any git/file mutation so users can - # always roll back to the exact state they had before this update. - # Returns the quick-snapshot id (or None when disabled/failed); the - # post-update cron-jobs safety net uses it to detect job loss. - pre_update_snapshot_id = _run_pre_update_backup(args) - - _windows_gateway_resume = _pause_windows_gateways_for_update() - if _windows_gateway_resume: - import atexit as _atexit - - _atexit.register( - _resume_windows_gateways_after_update, - _windows_gateway_resume, - ) - - # With gateways paused, anything still running from the venv interpreter - # (most commonly the Desktop app's `hermes serve` backend) will keep .pyd - # files locked and corrupt the dependency sync below. Refuse rather than - # race: killing the desktop backend is futile (the app supervises and - # respawns it), so the user must close the app. Deliberately NOT bypassed - # by plain --force: the desktop bootstrap updater passes --force to skip - # the hermes.exe shim guard above, but its lock probe only checks the shim - # and app.asar — a non-desktop venv python holding a .pyd would sail - # through and corrupt the sync (the exact failure this guard exists for). - # --force-venv is the explicit escape hatch. - if _is_windows() and not getattr(args, "force_venv", False): - _venv_holders = _detect_venv_python_processes() - if _venv_holders: - print(_format_venv_python_holders_message(_venv_holders)) - _resume_windows_gateways_after_update(_windows_gateway_resume) - sys.exit(2) - - # Try git-based update first, fall back to ZIP download on Windows - # when git file I/O is broken (antivirus, NTFS filter drivers, etc.) - use_zip_update = False - git_dir = PROJECT_ROOT / ".git" - - if not git_dir.exists(): - if sys.platform == "win32": - use_zip_update = True - else: - print("✗ Not a git repository. Please reinstall:") - print( - " curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash" - ) - sys.exit(1) - - # On Windows, git can fail with "unable to write loose object file: Invalid argument" - # due to filesystem atomicity issues. Set the recommended workaround. - if sys.platform == "win32" and git_dir.exists(): - subprocess.run( - [ - "git", - "-c", - "windows.appendAtomically=false", - "config", - "windows.appendAtomically", - "false", - ], - cwd=PROJECT_ROOT, - check=False, - capture_output=True, - ) - - # Build git command once — reused for fork detection and the update itself. - git_cmd = ["git"] - if sys.platform == "win32": - git_cmd = ["git", "-c", "windows.appendAtomically=false"] - - # Discard npm lockfile churn before any stash/branch logic. npm rewrites - # tracked package-lock.json files non-deterministically at install/build - # time (platform-specific optional deps, ideallyInert annotations, etc.), - # which is never an intentional edit on a managed install but leaves the - # tree dirty — forcing an autostash on every update and making branch - # switches fragile. Restoring them first lets the common case (only - # lockfile churn) update with a clean tree. - _discard_lockfile_churn(git_cmd, PROJECT_ROOT) - - # Detect if we're updating from a fork (before any branch logic) - origin_url = _get_origin_url(git_cmd, PROJECT_ROOT) - is_fork = _is_fork(origin_url) - - if is_fork: - print("⚠ Updating from fork:") - print(f" {origin_url}") - print() - - if use_zip_update: - # ZIP-based update for Windows when git is broken - try: - _update_via_zip(args) - finally: - _resume_windows_gateways_after_update(_windows_gateway_resume) - return - - # Fetch and pull - try: - - # Resolve the target branch up front so the fetch can be scoped to it. - # A bare `git fetch origin` pulls every ref, and this repo carries - # thousands of auto-generated branches — an unscoped fetch can stall for - # minutes on a non-single-branch checkout. Fetch only what we update - # against. - branch = _resolve_update_branch(args) - - print("→ Fetching updates...") - fetch_result = subprocess.run( - git_cmd + ["fetch", "origin", branch], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if fetch_result.returncode != 0: - stderr = fetch_result.stderr.strip() - if "Could not resolve host" in stderr or "unable to access" in stderr: - print("✗ Network error — cannot reach the remote repository.") - print(f" {stderr.splitlines()[0]}" if stderr else "") - elif ( - "Authentication failed" in stderr or "could not read Username" in stderr - ): - print( - "✗ Authentication failed — check your git credentials or SSH key." - ) - else: - print("✗ Failed to fetch updates from origin.") - if stderr: - print(f" {stderr.splitlines()[0]}") - sys.exit(1) - - # Get current branch (returns literal "HEAD" when detached) - result = subprocess.run( - git_cmd + ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=True, - ) - current_branch = result.stdout.strip() - - # If user is on a different branch than the update target, switch - # to the target. When the target is "main" this is the historical - # "always update against main" behavior; for any other target it's - # the same thing — get HEAD onto the requested branch first, then - # fast-forward. - if current_branch != branch: - label = ( - "detached HEAD" - if current_branch == "HEAD" - else f"branch '{current_branch}'" - ) - print(f" ⚠ Currently on {label} — switching to {branch} for update...") - # Stash before checkout so uncommitted work isn't lost - auto_stash_ref = _stash_local_changes_if_needed(git_cmd, PROJECT_ROOT) - checkout_result = subprocess.run( - git_cmd + ["checkout", branch], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if checkout_result.returncode != 0: - # Local checkout doesn't have this branch yet. Try to set - # it up as a tracking branch of origin/. This is - # the common case when the requested branch exists upstream - # but was never checked out locally. - track_result = subprocess.run( - git_cmd + ["checkout", "-B", branch, f"origin/{branch}"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if track_result.returncode != 0: - # Restore the user's prior branch + stash before bailing - # so we don't leave them stranded in a weird state. - if auto_stash_ref is not None: - _restore_stashed_changes( - git_cmd, - PROJECT_ROOT, - auto_stash_ref, - prompt_user=False, - input_fn=gw_input_fn, - ) - print(f"✗ Branch '{branch}' does not exist locally or on origin.") - if track_result.stderr.strip(): - print(f" {track_result.stderr.strip().splitlines()[0]}") - sys.exit(1) - else: - auto_stash_ref = _stash_local_changes_if_needed(git_cmd, PROJECT_ROOT) - - prompt_for_restore = ( - auto_stash_ref is not None - and not assume_yes - and (gateway_mode or (sys.stdin.isatty() and sys.stdout.isatty())) - ) - - # Check if there are updates - result = subprocess.run( - git_cmd + ["rev-list", f"HEAD..origin/{branch}", "--count"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=True, - ) - commit_count = int(result.stdout.strip()) - - if commit_count == 0: - _invalidate_update_cache() - - # Even if origin is up to date, the fork may be behind upstream - if is_fork and branch == "main": - _sync_with_upstream_if_needed(git_cmd, PROJECT_ROOT) - - # Restore stash and switch back to original branch if we moved - if auto_stash_ref is not None: - _restore_stashed_changes( - git_cmd, - PROJECT_ROOT, - auto_stash_ref, - prompt_user=prompt_for_restore, - input_fn=gw_input_fn, - ) - if current_branch not in {branch, "HEAD"}: - subprocess.run( - git_cmd + ["checkout", current_branch], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=False, - ) - - # "No new commits" does not mean the managed interpreter is safe. - # uv can retain the same CPython patch while python-build-standalone - # refreshes the embedded SQLite underneath it. Keep the existing - # update-boundary hook active on this retry path too. - from hermes_cli.managed_uv import ensure_uv, update_managed_uv - - runtime_repairs = [] - update_managed_uv(repair_observer=runtime_repairs.append) - ensure_uv(repair_observer=runtime_repairs.append) - runtime_repaired = next( - (result for result in runtime_repairs if result.repaired), - None, - ) - - # A current checkout does NOT imply a healthy install: a previous - # dependency sync may have failed partway (classic on Windows, - # where a running gateway/desktop backend keeps .pyd files locked - # and uv/pip dies with access-denied, stranding the venv between - # versions). Probe the venv's core imports and repair if broken — - # otherwise "Already up to date!" gaslights the user while their - # install stays bricked. - healthy, detail = _venv_core_imports_healthy() - if not healthy: - print("⚠ Checkout is current, but the venv is unhealthy:") - print(f" {detail}") - print("→ Repairing Python dependencies...") - _write_update_incomplete_marker() - from hermes_cli.managed_uv import ensure_uv - - repair_uv = ensure_uv() - # A managed install whose venv is gone entirely (interrupted - # repair after the old venv was moved aside) needs the venv - # recreated before dependencies can be installed into it. - venv_python_missing = not ( - PROJECT_ROOT - / "venv" - / ("Scripts" if _is_windows() else "bin") - / ("python.exe" if _is_windows() else "python") - ).exists() - if venv_python_missing and repair_uv: - print("→ Recreating virtual environment...") - subprocess.run( - [repair_uv, "venv", "venv"], - cwd=PROJECT_ROOT, - check=False, - ) - if repair_uv: - repair_env = {**os.environ, "VIRTUAL_ENV": str(PROJECT_ROOT / "venv")} - _install_python_dependencies_with_optional_fallback( - [repair_uv, "pip"], env=repair_env, group="all" - ) - else: - _install_python_dependencies_with_optional_fallback( - [sys.executable, "-m", "pip"], group="all" - ) - _clear_update_incomplete_marker() - healthy_after, detail_after = _venv_core_imports_healthy() - if healthy_after: - print("✓ Dependencies repaired!") - else: - print(f"⚠ Venv still unhealthy after repair: {detail_after}") - print(" Close all Hermes windows/gateways and re-run: hermes update") - else: - print("✓ Already up to date!") - if runtime_repaired is not None and not _is_windows(): - print() - print( - "⚠ Restart required to finish the managed Python runtime repair." - ) - print( - " Any running Hermes gateways, Desktop backends, or other " - "long-lived processes still use the previous runtime." - ) - print( - " Restart each of them before removing the parked venv" - + ( - f": {runtime_repaired.backup_venv}" - if runtime_repaired.backup_venv is not None - else "." - ) - ) - _resume_windows_gateways_after_update(_windows_gateway_resume) - return - - print(f"→ Found {commit_count} new commit(s)") - - print("→ Pulling updates...") - update_succeeded = False - # Capture the pre-pull SHA so we can auto-roll-back if the new code - # has a syntax error in a critical-path file (PR #28452 incident: - # orphan merge-conflict markers in hermes_cli/config.py bricked - # every user who ran ``hermes update`` for the 7 minutes between - # the bad commit and the fix landing). - pre_pull_sha = _capture_head_sha(git_cmd, PROJECT_ROOT) - try: - # Merge the ref we already fetched above (→ Fetching updates...) - # instead of `git pull`, which performs a SECOND network fetch of - # the same branch (~0.5-1.5 s of redundant round-trip per update). - # `merge --ff-only origin/` is byte-identical in effect to - # `pull --ff-only origin ` given the fresh tracking ref; - # the divergence fallback below is unchanged. - pull_result = subprocess.run( - git_cmd + ["merge", "--ff-only", f"origin/{branch}"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if pull_result.returncode != 0: - # ff-only failed — local and remote have diverged (e.g. upstream - # force-pushed or rebase). Since local changes are already - # stashed, reset to match the remote exactly. - print( - " ⚠ Fast-forward not possible (history diverged), resetting to match remote..." - ) - reset_result = subprocess.run( - git_cmd + ["reset", "--hard", f"origin/{branch}"], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if reset_result.returncode != 0: - print(f"✗ Failed to reset to origin/{branch}.") - if reset_result.stderr.strip(): - print(f" {reset_result.stderr.strip()}") - print( - f" Try manually: git fetch origin && git reset --hard origin/{branch}" - ) - sys.exit(1) - - # Post-pull syntax guard: validate critical-path files actually - # parse before declaring the update successful. If a bad commit - # made it through CI (e.g. admin-merge bypass of a failing - # ruff check), this catches it on the user side and rolls back - # so the CLI stays bootable. The user can then retry ``hermes - # update`` later once a fix lands upstream. - syntax_ok, failing_path, syntax_error = _validate_critical_files_syntax( - PROJECT_ROOT - ) - if not syntax_ok: - print() - print("✗ Pulled code has a syntax error in a critical file:") - print(f" {failing_path}") - if syntax_error: - # py_compile errors can be multi-line; show the first - # ~6 lines so the user sees the actual SyntaxError text. - for line in str(syntax_error).splitlines()[:6]: - print(f" {line}") - if pre_pull_sha: - print() - print(f"→ Rolling back to {pre_pull_sha[:10]}...") - rollback_result = subprocess.run( - git_cmd + ["reset", "--hard", pre_pull_sha], - cwd=PROJECT_ROOT, - capture_output=True, - text=True, encoding="utf-8", errors="replace", - ) - if rollback_result.returncode == 0: - print(" ✓ Rollback complete — your install is unchanged.") - print(" Try ``hermes update`` again later once a fix lands.") - else: - print(" ✗ Rollback failed. Recover manually with:") - print(f" cd {PROJECT_ROOT} && git reset --hard {pre_pull_sha}") - if rollback_result.stderr.strip(): - print(f" ({rollback_result.stderr.strip().splitlines()[0]})") - else: - print() - print(" Could not capture pre-pull SHA — recover manually with:") - print(f" cd {PROJECT_ROOT} && git reflog && git reset --hard ") - sys.exit(1) - - update_succeeded = True - finally: - if auto_stash_ref is not None: - # Don't attempt stash restore if the code update itself failed — - # working tree is in an unknown state. - if not update_succeeded: - print( - f" ℹ️ Local changes preserved in stash (ref: {auto_stash_ref})" - ) - print(" Restore manually with: git stash apply") - elif discard_local_changes: - # Non-interactive update + user opted into discarding local - # source edits (updates.non_interactive_local_changes: - # discard). Throw the stash away instead of re-applying it. - _discard_stashed_changes( - git_cmd, - PROJECT_ROOT, - auto_stash_ref, - ) - else: - _restore_stashed_changes( - git_cmd, - PROJECT_ROOT, - auto_stash_ref, - prompt_user=prompt_for_restore, - input_fn=gw_input_fn, - ) - - _invalidate_update_cache() - - # Clear stale .pyc bytecode cache — prevents ImportError on gateway - # restart when updated source references names that didn't exist in - # the old bytecode (e.g. get_hermes_home added to hermes_constants). - removed = _clear_bytecode_cache(PROJECT_ROOT) - if removed: - print( - f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" - ) - _record_bytecode_fingerprint() - - # Fork upstream sync logic (only for main branch on forks) - if is_fork and branch == "main": - _sync_with_upstream_if_needed(git_cmd, PROJECT_ROOT) - - # Reinstall Python dependencies. Prefer .[all], but if one optional extra - # breaks on this machine, keep base deps and reinstall the remaining extras - # individually so update does not silently strip working capabilities. - # - # Drop the core-install breadcrumb BEFORE touching the venv. If the - # install is killed mid-flight (Ctrl-C, terminal close, WSL OOM), the - # marker survives and the next ``hermes`` launch finishes the install - # via ``_recover_from_interrupted_install``. Cleared after the core - # ``.[all]`` install completes — lazy refresh uses a separate marker. - _write_update_incomplete_marker() - print("→ Updating Python dependencies...") - from hermes_cli.managed_uv import ensure_uv, update_managed_uv - - # Keep managed uv current — runs `uv self update` if we already have one. - update_managed_uv() - - uv_bin = ensure_uv() - - pip_cmd = [sys.executable, "-m", "pip"] - if not uv_bin: - uv_bin = _ensure_uv_for_termux(pip_cmd) - install_group = "all" - - if uv_bin: - uv_env = {**os.environ, "VIRTUAL_ENV": str(PROJECT_ROOT / "venv")} - if _is_termux_env(uv_env): - uv_env.pop("PYTHONPATH", None) - uv_env.pop("PYTHONHOME", None) - install_group = "termux-all" - print(" → Termux detected: using uv + curated termux-all optional profile...") - if _is_termux_env(uv_env) and _is_android_python(): - print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...") - _install_psutil_android_compat([uv_bin, "pip"], env=uv_env) - _install_python_dependencies_with_optional_fallback( - [uv_bin, "pip"], env=uv_env, group=install_group - ) - else: - # Use sys.executable to explicitly call the venv's pip module, - # avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu. - # Some environments lose pip inside the venv; bootstrap it back with - # ensurepip before trying the editable install. - pip_cmd = [sys.executable, "-m", "pip"] - try: - subprocess.run( - pip_cmd + ["--version"], - cwd=PROJECT_ROOT, - check=True, - capture_output=True, - ) - except subprocess.CalledProcessError: - subprocess.run( - [sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], - cwd=PROJECT_ROOT, - check=True, - ) - if _is_termux_env(): - install_group = "termux-all" - print(" → Termux detected: using curated termux-all optional profile...") - if _is_termux_env() and _is_android_python(): - print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...") - _install_psutil_android_compat(pip_cmd) - _install_python_dependencies_with_optional_fallback(pip_cmd, group=install_group) - - install_prefix = [uv_bin, "pip"] if uv_bin else pip_cmd - lazy_env = uv_env if uv_bin else None - - # Core ``.[all]`` install finished. Clear the generic core breadcrumb - # before the lazy-refresh phase — that phase uses its own marker so a - # later lazy failure cannot be "healed" by clearing the core marker - # based on a narrow 7-package import probe (#58004 review). - _clear_update_incomplete_marker() - - # The update process is still the old Python interpreter process. Run - # one final cache/module refresh immediately before lazy backend - # refresh, which imports newly-pulled modules that may depend on fresh - # symbols in hermes_constants or lazy_deps. The dependency install - # above may also have regenerated bytecode from build-cache copies — - # this second sweep catches those stragglers (#60242, #65240). - removed = _clear_bytecode_cache(PROJECT_ROOT) - if removed: - print( - f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" - ) - _record_bytecode_fingerprint() - _reload_updated_runtime_modules() - - # Upgrade pip before lazy refreshes — stale pip can fail source builds - # and leave partially-written packages (#57828). - _write_lazy_refresh_incomplete_marker() - _upgrade_pip_before_lazy_refresh(install_prefix, env=lazy_env) - - # Lazy refresh can corrupt the venv when a backend install fails. - # Clear the lazy marker only when refresh/repair is confirmed healthy. - lazy_ok = _refresh_active_lazy_features(install_prefix, env=lazy_env) - if lazy_ok: - _clear_lazy_refresh_incomplete_marker() - else: - print( - " ⚠ Lazy-refresh recovery incomplete — run `hermes` again " - "to finish import-based venv repair." - ) - - # Heal the active memory provider's bridge packages last — the core - # reinstall + lazy refresh above may have stripped or downgraded - # plugin.yaml-declared deps that aren't in extras (#53272, #70636). - _refresh_active_memory_provider_dependencies() - - node_failures = _update_node_dependencies() - _build_web_ui(PROJECT_ROOT / "web") - - # Rebuild the desktop app if the source tree changed since the last - # build. ``hermes desktop --build-only`` uses the content-hash stamp - # internally, so this is effectively a no-op when nothing changed. - # Only bother if the user has a desktop app installed (indicated by - # an existing packaged executable or desktop dist); people who have - # never run ``hermes desktop`` shouldn't be forced into a full - # Electron build by ``hermes update``. - desktop_dir = PROJECT_ROOT / "apps" / "desktop" - has_desktop_app = _desktop_packaged_executable(desktop_dir) is not None or _desktop_dist_exists(desktop_dir) - if (desktop_dir / "package.json").exists() and _resolve_node_runtime_npm() and has_desktop_app: - print("→ Checking if desktop app needs rebuilding...") - # Consult the content-hash stamp IN-PROCESS first. The spawned - # `hermes desktop --build-only` subprocess re-imports the whole - # CLI stack (~1-3 s) just to reach the same _desktop_build_needed - # check; when the stamp already says "up to date" we can skip the - # spawn entirely. The update path never passes --source, so the - # subprocess would run with source_mode=False — mirror that here. - # Any error in the pre-check falls through to the subprocess. - _skip_desktop_build = False - try: - _skip_desktop_build = not _desktop_build_needed( - desktop_dir, PROJECT_ROOT, source_mode=False - ) - except Exception: - _skip_desktop_build = False - if _skip_desktop_build: - print(" ✓ Desktop app up to date") - else: - _desktop_build_cmd = [sys.executable, "-m", "hermes_cli.main", "desktop", "--build-only"] - # Capture the (very loud) Electron/vite build output into - # update.log instead of streaming it to the terminal. On the rare - # nonzero exit, retry once after waiting again for the venv — this - # covers a still-settling rebuild window the first wait didn't fully - # catch — then surface the captured tail so the failure is - # debuggable. - # - # Start the build subprocess with the Hermes-managed Node on PATH: - # when `hermes update` runs inside the desktop updater chain - # (Desktop → hermes-setup → hermes update), the shell PATH - # customizations are lost, so a bare-PATH child would fail with - # `node: not found` before cmd_gui can self-heal. - from hermes_constants import with_hermes_node_path - - _build_env = with_hermes_node_path() - build_result = _run_logged_subprocess(_desktop_build_cmd, cwd=PROJECT_ROOT, env=_build_env) - if build_result.returncode != 0: - build_result = _run_logged_subprocess(_desktop_build_cmd, cwd=PROJECT_ROOT, env=_build_env) - if build_result.returncode != 0: - print(" ⚠ Desktop build failed (non-fatal; run `hermes desktop` to retry)") - tail = "\n".join((build_result.stdout or "").strip().splitlines()[-15:]) - if tail: - print(tail) - from hermes_constants import display_hermes_home as _dhh - print(f" Full build log: {_dhh()}/logs/update.log") - else: - print(" ✓ Desktop app up to date") - - print() - print("✓ Code updated!") - - # ── Post-update state.db integrity guard (#68474) ───────────────── - # Verify that state.db survived the update intact. If the live file - # is now corrupted (zeroed, missing header, integrity failure), - # automatically restore from the pre-update snapshot rather than - # letting the user discover silently that their sessions are gone. - try: - from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity - - _state_path = get_hermes_home() / "state.db" - if _state_path.exists(): - _state_ok = verify_sqlite_integrity( - _state_path, - check_header=True, - run_pragma=True, - ) - if _state_ok.get("valid"): - logger.debug( - "Post-update state.db integrity check: %s", - _state_ok.get("message"), - ) - else: - print() - print( - "⚠ state.db is corrupted after update: " - + _state_ok.get("message", "unknown error") - ) - _pre_snap_id = pre_update_snapshot_id - if _pre_snap_id: - _snap_state = ( - _quick_snapshot_root(get_hermes_home()) - / _pre_snap_id - / "state.db" - ) - if _snap_state.exists(): - _snap_ok = verify_sqlite_integrity( - _snap_state, check_header=True, run_pragma=True - ) - if _snap_ok.get("valid"): - try: - import shutil as _shutil - - _shutil.copy2(_snap_state, _state_path) - _restored_ok = verify_sqlite_integrity( - _state_path, - check_header=True, - run_pragma=True, - ) - if _restored_ok.get("valid"): - print( - " ✓ Auto-restored from pre-update " - f"snapshot ({_pre_snap_id})" - ) - else: - print( - " ✗ Auto-restore FAILED — restored " - "copy also failed integrity" - ) - except OSError as _exc: - print( - f" ✗ Auto-restore file copy failed: {_exc}" - ) - else: - print( - " ✗ Pre-update snapshot also failed integrity" - ) - else: - print( - " ⚠ Pre-update snapshot does not contain state.db" - ) - else: - print(" ⚠ No pre-update snapshot was taken") - print() - except Exception as exc: - logger.debug("Post-update state.db integrity check failed: %s", exc) - - # Seed the model-catalog disk cache from the freshly-pulled checkout. - # The repo ships the canonical catalog at - # website/static/api/model-catalog.json, and `git pull` just made it - # current — so copy it straight over ~/.hermes/cache/model_catalog.json - # instead of waiting on a network fetch (which can be bot-gated or hit a - # Portal hiccup). Keeps the model picker's curated/free lists in sync - # with the version the user just installed. Non-fatal on failure: the - # normal network refresh still applies on the next picker open. - try: - from hermes_cli.model_catalog import seed_cache_from_checkout - - if seed_cache_from_checkout(PROJECT_ROOT): - print(" ✓ Model catalog cache refreshed from checkout") - except Exception as e: - logger.debug("Model catalog seed during update failed: %s", e) - - # Sync bundled skills (copies new, updates changed, respects user deletions) - try: - from tools.skills_sync import sync_skills - - print() - print("→ Syncing bundled skills...") - result = sync_skills(quiet=True) - if result["copied"]: - print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}") - if result.get("updated"): - print( - f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}" - ) - if result.get("user_modified"): - print(f" ~ {len(result['user_modified'])} user-modified (kept)") - print( - " → see them: hermes skills list-modified " - "(diff/reset to resume updates)" - ) - if result.get("cleaned"): - print(f" − {len(result['cleaned'])} removed from manifest") - if result.get("relocated"): - print( - f" → {len(result['relocated'])} moved to new upstream paths: " - f"{', '.join(result['relocated'])}" - ) - if not result["copied"] and not result.get("updated"): - print(" ✓ Skills are up to date") - except Exception as e: - logger.debug("Skills sync during update failed: %s", e) - - # Sync bundled skills to all profiles (including the active one). - # seed_profile_skills() uses subprocess with an explicit HERMES_HOME so - # it is not affected by sync_skills()'s module-level HERMES_HOME cache, - # which means the active profile is reliably synced regardless of whether - # the caller's HERMES_HOME env var points at the default or a named profile. - try: - from hermes_cli.profiles import ( - list_profiles, - seed_profile_skills, - ) - - all_profiles = list_profiles() - if all_profiles: - print() - print("→ Syncing bundled skills to all profiles...") - for p in all_profiles: - try: - r = seed_profile_skills(p.path, quiet=True) - if r and r.get("skipped_opt_out"): - status = "opted out (--no-skills)" - elif r: - copied = len(r.get("copied", [])) - updated = len(r.get("updated", [])) - modified = len(r.get("user_modified", [])) - parts = [] - if copied: - parts.append(f"+{copied} new") - if updated: - parts.append(f"↑{updated} updated") - if modified: - parts.append(f"~{modified} user-modified") - status = ", ".join(parts) if parts else "up to date" - else: - status = "sync failed" - print(f" {p.name}: {status}") - except Exception as pe: - print(f" {p.name}: error ({pe})") - except Exception: - pass # profiles module not available or no profiles - - # Backfill per-profile .env files for profiles created before the - # .env-seeding fix (#44792). Copies the default install's .env so - # those profiles keep the credentials they were effectively using. - try: - from hermes_cli.profiles import backfill_profile_envs - - backfilled = backfill_profile_envs(quiet=True) - if backfilled: - print() - print( - f"→ Seeded .env for {len(backfilled)} profile(s) " - f"(copied from default): {', '.join(backfilled)}" - ) - except Exception: - pass # profiles module not available or no profiles - - # Sync Honcho host blocks to all profiles - try: - from plugins.memory.honcho.cli import sync_honcho_profiles_quiet - - synced = sync_honcho_profiles_quiet() - if synced: - print(f"\n-> Honcho: synced {synced} profile(s)") - except Exception: - pass # honcho plugin not installed or not configured - - # Check for config migrations - print() - print("→ Checking configuration for new options...") - - from hermes_cli.config import ( - get_missing_env_vars, - get_missing_config_fields, - check_config_version, - migrate_config, - ) - - missing_env = get_missing_env_vars(required_only=True) - missing_config = get_missing_config_fields() - current_ver, latest_ver = check_config_version() - - has_new_options = bool(missing_env or missing_config) - version_bump_only = ( - not has_new_options and current_ver < latest_ver - ) - needs_migration = has_new_options or current_ver < latest_ver - - if version_bump_only: - # Nothing for the user to fill in — only the config format version - # changed (new defaults already merge in transparently). Asking - # "configure new options now?" here is misleading: saying yes just - # bumps the version and looks like a no-op (issue: ScottFive / - # Tt2021). Apply it silently and say what actually happened. - print() - print( - f" ℹ Updating config format (v{current_ver} → v{latest_ver})…" - ) - try: - migrate_config(interactive=False, quiet=True) - print(" ✓ Config format updated (no new settings to configure)") - except Exception as _mig_err: - print(f" ⚠️ Config format update failed: {_mig_err}") - print(" Run 'hermes config migrate' to retry.") - elif needs_migration: - print() - # Show WHAT changed, not just a count, so the user can make an - # informed yes/no decision (previously the prompt named nothing). - def _print_items(items, label, key, fallback_key=None): - if not items: - return - print(f" {label}:") - shown = items[:8] - for it in shown: - if isinstance(it, dict): - name = it.get(key) or (fallback_key and it.get(fallback_key)) or "?" - desc = (it.get("description") or "").strip() - else: - # Defensive: some callers/mocks pass bare name strings. - name = str(it) - desc = "" - if desc: - print(f" • {name} — {desc}") - else: - print(f" • {name}") - extra = len(items) - len(shown) - if extra > 0: - print(f" … and {extra} more") - - if missing_env: - print( - f" ⚠️ {len(missing_env)} new required setting(s) need configuration" - ) - _print_items(missing_env, "New settings", "name") - if missing_config: - print(f" ℹ️ {len(missing_config)} new config option(s) available") - _print_items(missing_config, "New options", "key") - - print() - if assume_yes: - print( - " ℹ --yes: auto-applying config migration (skipping API-key prompts)." - ) - response = "y" - elif gateway_mode: - response = ( - _gateway_prompt( - "Would you like to configure new options now? [Y/n]", "n" - ) - .strip() - .lower() - ) - elif not (sys.stdin.isatty() and sys.stdout.isatty()): - print(" ℹ Non-interactive session — applying safe config migrations.") - response = "auto" - else: - try: - response = ( - input("Would you like to configure them now? [Y/n]: ") - .strip() - .lower() - ) - except EOFError: - response = "n" - - if response in {"", "y", "yes", "auto"}: - print() - # Gateway mode, --yes, and non-interactive update contexts - # (dashboard / web server actions) cannot prompt for API keys. - # Still run the non-interactive migration pass before restarting - # so new default config fields and version bumps are written - # before the freshly updated gateway validates config at startup. - interactive_migration = not ( - gateway_mode or assume_yes or response == "auto" - ) - results = migrate_config(interactive=interactive_migration, quiet=False) - - if results["env_added"] or results["config_added"]: - print() - print("✓ Configuration updated!") - if (gateway_mode or assume_yes or response == "auto") and missing_env: - print(" ℹ API keys require manual entry: hermes config migrate") - else: - print() - print("Skipped. Run 'hermes config migrate' later to configure.") - else: - print(" ✓ Configuration is up to date") - - # Safety net: config-version migrations have been observed to leave - # cron/jobs.json valid-but-empty, silently dropping every scheduled - # job (issue #34600). The desktop scheduler can also overwrite with - # its own small set, causing partial loss (issue #52144). If the - # live file now has fewer jobs than the pre-update snapshot, restore - # it and warn loudly. - try: - from hermes_cli.backup import restore_cron_jobs_if_emptied - - cron_restore = restore_cron_jobs_if_emptied(pre_update_snapshot_id) - if cron_restore: - print() - print( - " ⚠️ cron/jobs.json lost jobs during this update — " - f"restored {cron_restore['job_count']} job(s) from " - f"pre-update snapshot {cron_restore['snapshot_id']}." - ) - except Exception as exc: - # Never let the cron safety net break an otherwise-good update. - logger.debug("Cron jobs auto-restore check failed: %s", exc) - - print() - if node_failures: - print( - "⚠ Update partially complete — Node.js dependencies for " - f"{', '.join(node_failures)} did not refresh." - ) - print(" Code and Python deps are updated, but the dashboard/TUI may") - print(" be in a mixed state until the Node deps are rebuilt.") - else: - print("✓ Update complete!") - - # Search-index optimization notice (v23). Existing installs keep their - # working search index untouched on update; the compact v23 layout — - # which reclaims a large fraction of state.db on heavy users — is - # opt-in. Surface it here (the moment the user is already thinking - # about their install) with the exact command and the concrete size - # win. Show-once-ish: only when a legacy index is actually present. - try: - _print_fts_optimize_available_notice() - except Exception as e: - logger.debug("FTS optimize notice failed: %s", e) - - # Curator first-run heads-up. Only prints when curator is enabled AND - # has never run — i.e. the window where the ticker would otherwise - # have fired against a fresh skill library. Kept silent on steady - # state so we don't nag. - try: - _print_curator_first_run_notice() - except Exception as e: - logger.debug("Curator first-run notice failed: %s", e) - - # Most-recent curator run notice — show-once per run. Surfaces the - # rename map (`old-name → umbrella`) on the high-attention update - # surface so users learn about consolidations without having to - # check `hermes curator status`. Self-stamps after printing so it - # never repeats for the same run. - try: - _print_curator_recent_run_notice() - except Exception as e: - logger.debug("Curator recent-run notice failed: %s", e) - - # Repair RHEL-family root installs where /usr/local/bin isn't on PATH - # for non-login interactive shells. No-op on every other platform. - try: - _ensure_fhs_path_guard() - except Exception as e: - logger.debug("FHS PATH guard check failed: %s", e) - - # Self-heal the hermes-acp launcher for installs that predate it, so - # ACP hosts (Zed, JetBrains, Buzz) can resolve Hermes on PATH without - # a reinstall. No-op on Windows and when already present. - try: - _ensure_acp_launcher() - except Exception as e: - logger.debug("hermes-acp launcher self-heal failed: %s", e) - - # Refresh the cua-driver binary used by the Computer Use toolset. - # The upstream installer is gated on supported platforms and on the - # binary already being on PATH, so this is a no-op for users who - # don't have it. Tying the refresh to ``hermes update`` gives users a - # predictable cadence (matches when they pull new agent code) without - # adding startup latency or a per-launch GitHub API call. - try: - refresh_cua_driver = True - try: - from hermes_cli.config import load_config - - _update_cfg = (load_config() or {}).get("updates", {}) - if isinstance(_update_cfg, dict): - refresh_cua_driver = bool( - _update_cfg.get("refresh_cua_driver", True) - ) - except Exception as cfg_exc: - logger.debug("Could not read updates.refresh_cua_driver: %s", cfg_exc) - - if ( - refresh_cua_driver - and sys.platform in ("darwin", "win32", "linux") - and shutil.which("cua-driver") - ): - from hermes_cli.tools_config import install_cua_driver - - print() - print("→ Refreshing cua-driver (Computer Use)...") - # require_confirmed_update: only run the (multi-minute, - # silent) upstream installer when the driver's native - # check-update verb positively reports a newer release. - # An indeterminate check (offline, rate-limited, old - # driver) keeps the installed version — `hermes update` - # must stay fast; `hermes computer-use install --upgrade` - # remains the force path. - install_cua_driver(upgrade=True, require_confirmed_update=True) - except Exception as e: - logger.debug("cua-driver refresh failed: %s", e) - - # Write exit code *before* the gateway restart attempt. - # When running as ``hermes update --gateway`` (spawned by the gateway's - # /update command), this process lives inside the gateway's systemd - # cgroup. A graceful SIGUSR1 restart keeps the drain loop alive long - # enough for the exit-code marker to be written below, but the - # fallback ``systemctl restart`` path (see below) kills everything in - # the cgroup (KillMode=mixed → SIGKILL to remaining processes), - # including us and the wrapping bash shell. The shell never reaches - # its ``printf $status > .update_exit_code`` epilogue, so the - # exit-code marker file would never be created. The new gateway's - # update watcher would then poll for 30 minutes and send a spurious - # timeout message. - # - # Writing the marker here — after git pull + pip install succeed but - # before we attempt the restart — ensures the new gateway sees it - # regardless of how we die. - if gateway_mode: - _exit_code_path = get_hermes_home() / ".update_exit_code" - try: - _exit_code_path.write_text("0", encoding="utf-8") - except OSError: - pass - - gateway_fleet_restart_incomplete = False - - # Auto-restart ALL gateways after update. - # The code update (git pull) is shared across all profiles, so every - # running gateway needs restarting to pick up the new code. - try: - from hermes_cli.gateway import ( - is_macos, - supports_systemd_services, - _ensure_user_systemd_env, - find_gateway_pids, - find_profile_gateway_processes, - _prepare_profile_gateway_update_restart, - _get_service_pids, - _graceful_restart_via_sigusr1, - _wait_for_gateway_exit, - ) - import signal as _signal - - def _wait_for_service_active( - scope_cmd_: list, - svc_name_: str, - timeout: float = 10.0, - ) -> bool: - """Poll ``systemctl is-active`` until the unit reports active. - - systemd's Stopped -> Started transition after a graceful exit - (or a hard restart) is not instantaneous; a one-shot check - races that window and falsely reports the unit as down. - Poll every 0.5s up to ``timeout`` seconds before giving up. - """ - deadline = _time.monotonic() + max(timeout, 0.5) - while True: - try: - _verify = subprocess.run( - scope_cmd_ + ["is-active", svc_name_], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=5, - ) - if _verify.stdout.strip() == "active": - return True - except (FileNotFoundError, subprocess.TimeoutExpired): - pass - if _time.monotonic() >= deadline: - return False - _time.sleep(0.5) - - def _service_restart_sec( - scope_cmd_: list, - svc_name_: str, - default: float = 0.0, - ) -> float: - """Read the unit's ``RestartUSec`` (RestartSec) in seconds. - - After a graceful exit-75, systemd waits ``RestartSec`` before - respawning the unit. Callers that poll for ``is-active`` - must use a timeout >= ``RestartSec`` + transition slack, or - they'll give up *during* the cooldown window and wrongly - conclude the unit didn't relaunch. - """ - try: - _show = subprocess.run( - scope_cmd_ - + [ - "show", - svc_name_, - "--property=RestartUSec", - "--value", - ], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=5, - ) - except (FileNotFoundError, subprocess.TimeoutExpired): - return default - raw = (_show.stdout or "").strip() - # systemd emits values like "30s", "100ms", "1min 30s", or - # "infinity". Parse conservatively; on any miss return default. - if not raw or raw == "infinity": - return default - total = 0.0 - matched = False - for part in raw.split(): - for _suf, _mult in ( - ("ms", 0.001), - ("us", 0.000001), - ("min", 60.0), - ("s", 1.0), - ): - if part.endswith(_suf): - try: - total += float(part[: -len(_suf)]) * _mult - matched = True - except ValueError: - pass - break - return total if matched else default - - _manage_cmd_cache: dict = {} - - def _resolve_manage_cmd(scope_: str, scope_cmd_: list, svc_name_: str): - """Resolve the command prefix for manage-units operations. - - Read-only systemctl calls (``is-active``, ``show``, - ``list-units``) work unprivileged, but manage-units verbs - (``reset-failed``, ``start``, ``restart``) on a *system* - service trigger a polkit ``org.freedesktop.systemd1.manage-units`` - authentication prompt when run as a non-root user. That - interactive prompt runs inside our captured subprocess with a - 10-15s timeout — the user sees the prompt flash and "exit - directly" before they can answer, and the resulting - TimeoutExpired used to be swallowed silently. - - Strategy: if root, plain systemctl. If not root, try - non-interactive sudo (``sudo -n``) — first a blanket probe, - then a targeted ``systemctl reset-failed`` probe so a - least-privilege sudoers entry scoped to - ``systemctl ... hermes-gateway*`` also qualifies - (``reset-failed`` is an idempotent no-op we run before every - privileged restart anyway). If neither works, return None — - the caller must SKIP the restart (without draining the - gateway first!) and tell the user how to restart manually. - ``--no-ask-password`` guarantees polkit can never hang a - captured subprocess on this path. - """ - if scope_ in _manage_cmd_cache: - return _manage_cmd_cache[scope_] - cmd = scope_cmd_ + ["--no-ask-password"] - if ( - scope_ == "system" - and hasattr(os, "geteuid") - and os.geteuid() != 0 # windows-footgun: ok — systemd path, Linux-only - ): - sudo_cmd = ["sudo", "-n"] + scope_cmd_ + ["--no-ask-password"] - sudo_ok = False - try: - _probe = subprocess.run( - ["sudo", "-n", "true"], - capture_output=True, - timeout=5, - ) - sudo_ok = _probe.returncode == 0 - if not sudo_ok: - # Blanket sudo refused — a targeted sudoers entry - # (NOPASSWD for systemctl ... hermes-gateway*) - # may still allow the exact commands we need. - _probe = subprocess.run( - sudo_cmd + ["reset-failed", svc_name_], - capture_output=True, - timeout=5, - ) - sudo_ok = _probe.returncode == 0 - except (FileNotFoundError, subprocess.TimeoutExpired): - sudo_ok = False - cmd = sudo_cmd if sudo_ok else None - _manage_cmd_cache[scope_] = cmd - return cmd - - # Drain budget for graceful SIGUSR1 restarts. The gateway drains - # for up to ``agent.restart_drain_timeout`` (default 60s) before - # exiting with code 75; we wait slightly longer so the drain - # completes before we fall back to a hard restart. On older - # systemd units without SIGUSR1 wiring this wait just times out - # and we fall back to ``systemctl restart`` (the old behaviour). - try: - from hermes_constants import ( - DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT as _DEFAULT_DRAIN, - ) - except Exception: - _DEFAULT_DRAIN = 60.0 - _cfg_drain = None - try: - from hermes_cli.config import load_config - - _cfg_agent = load_config().get("agent") or {} - _cfg_drain = _cfg_agent.get("restart_drain_timeout") - except Exception: - pass - try: - _drain_budget = ( - float(_cfg_drain) - if _cfg_drain is not None - else float(_DEFAULT_DRAIN) - ) - except (TypeError, ValueError): - _drain_budget = float(_DEFAULT_DRAIN) - # Add a 15s margin so the drain loop + final exit finish before - # we escalate to ``systemctl restart`` / SIGTERM. - _drain_budget = max(_drain_budget, 30.0) + 15.0 - - restarted_services = [] - failed_or_stale_units = [] - killed_pids = set() - relaunched_profiles = [] - externally_supervised_profiles = [] - - # --- Systemd services (Linux) --- - # Discover all hermes-gateway* units (default + profiles) - if supports_systemd_services(): - try: - _ensure_user_systemd_env() - except Exception: - pass - - for scope, scope_cmd in [ - ("user", ["systemctl", "--user"]), - ("system", ["systemctl"]), - ]: - try: - result = subprocess.run( - scope_cmd - + [ - "list-units", - "hermes-gateway*", - "--plain", - "--no-legend", - "--no-pager", - ], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=10, - ) - except FileNotFoundError: - continue - except subprocess.TimeoutExpired as exc: - # Discovery timeout — skip this scope, keep the other. - print( - f" ⚠ systemctl timed out listing {scope}-scope " - f"gateway units ({exc.cmd if exc.cmd else 'unknown command'}). " - f"Check the gateway with: hermes gateway status" - ) - continue - - def _restart_one_systemd_gateway_unit(svc_name: str) -> None: - # Check if active - check = subprocess.run( - scope_cmd + ["is-active", svc_name], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=5, - ) - if check.stdout.strip() != "active": - return - - # Resolve how we may run manage-units verbs - # (reset-failed/start/restart) for this scope. - # None ⇒ no non-interactive privilege path; we - # must avoid those verbs entirely or polkit will - # throw an interactive auth prompt inside our - # captured 10-15s subprocess (the user sees it - # flash and "exit directly" — reported June 2026). - _manage_cmd = _resolve_manage_cmd( - scope, scope_cmd, svc_name - ) - - # Prefer a graceful SIGUSR1 restart so in-flight - # agent runs drain instead of being SIGKILLed. - # The gateway's SIGUSR1 handler calls - # request_restart(via_service=True) → drain → - # exit; systemd's Restart=always respawns the unit. - _main_pid = 0 - try: - _show = subprocess.run( - scope_cmd - + [ - "show", - svc_name, - "--property=MainPID", - "--value", - ], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=5, - ) - _main_pid = int((_show.stdout or "").strip() or 0) - except ( - ValueError, - subprocess.TimeoutExpired, - FileNotFoundError, - ): - _main_pid = 0 - - _graceful_ok = False - if _main_pid > 0: - print( - f" → {svc_name}: draining (up to {int(_drain_budget)}s)..." - ) - _graceful_ok = _graceful_restart_via_sigusr1( - _main_pid, - drain_timeout=_drain_budget, - ) - - if _graceful_ok: - # Gateway exited after a planned restart. - # ``Restart=always`` means systemd WILL respawn - # the unit — but only after - # ``RestartSec`` (default 60s on our unit - # file). That 60s wait is a crash-loop guard, - # and is the right default when the gateway - # dies unexpectedly. For a voluntary restart - # on update, it's dead time the user watches. - # - # Shortcut it: ``reset-failed`` + ``start`` - # skips RestartSec entirely (we're manually - # initiating the unit, not waiting for - # systemd's auto-restart logic). Takes about - # as long as the process takes to come up - # (~1-3s on a warm box). - # - # If the unit is already active because - # RestartSec elapsed while we were draining, - # ``start`` is a no-op and we fall through to - # the poll below. Either way we collapse the - # 60s+ delay to a ~5s one. - # - # The shortcut needs manage-units privileges. - # Without them (system service, non-root, no - # passwordless sudo) skip it — systemd's own - # auto-restart still relaunches the unit after - # RestartSec, no privileges required. - if _manage_cmd is not None: - subprocess.run( - _manage_cmd + ["reset-failed", svc_name], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=10, - ) - subprocess.run( - _manage_cmd + ["start", svc_name], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=15, - ) - # Short poll: the gateway should be up - # within a few seconds now that we - # bypassed RestartSec. - if _wait_for_service_active( - scope_cmd, - svc_name, - timeout=10.0, - ): - restarted_services.append(svc_name) - return - # Passive poll: systemd's auto-restart fires - # after RestartSec regardless of privileges. - # This is the primary path when _manage_cmd is - # None, and the fallback when the explicit - # start didn't take. - _restart_sec = _service_restart_sec( - scope_cmd, - svc_name, - default=0.0, - ) - _post_drain_timeout = max( - 10.0, - _restart_sec + 10.0, - ) - if _manage_cmd is None and _restart_sec > 5.0: - print( - f" → {svc_name}: waiting for systemd " - f"auto-restart (~{int(_restart_sec)}s; " - "no root for an immediate restart)..." - ) - if _wait_for_service_active( - scope_cmd, - svc_name, - timeout=_post_drain_timeout, - ): - restarted_services.append(svc_name) - return - # Process exited but wasn't respawned (older - # unit without Restart=on-failure or - # RestartForceExitStatus=75). Fall through - # to systemctl start/restart. - print( - f" ⚠ {svc_name} drained but didn't relaunch — forcing restart" - ) - - # Forcing a restart requires manage-units - # privileges. Without a non-interactive path, - # running systemctl here would spawn a polkit - # auth prompt inside a captured 10-15s subprocess - # — it flashes and dies before the user can - # answer. Skip with clear instructions instead. - if _manage_cmd is None: - failed_or_stale_units.append(svc_name) - print( - f" ⚠ {svc_name} is a system service and restarting it needs root.\n" - f" Restart it manually to load the new version:\n" - f" sudo systemctl restart {svc_name}\n" - f" To let `hermes update` restart it automatically, allow\n" - f" passwordless sudo for systemctl, or run updates with sudo." - ) - return - - # Fallback: blunt systemctl restart. This is - # what the old code always did; we get here only - # when the graceful path failed (unit missing - # SIGUSR1 wiring, drain exceeded the budget, - # restart-policy mismatch). - # - # Always `reset-failed` first. If systemd's own - # auto-restart attempts already parked the unit - # in a failed state (transient CHDIR / OOM / - # filesystem race after our drain + exit-75), - # a plain `systemctl restart` can wedge against - # the RestartSec backoff and leave the unit - # dead. Clearing the failed state first makes - # the restart idempotent. Mirrors the recovery - # path in `hermes gateway restart` - # (`systemd_restart()`) as of PR #20949. - subprocess.run( - _manage_cmd + ["reset-failed", svc_name], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=10, - ) - restart = subprocess.run( - _manage_cmd + ["restart", svc_name], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=15, - ) - if restart.returncode == 0: - # Verify the service actually survived the - # restart. systemctl restart returns 0 even - # if the new process crashes immediately. - if _wait_for_service_active( - scope_cmd, - svc_name, - timeout=10.0, - ): - restarted_services.append(svc_name) - else: - # Retry once — transient startup failures - # (stale module cache, import race) often - # resolve on the second attempt. Again - # clear any failed state first so the - # retry isn't blocked by the previous - # crash. - print( - f" ⚠ {svc_name} died after restart, retrying..." - ) - subprocess.run( - _manage_cmd + ["reset-failed", svc_name], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=10, - ) - subprocess.run( - _manage_cmd + ["restart", svc_name], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=15, - ) - if _wait_for_service_active( - scope_cmd, - svc_name, - timeout=10.0, - ): - restarted_services.append(svc_name) - print(f" ✓ {svc_name} recovered on retry") - else: - failed_or_stale_units.append(svc_name) - _scope_flag = "--user " if scope == "user" else "" - _sudo_hint = "sudo " if scope == "system" else "" - print( - f" ✗ {svc_name} failed to stay running after restart.\n" - f" Check logs: {_sudo_hint}journalctl {_scope_flag}-u {svc_name} --since '2 min ago'\n" - f" Recover manually:\n" - f" {_sudo_hint}systemctl {_scope_flag}reset-failed {svc_name}\n" - f" {_sudo_hint}systemctl {_scope_flag}restart {svc_name}" - ) - else: - failed_or_stale_units.append(svc_name) - print( - f" ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}" - ) - - def _on_unit_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None: - # Isolate the timeout to this unit and keep going - # (#68523). A scope-wide handler used to abort every - # later gateway and leave the fleet on mixed code. - failed_or_stale_units.append(svc_name) - print( - f" ⚠ systemctl timed out restarting {svc_name} " - f"({exc.cmd if exc.cmd else 'unknown command'}); " - f"continuing with remaining gateways" - ) - - _for_each_systemd_gateway_unit( - result.stdout, - process_unit=_restart_one_systemd_gateway_unit, - on_unit_timeout=_on_unit_timeout, - ) - - # --- Launchd services (macOS) --- - if is_macos(): - try: - from hermes_cli.gateway import ( - launchd_restart, - get_launchd_label, - get_launchd_plist_path, - ) - - plist_path = get_launchd_plist_path() - if plist_path.exists(): - check = subprocess.run( - ["launchctl", "list", get_launchd_label()], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - timeout=5, - ) - if check.returncode == 0: - try: - launchd_restart() - restarted_services.append(get_launchd_label()) - except subprocess.CalledProcessError as e: - stderr = (getattr(e, "stderr", "") or "").strip() - print(f" ⚠ Gateway restart failed: {stderr}") - except (FileNotFoundError, subprocess.TimeoutExpired, ImportError): - pass - - # --- Manual (non-service) gateways --- - # Kill any remaining gateway processes not managed by a service. - # Exclude PIDs that belong to just-restarted services so we don't - # immediately kill the process that systemd/launchd just spawned. - service_pids = _get_service_pids() - manual_pids = find_gateway_pids( - exclude_pids=service_pids, all_profiles=True - ) - profile_processes = { - proc.pid: proc - for proc in find_profile_gateway_processes(exclude_pids=service_pids) - if proc.pid in manual_pids - } - for pid, proc in profile_processes.items(): - restart_mode = _prepare_profile_gateway_update_restart( - proc.profile, pid - ) - if restart_mode is None: - continue - # Prefer a graceful SIGUSR1 drain so in-flight agent runs - # finish before the watcher respawns the gateway. If the - # gateway doesn't support SIGUSR1 or doesn't exit within - # the drain budget, fall back to SIGTERM — the watcher - # still sees the exit and relaunches either way. - # Announce the drain first: this wait can hold for the full - # budget per gateway with no other output, and on surfaces - # that stream update progress (the desktop updater most of - # all) the silence reads as a hung update (#44515). - print( - f" → {proc.profile}: draining gateway PID {pid} " - f"(up to {int(_drain_budget)}s)..." - ) - drained = _graceful_restart_via_sigusr1( - pid, - drain_timeout=_drain_budget, - ) - if not drained: - try: - os.kill(pid, _signal.SIGTERM) - except (ProcessLookupError, PermissionError): - pass - # Wait for the old process to fully exit before the watcher - # spawns the new gateway. Telegram holds the previous - # getUpdates long-poll session open on its servers for up to - # ~30s after the client disconnects. If the new gateway - # connects before that window expires it receives a 409 - # Conflict, which _handle_polling_conflict() recovers from - # via back-off retries — but a brief wait here reduces the - # chance of hitting that path at all, especially on fast - # machines where the watcher loop restarts in < 1s. - # We wait up to 5s for the process to exit (the OS-level - # close, not the Telegram server-side expiry), then let the - # watcher take over. The Telegram adapter's retry logic - # handles any remaining 409s if the server session is still - # live when the new gateway polls. - _wait_for_gateway_exit(timeout=5.0, force_after=None) - killed_pids.add(pid) - if restart_mode == "external-supervisor": - externally_supervised_profiles.append(proc.profile) - else: - relaunched_profiles.append(proc.profile) - - for pid in manual_pids: - if pid in profile_processes: - continue - try: - os.kill(pid, _signal.SIGTERM) - killed_pids.add(pid) - except (ProcessLookupError, PermissionError): - pass - - if restarted_services or killed_pids: - print() - for svc in restarted_services: - print(f" ✓ Restarted {svc}") - if relaunched_profiles: - names = ", ".join(relaunched_profiles) - print(f" ✓ Restarting manual gateway profile(s): {names}") - if externally_supervised_profiles: - names = ", ".join(externally_supervised_profiles) - print( - " ✓ Handed gateway profile(s) back to their external " - f"supervisor: {names}" - ) - unmapped_count = ( - len(killed_pids) - - len(relaunched_profiles) - - len(externally_supervised_profiles) - ) - if unmapped_count: - print(f" → Stopped {unmapped_count} manual gateway process(es)") - print(" Restart manually: hermes gateway run") - if unmapped_count > 1: - print( - " (or: hermes -p gateway run for each profile)" - ) - - if failed_or_stale_units: - gateway_fleet_restart_incomplete = True - if gateway_mode: - _exit_code_path = get_hermes_home() / ".update_exit_code" - try: - _exit_code_path.write_text("1", encoding="utf-8") - except OSError: - pass - _warn_incomplete_gateway_fleet_restart(failed_or_stale_units) - - if not restarted_services and not killed_pids: - # No gateways were running — nothing to do - pass - - # --- Post-restart survivor sweep ----------------------------- - # Issue #17648: some gateways ignore SIGTERM (stuck drain, - # blocked I/O, PID dead but zombie). The detached profile - # watchers wait 120s for the old PID to exit — if it never - # does, no respawn happens and the user keeps hitting - # ImportError against a stale sys.modules. Give the - # graceful paths a brief window to complete, then SIGKILL - # any remaining pre-update PIDs so the watcher / service - # manager can relaunch with fresh code. - try: - _time.sleep(3.0) - _service_pids_after = _get_service_pids() - _surviving = find_gateway_pids( - exclude_pids=_service_pids_after, - all_profiles=True, - ) - # Scope to PIDs we already tried to kill during this - # update (killed_pids). Anything new is a gateway that - # started AFTER our restart attempt — respecting user - # intent, we don't kill those. - _stuck = [pid for pid in _surviving if pid in killed_pids] - if _stuck: - print() - print( - f" ⚠ {len(_stuck)} gateway process(es) ignored SIGTERM — force-killing" - ) - from gateway.status import terminate_pid as _terminate_pid - for pid in _stuck: - try: - # Routes through taskkill /T /F on Windows, - # SIGKILL on POSIX — _signal.SIGKILL doesn't - # exist on Windows so the old raw os.kill call - # used to crash the entire update path. - _terminate_pid(pid, force=True) - except (ProcessLookupError, PermissionError, OSError): - pass - # Give the OS a beat to reap the processes so the - # watchers see them exit and respawn. - _time.sleep(1.5) - except Exception as _sweep_exc: - logger.debug("Post-restart survivor sweep failed: %s", _sweep_exc) - - except Exception as e: - logger.debug("Gateway restart during update failed: %s", e) - - _resume_windows_gateways_after_update(_windows_gateway_resume) - - # Warn if legacy Hermes gateway unit files are still installed. - # When both hermes.service (from a pre-rename install) and the - # current hermes-gateway.service are enabled, they SIGTERM-fight - # for the same bot token (see PR #11909). Flagging here means - # every `hermes update` surfaces the issue until the user migrates. - try: - from hermes_cli.gateway import ( - has_legacy_hermes_units, - _find_legacy_hermes_units, - supports_systemd_services, - ) - - if supports_systemd_services() and has_legacy_hermes_units(): - print() - print("⚠ Legacy Hermes gateway unit(s) detected:") - for name, path, is_sys in _find_legacy_hermes_units(): - scope = "system" if is_sys else "user" - print(f" {path} ({scope} scope)") - print() - print(" These pre-rename units (hermes.service) fight the current") - print(" hermes-gateway.service for the bot token and cause SIGTERM") - print(" flap loops. Remove them with:") - print() - print(" hermes gateway migrate-legacy") - print() - print(" (add `sudo` if any are in system scope)") - except Exception as e: - logger.debug("Legacy unit check during update failed: %s", e) - - # Restart a managed dashboard through systemd, or stop stale manual - # dashboard processes. Raw-killing a systemd-owned dashboard PID makes - # systemd treat it as a clean stop, leaving the Cloudflare origin dead. - # Preserve the safety rule above: a failed Node refresh leaves the - # currently running dashboard untouched. - _finish_dashboard_update_cleanup(node_failures) - - print() - print("Tip: You can now select a provider and model:") - print(" hermes model # Select provider and model") - - if gateway_fleet_restart_incomplete: - # Code update itself succeeded, but at least one gateway still - # runs pre-update modules — surface that as a failed update so - # automation / operators do not treat the fleet as healthy. - sys.exit(1) - - except subprocess.CalledProcessError as e: - if sys.platform == "win32": - print(f"⚠ Git update failed: {e}") - print("→ Falling back to ZIP download...") - print() - _update_via_zip(args) - else: - print(f"✗ Update failed: {e}") - sys.exit(1) - - def _coalesce_session_name_args(argv: list) -> list: """Join unquoted multi-word session names after -c/--continue and -r/--resume. diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py new file mode 100644 index 0000000000000..0384bdcdc67b7 --- /dev/null +++ b/hermes_cli/update_cmd.py @@ -0,0 +1,4861 @@ +"""Hermes update pipeline — extracted from ``hermes_cli/main.py``. + +Mechanical move (main.py decomposition): ``_cmd_update_impl``, ``_cmd_update_check`` +and every module-level helper used only by the update path, plus the update-only +constants they read. Function bodies are lifted verbatim; the only mechanical +change is that references to helpers/constants that STAY in ``hermes_cli.main`` +(and to moved-but-test-patched siblings) are routed through ``_m()`` — a lazy +``hermes_cli.main`` reference — so existing call sites and test monkeypatches +that target ``hermes_cli.main.`` (``PROJECT_ROOT``, ``_is_windows``, +``_run_pre_update_backup``, ...) keep working unchanged. ``main.py`` re-imports +every public-ish name from here (``# noqa: F401``) so the argparse wiring and +the test-patch surface still resolve on ``hermes_cli.main``. + +Three self-contained closures nested inside ``_cmd_update_impl`` +(``_print_items``, ``_wait_for_service_active``, ``_service_restart_sec``) were +hoisted to module level; they capture no enclosing state (verified via +``symtable``). ``_restart_one_systemd_gateway_unit``, ``_resolve_manage_cmd`` +and ``_on_unit_timeout`` DO capture enclosing locals and stay nested, +byte-identical. + +Imports are one-way: ``hermes_cli.main`` imports this module, never the reverse +at import time (``_m()`` resolves lazily at call time, when main.py is fully +loaded, so there is no import cycle). +""" + +import hashlib +import json +import logging +import os +import shlex +import shutil +import subprocess +import sys +import time as _time +from datetime import datetime +from pathlib import Path +from typing import Optional + +from hermes_cli.config import get_hermes_home + +logger = logging.getLogger(__name__) + + +def _m(): + """Lazy ``hermes_cli.main`` reference. + + Lets callers keep patching ``hermes_cli.main.`` (the historical + test surface) and have those patches reach this code path, and defers the + import so ``hermes_cli.main`` -> ``hermes_cli.update_cmd`` stays one-way + at import time. + """ + from hermes_cli import main + + return main + + +_UPDATE_RUNTIME_RELOAD_MODULES = ( + "hermes_constants", + "tools.environments.local", + "tools.lazy_deps", +) + +def _reload_updated_runtime_modules() -> None: + """Reload update-sensitive modules after the checkout changes in-place. + + ``hermes update`` keeps running in the pre-pull Python process. After a + large update, modules already present in ``sys.modules`` can still expose + old symbols even though their source files on disk are new. Refresh the + small module set used by lazy-backend refresh before that step imports + newly-updated code paths. + """ + try: + import importlib + + importlib.invalidate_caches() + for module_name in _UPDATE_RUNTIME_RELOAD_MODULES: + module = _m().sys.modules.get(module_name) + if module is None: + continue + try: + importlib.reload(module) + except Exception as exc: + logger.debug("Could not reload updated module %s: %s", module_name, exc) + except Exception as exc: + logger.debug("Could not refresh update runtime modules: %s", exc) + +# Critical files that Hermes must be able to import immediately after an +# update/install. Most are imported on every CLI startup; ``web_server.py`` +# is the desktop/dashboard backend path that a fresh Windows install launches +# right away. If any of these fail to parse after a pull, the user can be +# left with a bricked CLI or desktop backend. The post-pull syntax guard +# validates these and auto-rolls-back on failure. +_UPDATE_CRITICAL_FILES = ( + "hermes_cli/main.py", + "hermes_cli/config.py", + "hermes_cli/__init__.py", + "hermes_cli/web_server.py", + "cli.py", + "run_agent.py", + "model_tools.py", + "toolsets.py", + "hermes_constants.py", +) + +def _capture_head_sha(git_cmd, cwd) -> str | None: + """Return the current HEAD SHA, or None if it can't be resolved.""" + try: + result = subprocess.run( + git_cmd + ["rev-parse", "HEAD"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=True, + ) + return result.stdout.strip() or None + except (subprocess.CalledProcessError, OSError): + return None + +def _validate_critical_files_syntax(root) -> tuple[bool, str | None, str | None]: + """Compile each file in ``_UPDATE_CRITICAL_FILES`` to catch SyntaxErrors. + + These are the files imported on every ``hermes`` startup; if any of them + has a syntax error (orphan merge-conflict markers, bad ref to a name + that no longer exists, etc.) the CLI can't bootstrap at all. We validate + them after a successful ``git pull`` so we can auto-roll-back instead of + leaving the user with a bricked install. + + The compiled ``.pyc`` is written to a temp directory rather than the + source tree's ``__pycache__/`` so we don't race with concurrent test + workers that walk the same dir, and so we don't leave a stale pyc + behind in production if the next interpreter run picks a different + Python version. The pyc is discarded on function return either way — + we only care about the compile-or-not signal. + + Returns ``(ok, failing_path, error_message)``. ``ok=True`` means every + file parsed cleanly. + """ + import py_compile + import tempfile + + root = Path(root) + with tempfile.TemporaryDirectory(prefix="hermes-syntax-check-") as tmpdir: + for relpath in _UPDATE_CRITICAL_FILES: + path = root / relpath + if not path.exists(): + # Missing file is suspicious but not necessarily fatal — a future + # refactor may legitimately remove one of these. Skip and move on. + continue + # Mirror the relative path under the tmpdir so two different + # files with the same basename don't collide on the cfile name. + cfile = Path(tmpdir) / (relpath.replace("/", "__") + "c") + try: + py_compile.compile(str(path), cfile=str(cfile), doraise=True) + except py_compile.PyCompileError as exc: + return False, str(path), str(exc) + except OSError as exc: + return False, str(path), f"could not read: {exc}" + return True, None, None + +def _gateway_prompt(prompt_text: str, default: str = "", timeout: float = 300.0) -> str: + """File-based IPC prompt for gateway mode. + + Writes a prompt marker file so the gateway can forward the question to the + user, then polls for a response file. Falls back to *default* on timeout. + + Used by ``hermes update --gateway`` so interactive prompts (stash restore, + config migration) are forwarded to the messenger instead of being silently + skipped. + """ + import json as _json + import uuid as _uuid + from hermes_constants import get_hermes_home + + home = get_hermes_home() + prompt_path = home / ".update_prompt.json" + response_path = home / ".update_response" + + # Clean any stale response file + response_path.unlink(missing_ok=True) + + payload = { + "prompt": prompt_text, + "default": default, + "id": str(_uuid.uuid4()), + } + tmp = prompt_path.with_suffix(".tmp") + tmp.write_text(_json.dumps(payload), encoding="utf-8") + tmp.replace(prompt_path) + + # Poll for response + deadline = _time.monotonic() + timeout + while _time.monotonic() < deadline: + if response_path.exists(): + try: + answer = response_path.read_text(encoding="utf-8").strip() + response_path.unlink(missing_ok=True) + prompt_path.unlink(missing_ok=True) + return answer if answer else default + except (OSError, ValueError): + pass + _time.sleep(0.5) + + # Timeout — clean up and use default + prompt_path.unlink(missing_ok=True) + response_path.unlink(missing_ok=True) + print(f" (no response after {int(timeout)}s, using default: {default!r})") + return default + +def _npm_bin_exists(bin_dir: Path, name: str) -> bool: + """True when an npm bin shim for *name* exists (POSIX or Windows).""" + return any( + (bin_dir / candidate).exists() + for candidate in (name, f"{name}.cmd", f"{name}.ps1", f"{name}.exe") + ) + +def _web_build_toolchain_ready(*roots: Path) -> bool: + """True when ``tsc`` and ``vite`` shims are reachable from any of *roots*. + + Callers must pass every root the build would search; checking only one + reports a healthy tree as broken. + """ + bin_dirs = [ + bin_dir + for bin_dir in (root / "node_modules" / ".bin" for root in roots) + if bin_dir.is_dir() + ] + return bool(bin_dirs) and all( + any(_npm_bin_exists(bin_dir, tool) for bin_dir in bin_dirs) + for tool in ("tsc", "vite") + ) + +def _web_toolchain_roots(web_dir: Path) -> tuple[Path, ...]: + """Roots whose ``node_modules/.bin`` can satisfy the web build. + + ``npm run build`` prepends ``node_modules/.bin`` for the package and each + of its ancestors, so shims hoisted to the workspace root and shims nested + under a package that owns its lockfile (#42973) are equally valid. + """ + return (web_dir, web_dir.parent) + +def _print_curator_first_run_notice() -> None: + """Print a short heads-up about the skill curator after `hermes update`. + + Only fires when the curator is enabled AND has no recorded run yet, which + is exactly the window where the gateway ticker used to fire Curator + against a fresh skill library immediately after an update. We defer the + first real pass by one ``interval_hours``; this notice tells the user how + to preview or disable before then. Silent on steady state. + """ + try: + from agent import curator + except Exception: + return + try: + if not curator.is_enabled(): + return + state = curator.load_state() + except Exception: + return + if state.get("last_run_at"): + # Curator has run before (real or already seeded) — no notice needed. + return + try: + hours = curator.get_interval_hours() + except Exception: + hours = 24 * 7 + days = max(1, hours // 24) + print() + print("ℹ Skill curator") + print( + f" Background skill maintenance is enabled. First pass is deferred " + f"~{days}d after installation; only agent-created skills are in " + f"scope and nothing is ever auto-deleted (archive is recoverable)." + ) + print(" Preview now: hermes curator run --dry-run") + print(" Pause it: hermes curator pause") + print( + " Docs: https://hermes-agent.nousresearch.com/docs/user-guide/features/curator" + ) + +def _print_fts_optimize_available_notice() -> None: + """Advertise the opt-in v23 search-index optimization after `hermes update`. + + Only fires when the current profile's state.db is still on the legacy + (pre-v23) inline FTS layout. Leads with the reclaimable-space figure and + points at the exact command. Honors ``sessions.fts_optimize_notice``: + ``advise`` (default) prints an advisory notice, ``require`` prints a + firmer required-upgrade notice, ``off`` suppresses it. Silent for + fresh/already-optimized installs. + """ + mode = "advise" + try: + from hermes_cli.config import load_config + + mode = str( + ((load_config() or {}).get("sessions") or {}).get( + "fts_optimize_notice", "advise" + ) + ).strip().lower() + except Exception: + mode = "advise" + if mode == "off": + return + + try: + from hermes_constants import get_hermes_home + from hermes_state import SessionDB + except Exception: + return + db_path = get_hermes_home() / "state.db" + if not db_path.exists(): + return + try: + size_gb = db_path.stat().st_size / (1024 ** 3) + except OSError: + return + # Skip the notice for trivially small DBs — the win isn't worth the nag. + if size_gb < 0.5: + return + db = None + interrupted = False + try: + db = SessionDB(db_path=db_path, read_only=True) + # read_only opens skip schema init, so probe the layout directly. + row = db._conn.execute( + "SELECT sql FROM sqlite_master " + "WHERE type = 'table' AND name = 'messages_fts'" + ).fetchone() + # An interrupted `optimize-storage` run: the table is already the + # v23 shape, but backfill markers / demoted trash tables remain. + # Offer the command again — re-running resumes and finishes it. + interrupted = bool( + db._conn.execute( + "SELECT 1 FROM state_meta " + "WHERE key = 'fts_rebuild_high_water' LIMIT 1" + ).fetchone() + or db._conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1" + ).fetchone() + or db._conn.execute( + "SELECT 1 FROM state_meta WHERE key IN " + "('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1" + ).fetchone() + ) + except Exception: + return + finally: + if db is not None: + try: + db.close() + except Exception: + pass + sql = (row[0] if row else "") or "" + if not sql or ("tool_name" in sql and not interrupted): + # v23 layout already present (fresh/optimized) — nothing to offer. + return + + if interrupted: + print() + print("◆ Session database optimization incomplete") + print( + " A previous `hermes sessions optimize-storage` run was " + "interrupted. Search still works; re-run the command to resume " + "and finish reclaiming disk:" + ) + print(" hermes sessions optimize-storage") + return + + # Concrete size framing — lead with the savings the user cares about. + est_reclaim = size_gb * 0.6 + print() + if mode == "require": + print("◆ Session database upgrade required") + print( + f" Your search index uses the OLD storage layout and should be " + f"upgraded. The new layout typically frees ~60% of state.db " + f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is " + f"required for continued optimal operation." + ) + else: + print("◆ Reclaim ~60% of your session database disk") + print( + f" Your search index uses the old storage layout. Upgrading it " + f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB " + f"of your current {size_gb:.1f} GB." + ) + print(" Run when convenient: hermes sessions optimize-storage") + print( + " It runs in the foreground with a progress bar, is safe to " + "interrupt/re-run, and never changes your conversations." + ) + +def _print_curator_recent_run_notice() -> None: + """Print the most recent curator run summary, exactly once. + + The curator runs in the background (gateway tick + CLI session start), + so users learn about skill consolidations only by stumbling into a + rename. ``hermes update`` is a high-attention surface — surface the + most recent run's rename map here, once. + + Show-once: state stamps ``last_run_summary_shown_at`` after printing. + Subsequent ``hermes update`` invocations skip the block until a newer + curator run lands. Silent when the curator has never run, when the + most recent summary has already been shown, or when the summary has + no rename information to display (no archives). + """ + try: + from agent import curator + except Exception: + return + try: + state = curator.load_state() + except Exception: + return + + last_run_at = state.get("last_run_at") + if not last_run_at: + return # no curator run yet — first-run notice handles this case + + if state.get("last_run_summary_shown_at") == last_run_at: + return # already shown for this run + + summary = state.get("last_run_summary") or "" + if not summary: + return + + # Only print when there's something interesting to show — i.e. the + # rename map block was appended (multi-line summary). A bare "auto: + # no changes; llm: no change" doesn't warrant interrupting the + # update flow. + if "\n" not in summary: + # Still stamp it shown so we don't reconsider it on every update. + try: + state["last_run_summary_shown_at"] = last_run_at + curator.save_state(state) + except Exception: + pass + return + + # Format the timestamp as "Xh ago" for readability. + when = _format_time_ago(last_run_at) + print() + print(f"ℹ Skill curator — last run {when}") + for line in summary.splitlines(): + print(f" {line}") + print( + " (This message shows once per curator run. " + "View anytime: hermes curator status)" + ) + + # Stamp shown so we don't repeat on the next update. + try: + state["last_run_summary_shown_at"] = last_run_at + curator.save_state(state) + except Exception: + pass + +def _format_time_ago(iso_ts: str) -> str: + """Render an ISO timestamp as `Xh ago` / `Xd ago` / `Xm ago`. Best effort.""" + try: + from datetime import datetime, timezone + ts = datetime.fromisoformat(iso_ts.replace("Z", "+00:00")) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + delta = datetime.now(timezone.utc) - ts + secs = int(delta.total_seconds()) + if secs < 60: + return "just now" + if secs < 3600: + return f"{secs // 60}m ago" + if secs < 86400: + return f"{secs // 3600}h ago" + return f"{secs // 86400}d ago" + except Exception: + return "recently" + +def _finish_dashboard_update_cleanup(node_failures: list[str]) -> None: + """Refresh managed dashboards or stop stale manual ones after an update.""" + if node_failures: + print() + print(" ℹ Leaving running dashboard process(es) untouched because the") + print(" Node.js dependency refresh did not complete.") + return + + stop_result = _m()._kill_stale_dashboard_processes(restart_managed=True) + if not stop_result.get("unrecovered"): + return + + print() + print( + "⚠ A web dashboard/serve process was stopped during update and could " + "not be auto-restarted." + ) + print(" Re-launch it when you want the web UI back:") + print(" hermes dashboard --port ") + +def _atomic_replace_dir(src: str, dst: str) -> None: + """Replace directory *dst* with *src* without leaving *dst* half-deleted. + + The naive ``rmtree(dst); copytree(src, dst)`` has a destructive window: if + the copy fails partway (common on the Windows ZIP-update path, which only + runs because file I/O is already flaky on that machine), the old directory + is already gone and nothing replaced it — the install is left with a + deleted tree (issue #49145, where ``ui-tui/`` vanished and broke the TUI). + + Instead, stage the new copy into a sibling temp dir first; only once that + fully succeeds do we swap it in. A failure during staging raises with the + original *dst* still intact. + """ + staging = f"{dst}.hermes-update-staging" + backup = f"{dst}.hermes-update-old" + # Clear any leftovers from a previously-interrupted update. + for leftover in (staging, backup): + if os.path.exists(leftover): + shutil.rmtree(leftover, ignore_errors=True) + + # 1. Stage the new copy. If this fails, dst is untouched. + shutil.copytree(src, staging) + # 2. Swap: move the live dir aside, move staging into place. Both moves are + # same-filesystem renames; if the second fails we restore the backup. + if os.path.exists(dst): + os.rename(dst, backup) + try: + os.rename(staging, dst) + except OSError: + if os.path.exists(backup) and not os.path.exists(dst): + os.rename(backup, dst) # roll back to the original + raise + # 3. New dir is in place; drop the old one (best-effort — never fatal). + if os.path.exists(backup): + shutil.rmtree(backup, ignore_errors=True) + +def _update_via_zip(args): + """Update Hermes Agent by downloading a ZIP archive. + + Used on Windows when git file I/O is broken (antivirus, NTFS filter + drivers causing 'Invalid argument' errors on file creation). + """ + import tempfile + import zipfile + from urllib.request import urlretrieve + + # The ZIP fallback exists for Windows git-file-I/O breakage. It pulls a + # static archive from GitHub, which is fine for the default "main" + # channel but would silently ignore --branch and update from main even + # if the user asked for something else — exactly the silent-divergence + # bug --branch was added to prevent. Refuse to proceed in that case + # rather than lie. + branch = _m()._resolve_update_branch(args) + if branch != "main": + print( + f"✗ --branch={branch} is not supported on the Windows ZIP-fallback " + "update path." + ) + print( + " This path runs when git file I/O is broken on the system. " + "Either resolve the git-side breakage (typically an antivirus " + "or NTFS filter holding files open) and rerun `hermes update " + f"--branch {branch}`, or update against main with `hermes update`." + ) + _m().sys.exit(1) + zip_url = ( + f"https://github.com/NousResearch/hermes-agent/archive/refs/heads/{branch}.zip" + ) + + print("→ Downloading latest version...") + tmp_dir = tempfile.mkdtemp(prefix="hermes-update-") + try: + zip_path = os.path.join(tmp_dir, f"hermes-agent-{branch}.zip") + urlretrieve(zip_url, zip_path) + + print("→ Extracting...") + import stat as _stat + with zipfile.ZipFile(zip_path, "r") as zf: + # Validate paths to prevent zip-slip (path traversal) AND reject + # symlink members. A GitHub source ZIP for hermes-agent itself + # should never contain symlinks — they'd point outside the + # extracted tree and let an attacker who can compromise the + # update mirror plant arbitrary files via the update path. + tmp_dir_real = os.path.realpath(tmp_dir) + for member in zf.infolist(): + member_path = os.path.realpath(os.path.join(tmp_dir, member.filename)) + if ( + not member_path.startswith(tmp_dir_real + os.sep) + and member_path != tmp_dir_real + ): + raise ValueError( + f"Zip-slip detected: {member.filename} escapes extraction directory" + ) + # Unix mode lives in the upper 16 bits of external_attr; + # mask to the file-type bits. + mode = (member.external_attr >> 16) & 0o170000 + if _stat.S_ISLNK(mode): + raise ValueError( + f"ZIP contains unsupported symlink member: {member.filename}" + ) + zf.extractall(tmp_dir) + + # GitHub ZIPs extract to hermes-agent-/ + extracted = os.path.join(tmp_dir, f"hermes-agent-{branch}") + if not os.path.isdir(extracted): + # Try to find it + for d in os.listdir(tmp_dir): + candidate = os.path.join(tmp_dir, d) + if os.path.isdir(candidate) and d != "__MACOSX": + extracted = candidate + break + + # Copy updated files over existing installation, preserving venv/node_modules/.git + preserve = {"venv", "node_modules", ".git", ".env"} + update_count = 0 + for item in os.listdir(extracted): + if item in preserve: + continue + src = os.path.join(extracted, item) + dst = os.path.join(str(_m().PROJECT_ROOT), item) + if os.path.isdir(src): + # Atomic-ish replace: never leave dst half-deleted if the copy + # fails partway (the failure mode behind #49145 on Windows). + _atomic_replace_dir(src, dst) + else: + shutil.copy2(src, dst) + update_count += 1 + + print(f"✓ Updated {update_count} items from ZIP") + + except Exception as e: + print(f"✗ ZIP update failed: {e}") + _m().sys.exit(1) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + # Clear stale bytecode after ZIP extraction + removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) + if removed: + print( + f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" + ) + _m()._record_bytecode_fingerprint() + + # Reinstall Python dependencies. Prefer .[all], but if one optional extra + # breaks on this machine, keep base deps and reinstall the remaining extras + # individually so update does not silently strip working capabilities. + print("→ Updating Python dependencies...") + + from hermes_cli.managed_uv import ensure_uv, update_managed_uv + + # Keep managed uv current — runs `uv self update` if we already have one. + update_managed_uv() + + uv_bin = ensure_uv() + + pip_cmd = [_m().sys.executable, "-m", "pip"] + if not uv_bin: + uv_bin = _ensure_uv_for_termux(pip_cmd) + if uv_bin: + uv_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")} + if _m()._is_termux_env(uv_env): + uv_env.pop("PYTHONPATH", None) + uv_env.pop("PYTHONHOME", None) + _m()._install_python_dependencies_with_optional_fallback([uv_bin, "pip"], env=uv_env) + else: + # Use sys.executable to explicitly call the venv's pip module, + # avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu. + # Some environments lose pip inside the venv; bootstrap it back with + # ensurepip before trying the editable install. + try: + subprocess.run( + pip_cmd + ["--version"], + cwd=_m().PROJECT_ROOT, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError: + subprocess.run( + [_m().sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], + cwd=_m().PROJECT_ROOT, + check=True, + ) + _m()._install_python_dependencies_with_optional_fallback(pip_cmd) + + # ZIP path parity: heal the active memory provider's bridge packages + # after the dependency reinstall, same as the git-pull path (#53272, + # #70636). + _m()._refresh_active_memory_provider_dependencies() + + node_failures = _update_node_dependencies() + _m()._build_web_ui(_m().PROJECT_ROOT / "web") + + # Sync skills + try: + from tools.skills_sync import sync_skills + + print("→ Syncing bundled skills...") + result = sync_skills(quiet=True) + if result["copied"]: + print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}") + if result.get("updated"): + print( + f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}" + ) + if result.get("user_modified"): + print(f" ~ {len(result['user_modified'])} user-modified (kept)") + print( + " → see them: hermes skills list-modified " + "(diff/reset to resume updates)" + ) + if result.get("cleaned"): + print(f" − {len(result['cleaned'])} removed from manifest") + if result.get("relocated"): + print( + f" → {len(result['relocated'])} moved to new upstream paths: " + f"{', '.join(result['relocated'])}" + ) + if not result["copied"] and not result.get("updated"): + print(" ✓ Skills are up to date") + except Exception: + pass + + # Seed the model-catalog disk cache from the freshly-unpacked checkout + # (same rationale as the git-pull path in _cmd_update_impl). Non-fatal. + try: + from hermes_cli.model_catalog import seed_cache_from_checkout + + if seed_cache_from_checkout(_m().PROJECT_ROOT): + print(" ✓ Model catalog cache refreshed from checkout") + except Exception as e: + logger.debug("Model catalog seed during zip update failed: %s", e) + + # ── Post-update state.db integrity guard (#68474) ───────────────── + # Same as the git-pull path: verify state.db survived the ZIP update + # and auto-restore from the most recent pre-update snapshot if needed. + try: + from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity + + _state_path = get_hermes_home() / "state.db" + if _state_path.exists(): + _state_ok = verify_sqlite_integrity( + _state_path, check_header=True, run_pragma=True + ) + if not _state_ok.get("valid"): + print() + print( + "⚠ state.db is corrupted after update: " + + _state_ok.get("message", "unknown error") + ) + _snap_root = _quick_snapshot_root(get_hermes_home()) + if _snap_root.exists(): + _snap_dirs = sorted( + (d for d in _snap_root.iterdir() if d.is_dir()), + reverse=True, + ) + for _snap_dir in _snap_dirs: + _snap_state = _snap_dir / "state.db" + if _snap_state.exists(): + _snap_ok = verify_sqlite_integrity( + _snap_state, check_header=True, run_pragma=True + ) + if _snap_ok.get("valid"): + try: + import shutil as _shutil + + _shutil.copy2(_snap_state, _state_path) + _restored_ok = verify_sqlite_integrity( + _state_path, + check_header=True, + run_pragma=True, + ) + if _restored_ok.get("valid"): + print( + " ✓ Auto-restored from snapshot " + f"{_snap_dir.name}" + ) + else: + print( + " ✗ Auto-restore FAILED — restored " + "copy also failed integrity" + ) + break + except OSError as _exc: + print( + f" ✗ Auto-restore file copy failed: {_exc}" + ) + break + except Exception as exc: + logger.debug( + "Post-update state.db integrity check (zip path) failed: %s", exc + ) + + print() + if node_failures: + print( + "⚠ Update partially complete — Node.js dependencies for " + f"{', '.join(node_failures)} did not refresh." + ) + print(" Code and Python deps are updated, but the dashboard/TUI may") + print(" be in a mixed state until the Node deps are rebuilt.") + else: + print("✓ Update complete!") + try: + _print_curator_first_run_notice() + except Exception as e: + logger.debug("Curator first-run notice failed: %s", e) + try: + _print_curator_recent_run_notice() + except Exception as e: + logger.debug("Curator recent-run notice failed: %s", e) + # Don't stop a working dashboard when the Node refresh failed — see the + # git-update path for rationale (#30271). + _finish_dashboard_update_cleanup(node_failures) + +def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]: + status = subprocess.run( + git_cmd + ["status", "--porcelain"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=True, + ) + if not status.stdout.strip(): + return None + + # If the index has unmerged entries (e.g. from an interrupted merge/rebase), + # git stash will fail with "needs merge / could not write index". Clear the + # conflict state with `git reset` so the stash can proceed. Working-tree + # changes are preserved; only the index conflict markers are dropped. + unmerged = subprocess.run( + git_cmd + ["ls-files", "--unmerged"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if unmerged.stdout.strip(): + print("→ Clearing unmerged index entries from a previous conflict...") + subprocess.run(git_cmd + ["reset"], cwd=cwd, capture_output=True) + + from datetime import datetime, timezone + + stash_name = datetime.now(timezone.utc).strftime( + "hermes-update-autostash-%Y%m%d-%H%M%S" + ) + print("→ Local changes detected — stashing before update...") + prev_stash = subprocess.run( + git_cmd + ["rev-parse", "--verify", "refs/stash"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ).stdout.strip() + push = subprocess.run( + git_cmd + ["stash", "push", "--include-untracked", "-m", stash_name], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if push.stdout.strip(): + print(push.stdout.strip()) + stash_probe = subprocess.run( + git_cmd + ["rev-parse", "--verify", "refs/stash"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + stash_ref = stash_probe.stdout.strip() + stash_created = ( + stash_probe.returncode == 0 and bool(stash_ref) and stash_ref != prev_stash + ) + + if push.returncode != 0: + if stash_created: + # git stash push exits non-zero when it saved everything but could + # not delete some swept untracked files from the working tree + # (e.g. a root-owned directory: "warning: failed to remove ...: + # Permission denied"). The stash entry is complete — the changes + # are safe — so this is not a failure. Leave the undeletable + # files in place and continue the update. + if push.stderr.strip(): + print(push.stderr.strip()) + print( + " ⚠ Some untracked files could not be removed from the " + "working tree (permission denied)." + ) + print( + " They were still saved to the stash and were left in " + "place — the update will continue." + ) + # A partially-failed stash push also aborts its working-tree + # cleanup for TRACKED modifications — they are saved in the stash + # but still dirty the tree, which would break the checkout/pull + # that follows. Safe to reset: everything is in the stash entry. + subprocess.run( + git_cmd + ["reset", "--hard", "HEAD"], + cwd=cwd, + capture_output=True, + ) + else: + # No stash entry was created: the changes were NOT saved. This + # is a real failure — bail out before the update touches HEAD. + print("✗ Could not stash local changes — update aborted.") + if push.stderr.strip(): + print(f" {push.stderr.strip().splitlines()[0]}") + print( + " Commit, stash, or clean up your local changes manually, " + "then re-run `hermes update`." + ) + raise subprocess.CalledProcessError( + push.returncode, push.args, output=push.stdout, stderr=push.stderr + ) + + return stash_ref + +def _resolve_stash_selector( + git_cmd: list[str], cwd: Path, stash_ref: str +) -> Optional[str]: + stash_list = subprocess.run( + git_cmd + ["stash", "list", "--format=%gd %H"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=True, + ) + for line in stash_list.stdout.splitlines(): + selector, _, commit = line.partition(" ") + if commit.strip() == stash_ref: + return selector.strip() + return None + +def _print_stash_cleanup_guidance( + stash_ref: str, stash_selector: Optional[str] = None +) -> None: + print( + " Check `git status` first so you don't accidentally reapply the same change twice." + ) + print(" Find the saved entry with: git stash list --format='%gd %H %s'") + if stash_selector: + print(f" Remove it with: git stash drop {stash_selector}") + else: + print( + f" Look for commit {stash_ref}, then drop its selector with: git stash drop stash@{{N}}" + ) + +def _stash_apply_failed_only_on_existing_untracked(stderr: str) -> bool: + """True when a ``git stash apply`` failure is ONLY about untracked files + that already exist in the working tree. + + This is the tail end of the permission-denied autostash class: ``git stash + push --include-untracked`` swept undeletable files (e.g. a root-owned + ``packaging/`` directory) into the stash but could not remove them from + disk. On restore, git applies all tracked changes, then refuses to + overwrite those still-present files (``already exists, no checkout`` / + ``could not restore untracked files from stash``) and exits non-zero even + though nothing was lost. Any other error line (e.g. ``would be + overwritten by merge`` / ``Aborting``) means the tracked apply itself + failed and this returns False. + """ + lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()] + if not lines: + return False + saw_untracked_error = False + for ln in lines: + if "already exists, no checkout" in ln: + saw_untracked_error = True + elif "could not restore untracked files from stash" in ln: + saw_untracked_error = True + elif ln.startswith(("warning:", "hint:")): + continue + else: + return False + return saw_untracked_error + +def _restore_stashed_changes( + git_cmd: list[str], + cwd: Path, + stash_ref: str, + prompt_user: bool = False, + input_fn=None, +) -> bool: + if prompt_user: + print() + print("⚠ Local changes were stashed before updating.") + print( + " Restoring them may reapply local customizations onto the updated codebase." + ) + print(" Review the result afterward if Hermes behaves unexpectedly.") + print("Restore local changes now? [Y/n]") + if input_fn is not None: + response = input_fn("Restore local changes now? [Y/n]", "y") + else: + response = input().strip().lower() + if response not in {"", "y", "yes"}: + print("Skipped restoring local changes.") + print("Your changes are still preserved in git stash.") + print(f"Restore manually with: git stash apply {stash_ref}") + return False + + print("→ Restoring local changes...") + restore = subprocess.run( + git_cmd + ["stash", "apply", stash_ref], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + + # Check for unmerged (conflicted) files — can happen even when returncode is 0 + unmerged = subprocess.run( + git_cmd + ["diff", "--name-only", "--diff-filter=U"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + has_conflicts = bool(unmerged.stdout.strip()) + + if restore.returncode != 0 and not has_conflicts and ( + _stash_apply_failed_only_on_existing_untracked(restore.stderr) + ): + # Permission-denied autostash tail end: the tracked changes applied + # cleanly; the only "failure" is untracked files that never left the + # working tree (git could not delete them at stash time, so it now + # refuses to overwrite them). Their content was never touched — + # nothing is lost. Treat as restored. + print( + " ⚠ Some stashed untracked files already exist in the working " + "tree and were kept as-is." + ) + elif restore.returncode != 0 or has_conflicts: + print("✗ Update pulled new code, but restoring local changes hit conflicts.") + if restore.stdout.strip(): + print(restore.stdout.strip()) + if restore.stderr.strip(): + print(restore.stderr.strip()) + + # Show which files conflicted + conflicted_files = unmerged.stdout.strip() + if conflicted_files: + print("\nConflicted files:") + for f in conflicted_files.splitlines(): + print(f" • {f}") + + print("\nYour stashed changes are preserved — nothing is lost.") + print(f" Stash ref: {stash_ref}") + + # Always reset to clean state — leaving conflict markers in source + # files makes hermes completely unrunnable (SyntaxError on import). + # The user's changes are safe in the stash for manual recovery. + subprocess.run( + git_cmd + ["reset", "--hard", "HEAD"], + cwd=cwd, + capture_output=True, + ) + print("Working tree reset to clean state.") + print(f"Restore your changes later with: git stash apply {stash_ref}") + # Don't sys.exit — the code update itself succeeded, only the stash + # restore had conflicts. Let cmd_update continue with pip install, + # skill sync, and gateway restart. + return False + + stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) + if stash_selector is None: + print( + "⚠ Local changes were restored, but Hermes couldn't find the stash entry to drop." + ) + print( + " The stash was left in place. You can remove it manually after checking the result." + ) + _print_stash_cleanup_guidance(stash_ref) + else: + drop = subprocess.run( + git_cmd + ["stash", "drop", stash_selector], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if drop.returncode != 0: + print( + "⚠ Local changes were restored, but Hermes couldn't drop the saved stash entry." + ) + if drop.stdout.strip(): + print(drop.stdout.strip()) + if drop.stderr.strip(): + print(drop.stderr.strip()) + print( + " The stash was left in place. You can remove it manually after checking the result." + ) + _print_stash_cleanup_guidance(stash_ref, stash_selector) + + print("⚠ Local changes were restored on top of the updated codebase.") + print(" Review `git diff` / `git status` if Hermes behaves unexpectedly.") + return True + +def _discard_stashed_changes( + git_cmd: list[str], + cwd: Path, + stash_ref: str, +) -> bool: + """Throw away a stash created before an update, without applying it. + + Used only on a NON-interactive update when the user has set + ``updates.non_interactive_local_changes: discard`` — i.e. they've opted out + of keeping local source edits on this machine. Drops the stash entry + instead of re-applying it, so the working tree stays clean at the freshly + pulled HEAD. Unlike ``git reset --hard`` + ``git clean -fd``, this only + affects what was stashed (tracked changes + the untracked files we + explicitly captured) — ignored paths like node_modules/venv/build outputs + are never touched, since they were never stashed. + + Returns True if the stash was dropped, False on a git failure (in which + case the stash is left in place for safety). + """ + stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) + if stash_selector is None: + print( + "⚠ Configured to discard local changes on non-interactive update, " + "but Hermes couldn't find the stash entry to drop." + ) + _print_stash_cleanup_guidance(stash_ref) + return False + + drop = subprocess.run( + git_cmd + ["stash", "drop", stash_selector], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if drop.returncode != 0: + print( + "⚠ Configured to discard local changes, but Hermes couldn't drop " + "the saved stash entry." + ) + if drop.stderr.strip(): + print(f" {drop.stderr.strip().splitlines()[0]}") + _print_stash_cleanup_guidance(stash_ref, stash_selector) + return False + + print("→ Discarded local source changes (updates.non_interactive_local_changes=discard).") + return True + +OFFICIAL_REPO_URLS = { + "https://github.com/NousResearch/hermes-agent.git", + "git@github.com:NousResearch/hermes-agent.git", + "https://github.com/NousResearch/hermes-agent", + "git@github.com:NousResearch/hermes-agent", +} + +OFFICIAL_REPO_URL = "https://github.com/NousResearch/hermes-agent.git" + +SKIP_UPSTREAM_PROMPT_FILE = ".skip_upstream_prompt" + +def _get_origin_url(git_cmd: list[str], cwd: Path) -> Optional[str]: + """Get the URL of the origin remote, or None if not set.""" + try: + result = subprocess.run( + git_cmd + ["remote", "get-url", "origin"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if result.returncode == 0: + return result.stdout.strip() + except Exception: + pass + return None + +def _is_fork(origin_url: Optional[str]) -> bool: + """Check if the origin remote points to a fork (not the official repo).""" + if not origin_url: + return False + # Normalize URL for comparison (strip trailing .git if present) + normalized = origin_url.rstrip("/") + if normalized.endswith(".git"): + normalized = normalized[:-4] + for official in OFFICIAL_REPO_URLS: + official_normalized = official.rstrip("/") + if official_normalized.endswith(".git"): + official_normalized = official_normalized[:-4] + if normalized == official_normalized: + return False + return True + +def _has_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: + """Check if an 'upstream' remote already exists.""" + try: + result = subprocess.run( + git_cmd + ["remote", "get-url", "upstream"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + return result.returncode == 0 + except Exception: + return False + +def _add_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: + """Add the official repo as the 'upstream' remote. Returns True on success.""" + try: + result = subprocess.run( + git_cmd + ["remote", "add", "upstream", OFFICIAL_REPO_URL], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + return result.returncode == 0 + except Exception: + return False + +def _count_commits_between(git_cmd: list[str], cwd: Path, base: str, head: str) -> int: + """Count commits on `head` that are not on `base`. Returns -1 on error.""" + try: + result = subprocess.run( + git_cmd + ["rev-list", "--count", f"{base}..{head}"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if result.returncode == 0: + return int(result.stdout.strip()) + except Exception: + pass + return -1 + +def _should_skip_upstream_prompt() -> bool: + """Check if user previously declined to add upstream.""" + from hermes_constants import get_hermes_home + + return (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).exists() + +def _mark_skip_upstream_prompt(): + """Create marker file to skip future upstream prompts.""" + try: + from hermes_constants import get_hermes_home + + (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).touch() + except Exception: + pass + +def _sync_fork_with_upstream(git_cmd: list[str], cwd: Path) -> bool: + """Attempt to push updated main to origin (sync fork). + + Returns True if push succeeded, False otherwise. + """ + try: + result = subprocess.run( + git_cmd + ["push", "origin", "main", "--force-with-lease"], + cwd=cwd, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + return result.returncode == 0 + except Exception: + return False + +def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path) -> None: + """Check if fork is behind upstream and sync if safe. + + This implements the fork upstream sync logic: + - If upstream remote doesn't exist, ask user if they want to add it + - Compare origin/main with upstream/main + - If origin/main is strictly behind upstream/main, pull from upstream + - Try to sync fork back to origin if possible + """ + has_upstream = _has_upstream_remote(git_cmd, cwd) + + if not has_upstream: + # Check if user previously declined + if _should_skip_upstream_prompt(): + return + + # Ask user if they want to add upstream + print() + print("ℹ Your fork is not tracking the official Hermes repository.") + print(" This means you may miss updates from NousResearch/hermes-agent.") + print() + try: + response = ( + input("Add official repo as 'upstream' remote? [Y/n]: ").strip().lower() + ) + except (EOFError, KeyboardInterrupt): + print() + response = "n" + + if response in {"", "y", "yes"}: + print("→ Adding upstream remote...") + if _add_upstream_remote(git_cmd, cwd): + print( + " ✓ Added upstream: https://github.com/NousResearch/hermes-agent.git" + ) + has_upstream = True + else: + print(" ✗ Failed to add upstream remote. Skipping upstream sync.") + return + else: + print( + " Skipped. Run 'git remote add upstream https://github.com/NousResearch/hermes-agent.git' to add later." + ) + _mark_skip_upstream_prompt() + return + + # Fetch upstream main only. This sync compares upstream/main with + # origin/main, so there's no reason to pull every upstream ref — and a bare + # fetch drags in thousands of auto-generated branches. + print() + print("→ Fetching upstream...") + try: + subprocess.run( + git_cmd + ["fetch", "upstream", "main", "--quiet"], + cwd=cwd, + capture_output=True, + check=True, + ) + except subprocess.CalledProcessError: + print(" ✗ Failed to fetch upstream. Skipping upstream sync.") + return + + # Compare origin/main with upstream/main + origin_ahead = _count_commits_between(git_cmd, cwd, "upstream/main", "origin/main") + upstream_ahead = _count_commits_between( + git_cmd, cwd, "origin/main", "upstream/main" + ) + + if origin_ahead < 0 or upstream_ahead < 0: + print(" ✗ Could not compare branches. Skipping upstream sync.") + return + + # If origin/main has commits not on upstream, don't trample + if origin_ahead > 0: + print() + print(f"ℹ Your fork has {origin_ahead} commit(s) not on upstream.") + print(" Skipping upstream sync to preserve your changes.") + print(" If you want to merge upstream changes, run:") + print(" git pull upstream main") + return + + # If upstream is not ahead, fork is up to date + if upstream_ahead == 0: + print(" ✓ Fork is up to date with upstream") + return + + # origin/main is strictly behind upstream/main (can fast-forward) + print() + print(f"→ Fork is {upstream_ahead} commit(s) behind upstream") + print("→ Pulling from upstream...") + + try: + subprocess.run( + git_cmd + ["pull", "--ff-only", "upstream", "main"], + cwd=cwd, + check=True, + ) + except subprocess.CalledProcessError: + print( + " ✗ Failed to pull from upstream. You may need to resolve conflicts manually." + ) + return + + print(" ✓ Updated from upstream") + + # Try to sync fork back to origin + print("→ Syncing fork...") + if _sync_fork_with_upstream(git_cmd, cwd): + print(" ✓ Fork synced with upstream") + else: + print( + " ℹ Got updates from upstream but couldn't push to fork (no write access?)" + ) + print(" Your local repo is updated, but your fork on GitHub may be behind.") + +def _invalidate_update_cache(): + """Delete the update-check cache for ALL profiles so no banner + reports a stale "commits behind" count after a successful update. + + The git repo is shared across profiles — when one profile runs + ``hermes update``, every profile is now current. + """ + homes = [] + # Default profile home (Docker-aware — uses /opt/data in Docker) + from hermes_constants import get_default_hermes_root + + default_home = get_default_hermes_root() + homes.append(default_home) + # Named profiles under /profiles/ + profiles_root = default_home / "profiles" + if profiles_root.is_dir(): + for entry in profiles_root.iterdir(): + if entry.is_dir(): + homes.append(entry) + for home in homes: + try: + cache_file = home / ".update_check" + if cache_file.exists(): + cache_file.unlink() + except Exception: + pass + +def _write_marker_file(path: Path, *, label: str) -> None: + """Drop an update-recovery breadcrumb. Never raises.""" + try: + path.write_text( + f"started={_time.time()}\npid={os.getpid()}\n", encoding="utf-8" + ) + except OSError as exc: + logger.debug("Could not write %s marker: %s", label, exc) + +def _write_update_incomplete_marker() -> None: + """Drop the interrupted core-install breadcrumb. Never raises.""" + _write_marker_file(_m()._update_marker_path(), label="update-incomplete") + +def _write_lazy_refresh_incomplete_marker() -> None: + """Drop the interrupted lazy-refresh breadcrumb. Never raises.""" + _write_marker_file(_m()._lazy_refresh_marker_path(), label="lazy-refresh-incomplete") + +def _format_concurrent_instances_message( + matches: list[tuple[int, str]], scripts_dir: Path +) -> str: + """Build a human-readable explanation + remediation hint for the user.""" + shim = scripts_dir / "hermes.exe" + lines = ["✗ Another hermes.exe is running:"] + for pid, name in matches: + lines.append(f" PID {pid} {name}") + lines.append("") + lines.append(f" Updating now would fail to overwrite {shim} because") + lines.append(" Windows blocks REPLACE on a running executable.") + lines.append("") + lines.append(" Close Hermes Desktop, exit any open `hermes` REPLs, and") + lines.append(" stop the gateway (`hermes gateway stop`) before retrying.") + lines.append("") + if matches: + pid_args = " ".join(f"/PID {pid}" for pid, _ in matches) + lines.append(" If you've already closed everything and these PIDs are") + lines.append(" stale, terminate them directly, then retry the update:") + lines.append(f" taskkill {pid_args} /F") + lines.append("") + lines.append(" Override with `hermes update --force` if you've already") + lines.append(" confirmed those processes will not write to the venv.") + return "\n".join(lines) + +def _upgrade_pip_before_lazy_refresh( + install_cmd_prefix: list[str], + *, + env: dict[str, str] | None = None, +) -> None: + """Upgrade pip before lazy-backend refreshes. + + Older pip (e.g. 24.0 on Python 3.11) can fail setuptools-backed source + builds during lazy installs and leave a partially-written venv (#57828). + Never raises. + """ + try: + _m()._run_package_only_install( + install_cmd_prefix + ["install", "--upgrade", "pip"], + env=env, + ) + except subprocess.CalledProcessError as exc: + logger.debug("pip upgrade before lazy refresh failed: %s", exc) + +def _refresh_active_lazy_features( + install_cmd_prefix: list[str] | None = None, + *, + env: dict[str, str] | None = None, +) -> bool: + """Refresh lazy-installed backends after a code update. + + When pyproject.toml's ``[all]`` extra was slimmed down (May 2026), most + optional backends moved to ``tools/lazy_deps.py`` and only install on + first use. ``hermes update`` runs ``uv pip install -e .[all]`` which + leaves those packages untouched — so if we bump a pin in + :data:`LAZY_DEPS` (CVE response, transitive bug fix), users who already + activated the backend keep the stale version forever. + + This function asks lazy_deps which features the user has previously + activated and reinstalls them under the current pins. Features the + user never enabled stay quiet — no churn for cold backends. + + Returns True when the venv is safe to use (refresh succeeded, or no + active lazy backends, or post-failure import repair succeeded). Returns + False when a failed lazy install left broken core imports that automatic + repair could not fix (#57828). + + Never raises. A failure here must not block the rest of the update. + """ + try: + from tools import lazy_deps + except Exception as exc: + logger.debug("Lazy refresh skipped (import failed): %s", exc) + return True + + try: + active = lazy_deps.active_features() + except Exception as exc: + logger.debug("Lazy refresh skipped (active_features failed): %s", exc) + return True + + if not active: + return True + + print() + print(f"→ Refreshing {len(active)} active lazy backend(s)...") + + unexpected_failure = False + try: + results = lazy_deps.refresh_active_features(prompt=False) + except Exception as exc: + # refresh_active_features is documented as never-raise, but defend + # the update flow against future regressions. + print(f" ⚠ Lazy refresh failed unexpectedly: {exc}") + results = {} + unexpected_failure = True + + refreshed = [f for f, s in results.items() if s == "refreshed"] + current = [f for f, s in results.items() if s == "current"] + failed = [(f, s) for f, s in results.items() if s.startswith("failed:")] + skipped = [(f, s) for f, s in results.items() if s.startswith("skipped:")] + + if refreshed: + print(f" ↑ {len(refreshed)} refreshed: {', '.join(refreshed)}") + if current: + print(f" ✓ {len(current)} already current") + if skipped: + # Most common reason: security.allow_lazy_installs=false. Show one + # line so the user knows why; not an error. + names = ", ".join(f for f, _ in skipped) + reason = skipped[0][1].split(": ", 1)[-1] + print(f" · {len(skipped)} skipped ({reason}): {names}") + + if not failed and not unexpected_failure: + return True + + for feature, status in failed: + reason = status.split(": ", 1)[-1] + # Clip noisy pip stderr to keep update output legible. + if len(reason) > 200: + reason = reason[:200] + "..." + print(f" ⚠ {feature} failed to refresh: {reason}") + + if install_cmd_prefix is None: + print(" ⚠ Lazy refresh failed; rerun `hermes update` once resolved.") + return False + + # Immediate import-based recovery — metadata-only verifiers miss the case + # where DISTRIBUTION-INFO remains but import files were wiped (#57828). + # Unavailable probes are indeterminate, not healthy — keep the lazy marker. + status = _m()._repair_venv_via_import_probes(install_cmd_prefix, env=env) + if status == "repaired": + print( + " Lazy backend(s) keep their previous version until refresh succeeds." + ) + return True + if status == "healthy": + print( + " Lazy backend(s) keep their previous version; probed packages look intact." + ) + print(" Rerun `hermes update` once the upstream issue is resolved.") + return True + if status == "indeterminate": + print( + " ⚠ Leaving `.lazy-refresh-incomplete` until import probes can confirm health." + ) + return False + +def _refresh_active_memory_provider_dependencies() -> None: + """Refresh pip dependencies for the configured external memory provider. + + Memory-provider bridge packages are declared in each provider's + ``plugin.yaml`` (plus mode-dependent extras like Hindsight's + ``hindsight-all``), NOT in Hermes' editable-install extras or + ``LAZY_DEPS`` alone — so the core dependency reinstall above can strip + or downgrade them (#53272 mem0ai, #70636 hindsight-embed). Re-run the + provider's declared install for the ACTIVE provider only, after the + core install and lazy refresh, so the last write to any shared package + is the one the active provider needs. + + Never raises. A failure here must not block the rest of the update. + """ + try: + from hermes_cli.config import load_config + + cfg = load_config() + except Exception as exc: + logger.debug("Memory provider refresh skipped (config load failed): %s", exc) + return + + provider = "" + if isinstance(cfg, dict): + memory_cfg = cfg.get("memory") + if isinstance(memory_cfg, dict): + if memory_cfg.get("enabled") is False: + return + provider = str(memory_cfg.get("provider") or "").strip() + + # "default" / empty is the built-in file-backed store — no pip deps. + if not provider or provider in {"default", "builtin", "none"}: + return + + try: + from hermes_cli.memory_setup import _install_dependencies + except Exception as exc: + logger.debug("Memory provider refresh skipped (import failed): %s", exc) + return + + print() + print(f"→ Refreshing active memory provider dependencies ({provider})...") + + try: + _install_dependencies(provider, force=True) + except Exception as exc: + print(f" ⚠ {provider} dependencies failed to refresh: {exc}") + +def _is_android_python() -> bool: + return _m().sys.platform == "android" + +def _install_psutil_android_compat( + install_cmd_prefix: list[str], + *, + env: dict[str, str] | None = None, +) -> None: + """Install psutil on Android by patching upstream platform detection. + + psutil's setup currently gates Linux sources behind + ``sys.platform.startswith('linux')``. On Termux Python reports + ``sys.platform == 'android'``, so setup aborts with + "platform android is not supported" despite compiling fine when using the + Linux source path. + + We patch only the extracted build tree used for this install attempt; + nothing is persisted in the repository. + + Stopgap: remove this once https://github.com/giampaolo/psutil/pull/2762 + merges and ships in a release. The standalone installer script uses the + same shared helper and should be removed together. + """ + import tempfile + import urllib.request + from hermes_cli.psutil_android import PSUTIL_URL, prepare_patched_psutil_sdist + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + archive = tmp_path / "psutil.tar.gz" + urllib.request.urlretrieve(PSUTIL_URL, archive) + src_root = prepare_patched_psutil_sdist(archive, tmp_path) + + _m()._run_install_with_heartbeat( + install_cmd_prefix + ["install", "--no-build-isolation", str(src_root)], + env=env, + ) + +def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None: + """Best-effort uv bootstrap on Termux for faster update installs. + + The normal path (``ensure_uv()`` in managed_uv) installs the managed + standalone uv into ``$HERMES_HOME/bin/uv``, but on Termux the official + installer may not work (glibc vs bionic). Prefer a uv already on PATH + (e.g. ``pkg install uv``); only if there is none do we fall back to a + wheel-only ``pip install uv`` so we never source-build the Rust crate. + """ + from hermes_cli.managed_uv import resolve_uv + + existing = resolve_uv() + if existing: + return existing + if not _m()._is_termux_env(): + return None + # A Termux-packaged uv lands on PATH but not in the managed bin dir, so + # resolve_uv() misses it. Use it before pip, which has no Android wheel and + # would otherwise build uv from source on a low-memory device. + system_uv = shutil.which("uv") + if system_uv: + return system_uv + try: + print(" → Termux detected: trying to install uv for faster dependency updates...") + result = subprocess.run( + pip_cmd + ["install", "uv", "--only-binary", ":all:"], + cwd=_m().PROJECT_ROOT, + check=False, + ) + if result.returncode != 0: + return None + except Exception: + pass + # After pip install, check managed path first, then PATH + return resolve_uv() or shutil.which("uv") + +def _npm_manifest_paths() -> tuple[Path, ...]: + """Manifests whose changes must defeat the update-skip. + + The lockfile alone is NOT a sufficient key: on a local checkout a dev + can edit package.json (root or a workspace) without running npm — the + lockfile is then unchanged but `hermes update` is exactly the step + expected to sync node_modules (via the `npm install` fallback in + _run_npm_install_deterministic). + + The workspace list is pulled from the root package.json's `workspaces` + globs (npm's own source of truth) rather than hardcoded, so adding a + workspace can never silently escape the skip key. The root install + (step 1, --workspaces=false) still hoists shared deps for EVERY + workspace — desktop included — so all of them belong in the key, not + just the ones step 2 installs. Falls back to hashing just root + manifests if package.json is unreadable (never skips more than main + would have installed). + """ + root_pkg = _m().PROJECT_ROOT / "package.json" + paths = [_m().PROJECT_ROOT / "package-lock.json", root_pkg] + try: + workspaces = json.loads(root_pkg.read_text(encoding="utf-8")).get( + "workspaces", [] + ) + if isinstance(workspaces, dict): # legacy {"packages": [...]} form + workspaces = workspaces.get("packages", []) + for pattern in workspaces: + for match in sorted(_m().PROJECT_ROOT.glob(str(pattern))): + manifest = match / "package.json" + if manifest.is_file(): + paths.append(manifest) + except (OSError, json.JSONDecodeError, TypeError): + pass + return tuple(paths) + +def _npm_manifests_digest() -> str | None: + """Combined sha256 over the lockfile + all workspace package.json files. + + Returns None when the lockfile is missing (never skip then). + """ + if not (_m().PROJECT_ROOT / "package-lock.json").exists(): + return None + h = hashlib.sha256() + for p in _npm_manifest_paths(): + h.update(str(p.relative_to(_m().PROJECT_ROOT)).encode()) + try: + h.update(p.read_bytes()) + except OSError: + h.update(b"") + return h.hexdigest() + +def _npm_lockfile_changed(hermes_root: Path) -> bool: + current = _npm_manifests_digest() + if current is None: + return True + # Also check that node_modules exists; a matching hash with missing + # node_modules means the cache was recorded by another checkout. + if not (_m().PROJECT_ROOT / "node_modules").is_dir(): + return True + # A matching lockfile hash over a tree whose web build toolchain never + # landed must NOT skip the reinstall — otherwise every later `hermes + # update` keeps rebuilding against a half-installed tree and serving a + # stale dist. + web_dir = _m().PROJECT_ROOT / "web" + if (web_dir / "package.json").is_file() and not _web_build_toolchain_ready( + *_web_toolchain_roots(web_dir) + ): + return True + try: + # Key the cache by PROJECT_ROOT so parallel worktrees don't collide. + cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12] + cache_file = hermes_root / f".npm_lock_hash_{cache_key}" + if not cache_file.exists(): + return True + return cache_file.read_text(encoding="utf-8").strip() != current + except OSError: + return True + +def _record_npm_lockfile_hash(hermes_root: Path) -> None: + digest = _npm_manifests_digest() + if digest is None: + return + try: + cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12] + cache_file = hermes_root / f".npm_lock_hash_{cache_key}" + cache_file.write_text(digest, encoding="utf-8") + except OSError: + logger.debug("Could not write npm lockfile hash cache") + +def _update_node_dependencies() -> list[str]: + """Refresh Node deps in the repo root and update workspaces. + + Returns the list of labels whose npm install failed (empty on success), + so the caller can treat a Node refresh failure as a partial update rather + than silently reporting ``Update complete!`` (#30271). + """ + if not (_m().PROJECT_ROOT / "package.json").exists(): + return [] + + npm = _m()._resolve_node_runtime_npm() + if not npm: + # If the only npm reachable inside this WSL shell is the Windows one, + # flag it loudly: silently skipping leaves ui-tui deps stale while the + # rest of the update proceeds, and running it would corrupt the tree. + from hermes_constants import is_wsl + + path_npm = shutil.which("npm") + if is_wsl() and path_npm and _m()._is_windows_npm_path(path_npm): + print("→ Updating Node.js dependencies...") + print(" ⚠ Skipped: only a Windows npm is reachable from this WSL shell.") + print(" Install Node.js inside the WSL distro (nvm, or your distro's") + print(" package manager), then re-run `hermes update`.") + failed = ["repo root"] + if any( + (_m().PROJECT_ROOT / workspace / "package.json").exists() + for workspace in ("ui-tui", "web") + ): + failed.append("ui-tui, web workspaces") + return failed + return [] + + from hermes_constants import get_default_hermes_root + + # This cache describes PROJECT_ROOT/node_modules, which is shared by every + # Hermes profile using this checkout. Keep one per-checkout cache under the + # shared Hermes root rather than rerunning npm once per named profile. + shared_hermes_root = get_default_hermes_root() + if not _m()._npm_lockfile_changed(shared_hermes_root): + logger.info("npm lockfile unchanged, skipping npm install") + return [] + + # With a single workspace lockfile the root install would cover ALL + # workspaces — but apps/desktop pulls in Electron as a devDependency, + # and its postinstall downloads a ~200MB binary. Most users don't + # need desktop during `hermes update`, so we install root-only first + # then add just the workspaces the CLI/TUI/web build actually requires. + # Desktop deps are installed on demand by the desktop launcher + # (see _desktop_build_needed). + print("→ Updating Node.js dependencies...") + + def _partial_update_failure(*labels: str) -> list[str]: + print() + print(" ⚠ Node.js dependency refresh did not complete cleanly; the") + print(" installation may be in a mixed state (updated code, stale Node") + print(" deps). Fix npm and re-run `hermes update`.") + return list(labels) + + extra_args = ["--no-fund", "--no-audit", "--progress=false"] + + from hermes_constants import with_hermes_node_path + + nixos_env = with_hermes_node_path(_m()._nixos_build_env()) + + # Step 1: root install (no workspace recursion). + # NOTE: capture_output=False here is deliberate (#18840) — optional + # postinstall scripts (e.g. @askjo/camofox-browser's browser-binary fetch) + # print download progress, and capturing it makes a long download look + # hung. The chatty npm-deprecation noise during `hermes update` comes from + # the *desktop* build, not this step; that one is captured to update.log. + root_args = [*extra_args, "--workspaces=false"] + root_result = _m()._run_npm_install_deterministic( + npm, + _m().PROJECT_ROOT, + extra_args=tuple(root_args), + capture_output=False, + env=nixos_env, + ) + if root_result.returncode != 0: + print(" ⚠ npm install failed in repo root") + stderr = (root_result.stderr or "").strip() if root_result.stderr else "" + if stderr: + print(f" {stderr.splitlines()[-1]}") + return _partial_update_failure("repo root") + + # Step 2: install only the workspaces update needs (ui-tui, web). + # --workspace selects specific workspaces; the rest (desktop) are skipped. + ws_args = [*extra_args, "--workspace", "ui-tui", "--workspace", "web"] + ws_result = _m()._run_npm_install_deterministic( + npm, + _m().PROJECT_ROOT, + extra_args=tuple(ws_args), + capture_output=False, + env=nixos_env, + ) + if ws_result.returncode == 0: + _record_npm_lockfile_hash(shared_hermes_root) + print(" ✓ repo root + ui-tui, web workspaces (desktop skipped)") + return [] + + print(" ⚠ npm workspace install failed") + stderr = (ws_result.stderr or "").strip() if ws_result.stderr else "" + if stderr: + print(f" {stderr.splitlines()[-1]}") + return _partial_update_failure("ui-tui, web workspaces") + +def _log_only_write(text: str) -> None: + """Write ``text`` to ``~/.hermes/logs/update.log`` only, never the terminal. + + During ``hermes update`` ``sys.stdout`` is an ``_UpdateOutputStream`` that + mirrors to both the terminal and ``update.log``. Loud, low-signal + subprocess output (npm installs, the Electron/vite build, the cua-driver + installer's "Next steps" wall) should be captured and tucked into the log + so failures stay debuggable, without flooding the user's terminal. This + reaches past the mirroring stream straight to the underlying log handle. + """ + if not text: + return + stream = _m().sys.stdout + log_file = getattr(stream, "_log", None) + if log_file is None: + return + try: + log_file.write(text if text.endswith("\n") else text + "\n") + log_file.flush() + except Exception: + pass + +def _run_logged_subprocess(cmd, *, cwd=None, env=None): + """Run ``cmd`` capturing combined output into update.log (not the terminal). + + Returns the ``CompletedProcess`` (with ``stdout`` populated) so the caller + can decide whether to surface the captured output on failure. + """ + result = subprocess.run( + cmd, + cwd=cwd, + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + ) + _log_only_write(result.stdout or "") + return result + +def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): + """Implement ``hermes update --check``: fetch and report without installing. + + ``branch`` selects which branch the check compares against. Default is + "main"; callers can pass another branch to ask "are there new commits + on origin/?" without performing the update. + + ``branch_explicit`` is True iff the caller passed --branch on the CLI. + Installs that can't honor non-default branches (e.g. Docker) surface a + one-line notice instead of silently dropping the flag. + """ + from hermes_cli.config import detect_install_method, recommended_update_command_for_method + method = detect_install_method(_m().PROJECT_ROOT) + if method == "docker": + # Docker can't ``git fetch`` from within the container. Surface the + # same long-form ``docker pull`` guidance ``hermes update`` (apply + # path) uses — telling the user to "reinstall via curl" or that + # ".git is missing" would point them at the wrong remediation. + from hermes_cli.config import format_docker_update_message + print(format_docker_update_message()) + _m().sys.exit(1) + + if method in {"nix", "nixos"}: + print(recommended_update_command_for_method(method)) + _m().sys.exit(1) + + git_dir = _m().PROJECT_ROOT / ".git" + if not git_dir.exists(): + print("✗ Not a git repository — cannot check for updates.") + _m().sys.exit(1) + + git_cmd = ["git"] + if _m().sys.platform == "win32": + git_cmd = ["git", "-c", "windows.appendAtomically=false"] + + # Fetch only the branch we compare against; prefer upstream as the canonical + # reference. A bare `git fetch ` pulls every ref, and this repo has + # thousands of auto-generated branches, so scope the fetch to . + # Note: upstream/ may not exist for non-main branches (a fork's + # bb/gui has no upstream counterpart), so when the caller picks a + # non-default branch we skip the upstream probe and use origin directly. + # Installer checkouts are shallow (`git clone --depth 1`). A plain + # `git fetch` would unshallow the repo (dragging in the whole history — + # the exact cost the shallow clone avoided) and the rev-list count below + # would then report a huge bogus "behind" number. Detect shallow up front: + # fetch with --depth 1 to preserve the boundary and report presence-only. + is_shallow = ( + subprocess.run( + git_cmd + ["rev-parse", "--is-shallow-repository"], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ).stdout.strip() + == "true" + ) + depth_args = ["--depth", "1"] if is_shallow else [] + + if branch == "main": + print("→ Fetching from upstream...") + fetch_result = subprocess.run( + git_cmd + ["fetch"] + depth_args + ["upstream", branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if fetch_result.returncode != 0: + # Fallback to origin if upstream doesn't exist + print("→ Fetching from origin...") + fetch_result = subprocess.run( + git_cmd + ["fetch"] + depth_args + ["origin", branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + upstream_exists = False + compare_branch = f"origin/{branch}" + else: + upstream_exists = True + compare_branch = f"upstream/{branch}" + else: + # Non-default branch: compare against origin/ directly. + print("→ Fetching from origin...") + fetch_result = subprocess.run( + git_cmd + ["fetch"] + depth_args + ["origin", branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + upstream_exists = False + compare_branch = f"origin/{branch}" + + if fetch_result.returncode != 0: + stderr = fetch_result.stderr.strip() + if "Could not resolve host" in stderr or "unable to access" in stderr: + print("✗ Network error — cannot reach the remote repository.") + elif "Authentication failed" in stderr or "could not read Username" in stderr: + print("✗ Authentication failed — check your git credentials or SSH key.") + else: + print("✗ Failed to fetch.") + if stderr: + print(f" {stderr.splitlines()[0]}") + _m().sys.exit(1) + + # Verify the compare ref actually exists before asking rev-list about it. + # Without this, `git rev-list HEAD..origin/ --count` exits 128 and + # (with check=True) raises CalledProcessError, surfacing a Python + # traceback. Friendlier to detect-and-report. + verify_result = subprocess.run( + git_cmd + ["rev-parse", "--verify", "--quiet", compare_branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if verify_result.returncode != 0: + print(f"✗ Branch '{branch}' not found on {compare_branch.split('/', 1)[0]}.") + _m().sys.exit(1) + + if is_shallow: + # No history to count across the shallow boundary. Compare tip SHAs and + # report presence-only (mirrors the banner's _check_via_local_git). + head_sha = subprocess.run( + git_cmd + ["rev-parse", "HEAD"], + cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", + ).stdout.strip() + target_sha = subprocess.run( + git_cmd + ["rev-parse", compare_branch], + cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", + ).stdout.strip() + if head_sha and target_sha and head_sha == target_sha: + print("✓ Already up to date.") + else: + print(f"⚕ Update available (behind {compare_branch}).") + from hermes_cli.config import recommended_update_command + + print(f" Run '{recommended_update_command()}' to install.") + return + + rev_result = subprocess.run( + git_cmd + ["rev-list", f"HEAD..{compare_branch}", "--count"], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=True, + ) + behind = int(rev_result.stdout.strip()) + + if behind == 0: + print("✓ Already up to date.") + else: + commits_word = "commit" if behind == 1 else "commits" + print(f"⚕ Update available: {behind} {commits_word} behind {compare_branch}.") + from hermes_cli.config import recommended_update_command + + print(f" Run '{recommended_update_command()}' to install.") + +def _ensure_fhs_path_guard() -> None: + """Ensure /usr/local/bin is on PATH for RHEL-family root non-login shells. + + Mirrors the post-symlink probe added to ``scripts/install.sh`` so that + existing FHS-layout root installs on RHEL/CentOS/Rocky/Alma 8+ get + repaired on ``hermes update`` without requiring a reinstall. The + installer's assumption that ``/usr/local/bin`` is on PATH for every + standard shell breaks on those distros in non-login interactive shells + (su, sudo -s, tmux panes, some web terminals): /etc/bashrc doesn't + add /usr/local/bin and /root/.bash_profile doesn't either. Symptom: + ``hermes`` prints ``command not found`` even though the symlink lives + at /usr/local/bin/hermes. + + Silent no-op on: non-Linux, non-root, non-FHS installs, and any system + where ``bash -i -c 'command -v hermes'`` already resolves. Idempotent. + """ + if _m().sys.platform != "linux": + return + try: + if os.geteuid() != 0: # windows-footgun: ok — Linux FHS helper, guarded by sys.platform == "linux" above + AttributeError catch + return + except AttributeError: + return + # Only act when this is actually an FHS-layout install (command link at + # /usr/local/bin/hermes, code at /usr/local/lib/hermes-agent). + fhs_link = Path("/usr/local/bin/hermes") + if not fhs_link.is_symlink() and not fhs_link.exists(): + return + + # Probe a fresh non-login interactive bash the way the user will use it. + # ``bash -i -c`` sources ~/.bashrc but NOT ~/.bash_profile or /etc/profile, + # which is the exact scenario where RHEL root loses /usr/local/bin. + home = os.environ.get("HOME") or "/root" + try: + probe = subprocess.run( + [ + "env", + "-i", + f"HOME={home}", + f"TERM={os.environ.get('TERM', 'dumb')}", + "bash", + "-i", + "-c", + "command -v hermes", + ], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=10, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return # no bash or probe hung — don't block update on this + if probe.returncode == 0: + return # already on PATH, nothing to do + + path_line = 'export PATH="/usr/local/bin:$PATH"' + path_comment = ( + "# Hermes Agent — ensure /usr/local/bin is on PATH " "(RHEL non-login shells)" + ) + wrote_any = False + for candidate in (".bashrc", ".bash_profile"): + cfg = Path(home) / candidate + if not cfg.is_file(): + continue + try: + existing = cfg.read_text(errors="replace", encoding="utf-8") + except OSError: + continue + # Idempotency: skip if any uncommented PATH= line already references + # /usr/local/bin. Mirrors the grep pattern used by install.sh. + already_guarded = any( + "/usr/local/bin" in line + and "PATH" in line + and not line.lstrip().startswith("#") + for line in existing.splitlines() + ) + if already_guarded: + continue + try: + with cfg.open("a", encoding="utf-8") as f: + f.write("\n" + path_comment + "\n" + path_line + "\n") + except OSError as e: + print(f" ⚠ Could not update {cfg}: {e}") + continue + print(f" ✓ Added /usr/local/bin to PATH in {cfg}") + wrote_any = True + if wrote_any: + print(" (reload your shell or run 'source ~/.bashrc' to pick it up)") + +def _ensure_acp_launcher() -> None: + """Self-heal: install a ``hermes-acp`` launcher next to the ``hermes`` one. + + Mirrors the launcher block in ``scripts/install.sh`` so existing installs + gain the ACP command on ``hermes update`` without a reinstall. ACP hosts + (Zed, JetBrains, Buzz Desktop) spawn the agent by resolving the + ``hermes-acp`` command name against the login-shell PATH; the console + script of that name lives inside the install's venv, which is not on that + PATH, so those hosts report Hermes as not installed even when it is. + + The shim simply delegates to the sibling ``hermes`` launcher with the + ``acp`` subcommand, which makes it correct for every install layout + (venv wrapper, FHS symlink, pipx/pip console script) without having to + reconstruct interpreter/entrypoint paths. + + No-op on Windows (install.ps1 puts ``venv\\Scripts`` on the user PATH, so + ``hermes-acp.exe`` already resolves) and wherever a ``hermes-acp`` is + already present next to the ``hermes`` command. Unwritable directories + (e.g. ``/usr/local/bin`` as non-root) are skipped silently. Idempotent. + """ + if _m().sys.platform == "win32": + return + for bin_dir in (Path.home() / ".local" / "bin", Path("/usr/local/bin")): + hermes_cmd = bin_dir / "hermes" + acp_cmd = bin_dir / "hermes-acp" + try: + if not (hermes_cmd.is_file() or hermes_cmd.is_symlink()): + continue + # Already present — a console script (pip/pipx install), an + # earlier shim, or a symlink. is_symlink() catches broken + # symlinks that exists() would miss; never follow-and-overwrite + # (the #21454 failure mode). + if acp_cmd.exists() or acp_cmd.is_symlink(): + continue + shim = ( + "#!/usr/bin/env bash\n" + "# Hermes Agent — ACP launcher (written by `hermes update`).\n" + "# ACP hosts (Zed, JetBrains, Buzz) resolve the agent by this\n" + "# command name on the login-shell PATH.\n" + f'exec "{hermes_cmd}" acp "$@"\n' + ) + acp_cmd.write_text(shim, encoding="utf-8") + acp_cmd.chmod(acp_cmd.stat().st_mode | 0o755) + except OSError: + continue + print(f" ✓ Installed hermes-acp launcher → {acp_cmd}") + +_PRE_UPDATE_SNAPSHOT_KEEP = 1 + +# Per-file size cap for the pre-update quick snapshot. Anything larger is +# skipped with a warning: the snapshot exists to protect small, hard-to- +# regenerate state (pairing JSONs, cron jobs, config, auth) — not to copy a +# multi-GB state.db on every update (observed: a 24 GB state.db added ~60s +# of wall time and silently ate 24 GB of disk per update). +_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE = 1 << 30 # 1 GiB + +def _resolve_pre_update_backup_mode(args) -> str: + """Resolve the pre-update backup mode: ``"off"``, ``"quick"``, or ``"full"``. + + CLI flags win over config; ``--no-backup`` beats ``--backup`` when both + are set. Config accepts the mode strings plus legacy booleans: + ``true`` → ``full`` (the old zip behavior), ``false`` → ``off`` + (an explicit opt-out now disables the quick snapshot too — previously + it ran unconditionally, ignoring the user's setting). A missing key + defaults to ``quick``. + """ + if getattr(args, "no_backup", False): + return "off" + if getattr(args, "backup", False): + return "full" + + try: + from hermes_cli.config import load_config + + cfg = load_config() + except Exception as exc: + logging.getLogger(__name__).debug( + "Could not load config for pre-update backup: %s", exc + ) + cfg = {} + + updates_cfg = cfg.get("updates", {}) if isinstance(cfg, dict) else {} + raw = updates_cfg.get("pre_update_backup", "quick") + + if raw is True: + return "full" + if raw is False: + return "off" + mode = str(raw).strip().lower() + if mode in ("off", "false", "none", "disabled"): + return "off" + if mode in ("full", "zip", "true"): + return "full" + if mode == "quick": + return "quick" + logging.getLogger(__name__).warning( + "Unknown updates.pre_update_backup value %r — using 'quick'", raw + ) + return "quick" + +def _run_pre_update_backup(args) -> Optional[str]: + """Run the pre-update safety backup and return the quick-snapshot id. + + Single consolidated mechanism gated on ``updates.pre_update_backup``: + + - ``off`` — nothing runs. Explicit user opt-out is honored fully. + - ``quick`` (default) — a state snapshot of critical small files + (pairing JSONs, cron jobs, config, auth; see ``_QUICK_STATE_FILES``) + under ``state-snapshots/``. Files over 1 GiB are skipped with a + warning so a bloated state.db can never stall the update + (issues #15733, #34600 are the reason this safety net exists). + - ``full`` — the quick snapshot PLUS a full zip of HERMES_HOME under + ``backups/`` (restorable via ``hermes import``; the #48200 wrong-path + wipe is the reason this level exists). + + ``--backup`` forces ``full`` for one run; ``--no-backup`` forces ``off``. + Never raises — a backup failure should not block the update itself. + + Returns the quick-snapshot id (used by the post-update cron-jobs + restore safety net), or ``None`` when mode is ``off`` or the snapshot + failed. + """ + mode = _resolve_pre_update_backup_mode(args) + + if mode == "off": + if getattr(args, "no_backup", False): + print("◆ Pre-update backup: skipped (--no-backup)") + print() + # Config-level off is silent — the user opted out; don't spam them + # on every update. + return None + + snapshot_id = None + try: + from hermes_cli.backup import ( + _quick_snapshot_root, + create_quick_snapshot, + verify_sqlite_integrity, + ) + + # NOTE: this function later does `from hermes_constants import + # get_hermes_home`, which makes the name function-local — the + # module-level import is shadowed and unbound here. Alias explicitly. + from hermes_cli.config import get_hermes_home as _get_home + + snapshot_id = create_quick_snapshot( + label="pre-update", + keep=_PRE_UPDATE_SNAPSHOT_KEEP, + max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, + ) + + # After the snapshot, verify the source state.db is still intact. + # The snapshot was taken via _safe_copy_db (read-only SQLite backup + # API), but a concurrent process (antivirus, force-killed gateway + # releasing file handles, Windows filter driver) can corrupt the live + # file at any point. A silent zeroing at this point would proceed with + # the update and exit code 0 — exactly the #68474 symptom. + if snapshot_id: + _src_path = _get_home() / "state.db" + if _src_path.exists(): + _integrity = verify_sqlite_integrity( + _src_path, + check_header=True, + run_pragma=True, + max_bytes=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, + ) + if not _integrity.get("valid"): + _msg = _integrity.get("message", "unknown error") + print( + f" ⚠ state.db integrity check FAILED after snapshot: {_msg}" + ) + # Check if the snapshot itself is valid. + _snap_root = _quick_snapshot_root(_get_home()) + _snap_state = _snap_root / snapshot_id / "state.db" + if _snap_state.exists(): + _snap_ok = verify_sqlite_integrity( + _snap_state, check_header=True, run_pragma=True + ) + if _snap_ok.get("valid"): + print( + " ✓ Snapshot copy is valid — continuing update." + ) + print( + " If state.db is lost after update it will be auto-restored." + ) + else: + print( + " ✗ Snapshot copy ALSO failed integrity — " + "the source was already corrupted before the backup." + ) + else: + print( + " ⚠ Snapshot does not contain state.db (was skipped or too large)." + ) + print() + if snapshot_id: + print(f"◆ Pre-update snapshot: {snapshot_id}") + except Exception as exc: + # Never let a snapshot failure block an update. + logging.getLogger(__name__).debug("Pre-update snapshot failed: %s", exc) + + if mode != "full": + if snapshot_id: + print() + return snapshot_id + + try: + from hermes_cli.backup import create_pre_update_backup + except Exception as exc: + print( + f"⚠ Pre-update backup: could not load backup module ({exc}); continuing update." + ) + print() + return snapshot_id + + try: + from hermes_cli.config import load_config + + _keep = (load_config() or {}).get("updates", {}).get("backup_keep", 5) + except Exception: + _keep = 5 + + print("◆ Creating pre-update backup...") + t0 = _time.monotonic() + try: + out_path = create_pre_update_backup(keep=int(_keep)) + except Exception as exc: # defensive — helper already swallows, but just in case + print(f" ⚠ Backup failed: {exc}") + print(" Continuing with update.") + print() + return snapshot_id + + elapsed = _time.monotonic() - t0 + + if out_path is None: + print(" ⚠ Backup skipped (no files found or write failed); continuing update.") + print() + return snapshot_id + + try: + size_bytes = out_path.stat().st_size + except OSError: + size_bytes = 0 + + # Human-readable size + size_str = f"{size_bytes} B" + for unit in ("KB", "MB", "GB"): + if size_bytes < 1024: + break + size_bytes /= 1024 + size_str = f"{size_bytes:.1f} {unit}" + + # Render path using display_hermes_home so the user sees ~/.hermes/... + try: + from hermes_constants import get_hermes_home, display_hermes_home + + home = get_hermes_home() + try: + display_path = f"{display_hermes_home()}/{out_path.relative_to(home)}" + except ValueError: + display_path = str(out_path) + except Exception: + display_path = str(out_path) + + print(f" Saved: {display_path} ({size_str}, {elapsed:.1f}s)") + print(f" Restore: hermes import {out_path}") + print(" Disable: set updates.pre_update_backup: quick (or off) in config.yaml") + print() + return snapshot_id + +def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool: + """Write a planned-stop marker into a specific profile home.""" + try: + from datetime import timezone + + from gateway.status import _get_process_start_time + from utils import atomic_json_write + + record = { + "target_pid": pid, + "target_start_time": _get_process_start_time(pid), + "stopper_pid": os.getpid(), + "written_at": datetime.now(timezone.utc).isoformat(), + } + atomic_json_write( + Path(profile_path) / ".gateway-planned-stop.json", + record, + indent=None, + separators=(",", ":"), + ) + return True + except (OSError, PermissionError): + return False + +def _wait_for_windows_update_gateway_exit( + pids: list[int], *, timeout: float +) -> set[int]: + """Wait for the given gateway PIDs to exit, returning survivors.""" + if not pids: + return set() + + from gateway.status import _pid_exists + + remaining = set(pids) + deadline = _time.monotonic() + max(timeout, 0.0) + while remaining and _time.monotonic() < deadline: + for pid in list(remaining): + try: + if not _pid_exists(pid): + remaining.discard(pid) + except Exception: + remaining.discard(pid) + if remaining: + _time.sleep(0.25) + + survivors: set[int] = set() + for pid in remaining: + try: + if _pid_exists(pid): + survivors.add(pid) + except Exception: + pass + return survivors + +def _venv_core_imports_healthy() -> tuple[bool, str]: + """Probe the project venv for the core imports the backend needs to boot. + + Runs a tiny import check inside the venv interpreter (NOT this process — + ``hermes update`` may be driven by a different Python). Catches the + half-updated-venv state: git checkout current but a dependency sync that + failed or was killed partway (e.g. Windows access-denied on a loaded + .pyd), leaving imports like ``fastapi``'s new transitive deps missing. + Without this probe, ``hermes update`` on a current checkout prints + "Already up to date!" and returns without ever re-syncing dependencies — + the user's install stays broken no matter how many times they update + (ryanc's incident, July 2026). + + Returns ``(healthy, detail)``. Never raises; unknown states report + healthy so a probe failure can't force needless reinstalls. + """ + venv_dir = _m().PROJECT_ROOT / "venv" + python_name = "python.exe" if _m()._is_windows() else "python" + bin_dir = "Scripts" if _m()._is_windows() else "bin" + venv_python = venv_dir / bin_dir / python_name + if not venv_python.exists(): + # No venv interpreter at all. In a dev checkout that's normal (the + # dev may run hermes from any interpreter), so report healthy to + # avoid forcing reinstalls. But on a MANAGED install (the Windows + # installer / desktop bootstrap stamps `.hermes-bootstrap-complete`, + # and an interrupted update leaves `.update-incomplete`), the venv + # IS the install — its absence means a repair got interrupted after + # the old venv was moved aside, and "Already up to date!" would + # gaslight the user while nothing can run. + managed_markers = ( + _m().PROJECT_ROOT / ".hermes-bootstrap-complete", + _m()._update_marker_path(), + ) + if any(m.exists() for m in managed_markers): + return False, f"venv python missing ({venv_python})" + return True, "" + + # Core web/serve imports plus their newest transitive deps. Import (not + # just metadata) — a package can have intact dist-info but a missing + # module after an interrupted uninstall/install cycle. + check = ( + "import importlib\n" + "mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n" + "missing = []\n" + "for m in mods:\n" + " try: importlib.import_module(m)\n" + " except Exception as e: missing.append(f'{m}: {e}')\n" + "print('\\n'.join(missing))\n" + ) + try: + result = subprocess.run( + [str(venv_python), "-c", check], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=60, + cwd=_m().PROJECT_ROOT, + ) + except Exception as exc: + logger.debug("venv health probe failed to run: %s", exc) + return True, "" + + missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] + if result.returncode != 0 and not missing: + # Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy. + detail = (result.stderr or "").strip().splitlines() + return False, detail[0] if detail else "venv python failed to run" + if missing: + return False, "; ".join(missing[:4]) + return True, "" + +def _detect_venv_python_processes( + *, exclude_pids: set[int] | None = None +) -> list[tuple[int, str, str]]: + """Find live processes running from the project venv's interpreter. + + The hermes.exe shim guard misses the biggest lock-holder class on + Windows: the Desktop app's backend (``python.exe -m hermes_cli.main + serve``) and anything else running straight off ``venv\\Scripts\\python + (w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a + dependency sync mid-update dies with access-denied and strands the venv + half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026). + + Killing them from here is pointless — the Desktop app supervises its + backend and respawns it within seconds — so the caller should refuse and + tell the user to close the app instead. Returns ``(pid, name, cmdline)`` + tuples; empty off-Windows / without psutil / when nothing matches. The + calling process and its ancestors are always excluded (a CLI ``hermes + update`` itself runs from the venv python). Never raises. + """ + if not _m()._is_windows(): + return [] + try: + import psutil + except Exception: + return [] + + venv_dir = _m().PROJECT_ROOT / "venv" + try: + venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep + except OSError: + venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep + try: + root_prefix = str(_m().PROJECT_ROOT.resolve()).lower().rstrip(os.sep) + os.sep + except OSError: + root_prefix = str(_m().PROJECT_ROOT).lower().rstrip(os.sep) + os.sep + + skip: set[int] = set(exclude_pids or set()) + skip.add(os.getpid()) + try: + for anc in psutil.Process().parents(): + skip.add(int(anc.pid)) + except Exception: + pass + + matches: list[tuple[int, str, str]] = [] + try: + proc_iter = psutil.process_iter(["pid", "exe", "name", "cmdline", "cwd"]) + except Exception: + return [] + for proc in proc_iter: + try: + info = proc.info + except Exception: + continue + pid = info.get("pid") + exe = info.get("exe") + if not exe or pid is None or int(pid) in skip: + continue + try: + exe_norm = str(Path(exe).resolve()).lower() + except (OSError, ValueError): + exe_norm = str(exe).lower() + cmdline_raw = " ".join(info.get("cmdline") or []) + cmdline_low = cmdline_raw.lower() + cwd_low = str(info.get("cwd") or "").lower().rstrip(os.sep) + os.sep + + # Primary match: the executable itself lives under this venv + # (venv\Scripts\python(w).exe — the desktop backend / gateway case). + is_holder = exe_norm.startswith(venv_prefix) + # Fallback: uv/base-interpreter trampolines run a python whose exe is + # OUTSIDE the venv but which still imports from it and holds its .pyd + # files. Catch those by what they're running: a cmdline that references + # this venv's path, or a `-m hermes_cli.main ...` invocation tied to + # this install (install root in the cmdline or as the working dir). + if not is_holder and venv_prefix in cmdline_low: + is_holder = True + if not is_holder and "hermes_cli.main" in cmdline_low: + if root_prefix in cmdline_low or cwd_low.startswith(root_prefix): + is_holder = True + if not is_holder: + continue + name = info.get("name") or Path(exe).name + matches.append((int(pid), str(name), cmdline_raw[:120])) + return matches + +def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str: + """Explain which venv processes block the update and how to clear them.""" + lines = [ + "✗ Other Hermes processes are running from this install's venv:", + ] + for pid, name, cmdline in matches[:6]: + hint = "" + low = cmdline.lower() + if "serve" in low or "dashboard" in low: + hint = " ← Hermes Desktop backend (close the desktop app)" + elif "gateway" in low: + hint = " ← gateway" + lines.append(f" PID {pid} {name} {cmdline}{hint}") + if len(matches) > 6: + lines.append(f" ... and {len(matches) - 6} more") + lines.append("") + lines.append( + " On Windows these keep native extension files (.pyd) locked, so the" + ) + lines.append( + " dependency update would fail partway and leave a broken install." + ) + lines.append( + " Close the Hermes desktop app / other Hermes terminals, then re-run:" + ) + lines.append(" hermes update") + lines.append(" (or use `hermes update --force-venv` to proceed anyway at your own risk)") + return "\n".join(lines) + +def _pause_windows_gateways_for_update() -> dict | None: + """Stop running Windows gateways before mutating the checkout or venv. + + Windows scheduled/startup gateways run through pythonw.exe, so the generic + hermes.exe concurrent-instance guard does not see them. They still import + from the checkout and can keep files locked while ``git`` or ``uv`` updates + the install. Stop only PIDs that the gateway discovery code identifies. + """ + if not _m()._is_windows(): + return None + + try: + from gateway.status import terminate_pid + from hermes_cli.gateway import ( + _capture_gateway_argv, + _get_restart_drain_timeout, + find_gateway_pids, + find_profile_gateway_processes, + ) + except Exception as exc: + logger.debug("Could not prepare Windows gateway pause for update: %s", exc) + return None + + try: + running_pids = list(dict.fromkeys(find_gateway_pids(all_profiles=True))) + except Exception as exc: + logger.debug("Could not discover Windows gateway PIDs before update: %s", exc) + return None + if not running_pids: + # No gateway is running right now, but the user may have installed an + # autostart entry (Scheduled Task or Startup-folder login item) — that + # is an explicit "I want a gateway" signal. A gateway that died between + # updates (e.g. the spawning terminal/TUI closed, taking its child with + # it) would otherwise never come back: the autostart entry only fires on + # the next login, and the update flow's resume path only relaunched + # gateways that were running when the update began. Cold-start one after + # the update so an installed gateway is actually up post-update. Users + # who run gateway-less (no autostart entry) get nothing forced on them. + try: + from hermes_cli import gateway_windows + + if gateway_windows.is_installed(): + return { + "resume_needed": True, + "profiles": {}, + "unmapped_pids": [], + "unmapped": [], + "cold_start_if_installed": True, + } + except Exception as exc: + logger.debug( + "Could not check Windows gateway autostart state before update: %s", + exc, + ) + return None + + profile_processes = {} + try: + profile_processes = { + proc.pid: proc for proc in find_profile_gateway_processes() + } + except Exception as exc: + logger.debug("Could not map Windows gateway PIDs to profiles: %s", exc) + + profiles: dict[str, int] = {} + mapped_pids = [] + for pid in running_pids: + proc = profile_processes.get(pid) + if proc is None: + continue + profiles[str(proc.profile)] = int(pid) + mapped_pids.append(int(pid)) + _write_update_planned_stop_marker(Path(proc.path), int(pid)) + + print("→ Stopping Windows gateway process(es) before updating Hermes...") + try: + drain_timeout = max(float(_get_restart_drain_timeout()), 1.0) + except Exception: + drain_timeout = 10.0 + survivors = _m()._wait_for_windows_update_gateway_exit( + mapped_pids, + timeout=drain_timeout, + ) + unmapped_pids = [pid for pid in running_pids if pid not in profile_processes] + + # Snapshot each unmapped gateway's command line *before* we force-kill it, + # so ``_resume_windows_gateways_after_update`` can respawn it by replaying + # its own argv. Unmapped gateways are ones with no profile→PID-file mapping + # — e.g. a Windows Scheduled Task running ``pythonw.exe -m hermes_cli.main + # gateway run``. Without this snapshot they were force-killed and never + # restarted (the "Restart manually after update" dead-end from #50090). + unmapped: list[dict] = [] + for pid in unmapped_pids: + argv = None + try: + argv = _capture_gateway_argv(int(pid)) + except Exception as exc: + logger.debug("Could not capture argv for unmapped gateway %s: %s", pid, exc) + unmapped.append({"pid": int(pid), "argv": argv}) + + force_killed = [] + for pid in sorted(set(survivors).union(unmapped_pids)): + try: + terminate_pid(int(pid), force=True) + force_killed.append(int(pid)) + except (ProcessLookupError, PermissionError, OSError): + pass + + if profiles: + print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}") + if force_killed: + print(f" → Force-stopped {len(force_killed)} gateway process(es)") + + if unmapped_pids: + respawnable = sum(1 for u in unmapped if u.get("argv")) + print( + f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping" + ) + if respawnable < len(unmapped_pids): + # Some had no recoverable command line (psutil missing, access + # denied, already gone): those still need a manual restart. + print(" Restart manually after update: hermes gateway run") + + return { + "resume_needed": True, + "profiles": profiles, + "unmapped_pids": unmapped_pids, + "unmapped": unmapped, + } + +def _cold_start_windows_gateway_after_update() -> None: + """Start a fresh detached gateway after update when one is installed but down. + + Invoked from ``_resume_windows_gateways_after_update`` for the + ``cold_start_if_installed`` case: no gateway was running when the update + began, but an autostart entry (Scheduled Task / Startup-folder login item) + is installed, signalling the user wants a gateway. Unlike the relaunch + paths — which watch an old PID and respawn once it exits — this is a direct + fresh spawn via the same hidden-console + breakaway path that + ``hermes gateway start`` uses (``gateway_windows._spawn_detached``). + + Best-effort and idempotent: re-checks that nothing is running first so a + concurrent start (e.g. the autostart entry firing) can't produce a + duplicate gateway. + """ + if not _m()._is_windows(): + return + try: + from hermes_cli import gateway_windows + from hermes_cli.gateway import find_gateway_pids + except Exception as exc: + logger.debug("Could not load Windows gateway cold-start helpers: %s", exc) + return + + # Re-check liveness right before spawning — between pause and resume the + # autostart entry may have already brought a gateway up, or a leftover + # process may have re-registered. Don't double-start. + try: + if list(find_gateway_pids(all_profiles=True)): + return + except Exception as exc: + logger.debug("Could not re-check gateway liveness before cold-start: %s", exc) + return + + try: + pid = gateway_windows._spawn_detached() + except Exception as exc: + logger.debug("Could not cold-start Windows gateway after update: %s", exc) + return + + if pid: + print() + print(f" ✓ Starting Windows gateway after update (PID {pid})") + +def _for_each_systemd_gateway_unit( + list_units_stdout: str, + *, + process_unit, + on_unit_timeout, +) -> None: + """Process each ``hermes-gateway*.service`` from ``systemctl list-units``. + + ``subprocess.TimeoutExpired`` raised by ``process_unit`` is isolated to + that unit via ``on_unit_timeout`` so one wedged systemctl call cannot + abort the rest of the fleet (#68523). + """ + for line in (list_units_stdout or "").strip().splitlines(): + parts = line.split() + if not parts: + continue + unit = parts[0] + if not unit.endswith(".service"): + continue + # list-units is already pattern-filtered, but keep the name gate so a + # stray non-gateway line cannot enter the restart path. + if not unit.startswith("hermes-gateway"): + continue + svc_name = unit.removesuffix(".service") + try: + process_unit(svc_name) + except subprocess.TimeoutExpired as exc: + on_unit_timeout(svc_name, exc) + +def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None: + """Print an explicit incomplete-update warning for unrestarted units.""" + if not failed_units: + return + # Preserve discovery order while de-duplicating. + seen = set() + ordered = [] + for name in failed_units: + if name in seen: + continue + seen.add(name) + ordered.append(name) + print() + print("⚠ Update incomplete — some gateway units were not restarted:") + for name in ordered: + print(f" - {name}") + print(" Skipped units may still be running pre-update code (mixed") + print(" sys.modules). Restart them manually, then verify:") + print(" hermes gateway status") + print(" systemctl --user restart # user-scope") + print(" sudo systemctl restart # system-scope") + +def _refresh_windows_gateway_launchers() -> None: + """Regenerate installed Windows gateway launcher scripts after update. + + The Scheduled Task / Startup-folder launchers (``gateway.cmd`` + + ``gateway.vbs``) are persistence artifacts written once at install time — + ``hermes update`` never touched them, so installs created before the + hidden-console rework (aa2ae36c3f) kept launching the gateway through + ``pythonw.exe`` forever: every descendant spawn flashed a conhost + (#54220/#56747) and, since #70344, the console-less gateway died at + startup with ``RuntimeError: sys.stderr is None`` (#71671). + + The task's /TR points at a stable script path, so rewriting the files in + place retargets the task without any schtasks call (no UAC needed). + ``_write_task_script`` is idempotent and renders from current code, so + this is a no-op for modern installs. Best-effort: a failed refresh must + never fail the update. + """ + if not _m()._is_windows(): + return + try: + from hermes_cli import gateway_windows + + if not gateway_windows.is_installed(): + return + gateway_windows._write_task_script() + print(" ✓ Refreshed Windows gateway launcher scripts") + except Exception as exc: + logger.debug("Could not refresh Windows gateway launchers after update: %s", exc) + +def _resume_windows_gateways_after_update(token: dict | None) -> None: + """Restart Windows profile gateways previously paused for update.""" + if not token or not token.get("resume_needed"): + return + token["resume_needed"] = False + if not _m()._is_windows(): + return + + # Regenerate the persisted launcher scripts before respawning anything, + # so a legacy pythonw-era Scheduled Task / Startup entry comes back on + # the current hidden-console design at the next login too. + _m()._refresh_windows_gateway_launchers() + + profiles = token.get("profiles") or {} + unmapped = token.get("unmapped") or [] + cold_start = bool(token.get("cold_start_if_installed")) + if not profiles and not any(u.get("argv") for u in unmapped): + if cold_start: + _m()._cold_start_windows_gateway_after_update() + return + + try: + from hermes_cli.gateway import ( + launch_detached_gateway_restart_by_cmdline, + launch_detached_profile_gateway_restart, + ) + except Exception as exc: + logger.debug("Could not load Windows gateway restart helper: %s", exc) + return + + relaunched = [] + for profile, old_pid in sorted(profiles.items()): + try: + if launch_detached_profile_gateway_restart(str(profile), int(old_pid)): + relaunched.append(str(profile)) + except Exception as exc: + logger.debug( + "Could not restart Windows gateway profile %s after update: %s", + profile, + exc, + ) + + # Respawn unmapped gateways (no profile→PID-file mapping, e.g. a Scheduled + # Task) by replaying the argv we snapshotted before force-killing them. + unmapped_relaunched = 0 + for entry in unmapped: + argv = entry.get("argv") + old_pid = entry.get("pid") + if not argv or not old_pid: + continue + try: + if launch_detached_gateway_restart_by_cmdline(int(old_pid), list(argv)): + unmapped_relaunched += 1 + except Exception as exc: + logger.debug( + "Could not restart unmapped Windows gateway (pid %s) after update: %s", + old_pid, + exc, + ) + + if relaunched: + print() + print(f" ✓ Restarting Windows gateway profile(s): {', '.join(relaunched)}") + if unmapped_relaunched: + if not relaunched: + print() + print( + f" ✓ Restarting {unmapped_relaunched} unmapped Windows gateway process(es)" + ) + +def _discard_lockfile_churn(git_cmd, repo_root): + """Restore tracked ``package-lock.json`` files that npm dirtied locally. + + npm rewrites lockfiles non-deterministically at install/build time. On a + managed install those diffs are never intentional, so we discard them so + ``hermes update`` sees a clean tree instead of autostashing every run. + Best-effort; only ever touches files named ``package-lock.json``. + """ + try: + diff = subprocess.run( + git_cmd + ["diff", "--name-only"], + cwd=repo_root, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if diff.returncode != 0: + return + dirty_package_dirs = { + Path(line.strip()).parent + for line in diff.stdout.splitlines() + if line.strip().endswith("package.json") + } + dirty = [ + line.strip() + for line in diff.stdout.splitlines() + if line.strip().endswith("package-lock.json") + and Path(line.strip()).parent not in dirty_package_dirs + ] + if not dirty: + return + subprocess.run( + git_cmd + ["checkout", "--", *dirty], + cwd=repo_root, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=False, + ) + print(f"→ Discarded npm lockfile churn ({len(dirty)} file(s))") + except Exception: + # Never let lockfile cleanup block an update. + pass + +def _cmd_update_impl(args, gateway_mode: bool): + """Body of ``cmd_update`` — kept separate so the wrapper can always + restore stdio even on ``sys.exit``.""" + # In gateway mode, use file-based IPC for prompts instead of stdin + gw_input_fn = ( + (lambda prompt, default="": _gateway_prompt(prompt, default)) + if gateway_mode + else None + ) + assume_yes = bool(getattr(args, "yes", False)) + + # Whether this update is running without a human at the keyboard. + # Interactive terminal updates always stash-and-ask (unchanged behavior); + # only non-interactive updates (desktop/chat app, gateway, `--yes`) consult + # the `updates.non_interactive_local_changes` config setting to decide + # whether to auto-restore stashed local source changes or throw them away. + _non_interactive_update = ( + gateway_mode + or assume_yes + or not (_m().sys.stdin.isatty() and _m().sys.stdout.isatty()) + ) + discard_local_changes = False + if _non_interactive_update: + try: + from hermes_cli.config import load_config + + _update_cfg = (load_config() or {}).get("updates", {}) + if isinstance(_update_cfg, dict): + _mode = str(_update_cfg.get("non_interactive_local_changes", "stash")).lower() + discard_local_changes = _mode == "discard" + except Exception as exc: + # Never let a config read failure change the safe default. + logger.debug("Could not read updates.non_interactive_local_changes: %s", exc) + discard_local_changes = False + + print("⚕ Updating Hermes Agent...") + print() + + # On Windows, abort early if another hermes.exe is holding the venv shim + # open. Continuing would result in a string of WinError 32 warnings and + # then either a deferred-rename leftover or a failed git-pull fast path + # that silently falls back to the slower ZIP route. See issue #26670. + if _m()._is_windows() and not getattr(args, "force", False): + scripts_dir = _m()._venv_scripts_dir() + if scripts_dir is not None: + concurrent = _m()._detect_concurrent_hermes_instances(scripts_dir) + if concurrent: + print(_format_concurrent_instances_message(concurrent, scripts_dir)) + _m().sys.exit(2) + + # Pre-update backup — runs before any git/file mutation so users can + # always roll back to the exact state they had before this update. + # Returns the quick-snapshot id (or None when disabled/failed); the + # post-update cron-jobs safety net uses it to detect job loss. + pre_update_snapshot_id = _m()._run_pre_update_backup(args) + + _windows_gateway_resume = _m()._pause_windows_gateways_for_update() + if _windows_gateway_resume: + import atexit as _atexit + + _atexit.register( + _m()._resume_windows_gateways_after_update, + _windows_gateway_resume, + ) + + # With gateways paused, anything still running from the venv interpreter + # (most commonly the Desktop app's `hermes serve` backend) will keep .pyd + # files locked and corrupt the dependency sync below. Refuse rather than + # race: killing the desktop backend is futile (the app supervises and + # respawns it), so the user must close the app. Deliberately NOT bypassed + # by plain --force: the desktop bootstrap updater passes --force to skip + # the hermes.exe shim guard above, but its lock probe only checks the shim + # and app.asar — a non-desktop venv python holding a .pyd would sail + # through and corrupt the sync (the exact failure this guard exists for). + # --force-venv is the explicit escape hatch. + if _m()._is_windows() and not getattr(args, "force_venv", False): + _venv_holders = _m()._detect_venv_python_processes() + if _venv_holders: + print(_format_venv_python_holders_message(_venv_holders)) + _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + _m().sys.exit(2) + + # Try git-based update first, fall back to ZIP download on Windows + # when git file I/O is broken (antivirus, NTFS filter drivers, etc.) + use_zip_update = False + git_dir = _m().PROJECT_ROOT / ".git" + + if not git_dir.exists(): + if _m().sys.platform == "win32": + use_zip_update = True + else: + print("✗ Not a git repository. Please reinstall:") + print( + " curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash" + ) + _m().sys.exit(1) + + # On Windows, git can fail with "unable to write loose object file: Invalid argument" + # due to filesystem atomicity issues. Set the recommended workaround. + if _m().sys.platform == "win32" and git_dir.exists(): + subprocess.run( + [ + "git", + "-c", + "windows.appendAtomically=false", + "config", + "windows.appendAtomically", + "false", + ], + cwd=_m().PROJECT_ROOT, + check=False, + capture_output=True, + ) + + # Build git command once — reused for fork detection and the update itself. + git_cmd = ["git"] + if _m().sys.platform == "win32": + git_cmd = ["git", "-c", "windows.appendAtomically=false"] + + # Discard npm lockfile churn before any stash/branch logic. npm rewrites + # tracked package-lock.json files non-deterministically at install/build + # time (platform-specific optional deps, ideallyInert annotations, etc.), + # which is never an intentional edit on a managed install but leaves the + # tree dirty — forcing an autostash on every update and making branch + # switches fragile. Restoring them first lets the common case (only + # lockfile churn) update with a clean tree. + _discard_lockfile_churn(git_cmd, _m().PROJECT_ROOT) + + # Detect if we're updating from a fork (before any branch logic) + origin_url = _m()._get_origin_url(git_cmd, _m().PROJECT_ROOT) + is_fork = _is_fork(origin_url) + + if is_fork: + print("⚠ Updating from fork:") + print(f" {origin_url}") + print() + + if use_zip_update: + # ZIP-based update for Windows when git is broken + try: + _update_via_zip(args) + finally: + _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + return + + # Fetch and pull + try: + + # Resolve the target branch up front so the fetch can be scoped to it. + # A bare `git fetch origin` pulls every ref, and this repo carries + # thousands of auto-generated branches — an unscoped fetch can stall for + # minutes on a non-single-branch checkout. Fetch only what we update + # against. + branch = _m()._resolve_update_branch(args) + + print("→ Fetching updates...") + fetch_result = subprocess.run( + git_cmd + ["fetch", "origin", branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if fetch_result.returncode != 0: + stderr = fetch_result.stderr.strip() + if "Could not resolve host" in stderr or "unable to access" in stderr: + print("✗ Network error — cannot reach the remote repository.") + print(f" {stderr.splitlines()[0]}" if stderr else "") + elif ( + "Authentication failed" in stderr or "could not read Username" in stderr + ): + print( + "✗ Authentication failed — check your git credentials or SSH key." + ) + else: + print("✗ Failed to fetch updates from origin.") + if stderr: + print(f" {stderr.splitlines()[0]}") + _m().sys.exit(1) + + # Get current branch (returns literal "HEAD" when detached) + result = subprocess.run( + git_cmd + ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=True, + ) + current_branch = result.stdout.strip() + + # If user is on a different branch than the update target, switch + # to the target. When the target is "main" this is the historical + # "always update against main" behavior; for any other target it's + # the same thing — get HEAD onto the requested branch first, then + # fast-forward. + if current_branch != branch: + label = ( + "detached HEAD" + if current_branch == "HEAD" + else f"branch '{current_branch}'" + ) + print(f" ⚠ Currently on {label} — switching to {branch} for update...") + # Stash before checkout so uncommitted work isn't lost + auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT) + checkout_result = subprocess.run( + git_cmd + ["checkout", branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if checkout_result.returncode != 0: + # Local checkout doesn't have this branch yet. Try to set + # it up as a tracking branch of origin/. This is + # the common case when the requested branch exists upstream + # but was never checked out locally. + track_result = subprocess.run( + git_cmd + ["checkout", "-B", branch, f"origin/{branch}"], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if track_result.returncode != 0: + # Restore the user's prior branch + stash before bailing + # so we don't leave them stranded in a weird state. + if auto_stash_ref is not None: + _m()._restore_stashed_changes( + git_cmd, + _m().PROJECT_ROOT, + auto_stash_ref, + prompt_user=False, + input_fn=gw_input_fn, + ) + print(f"✗ Branch '{branch}' does not exist locally or on origin.") + if track_result.stderr.strip(): + print(f" {track_result.stderr.strip().splitlines()[0]}") + _m().sys.exit(1) + else: + auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT) + + prompt_for_restore = ( + auto_stash_ref is not None + and not assume_yes + and (gateway_mode or (_m().sys.stdin.isatty() and _m().sys.stdout.isatty())) + ) + + # Check if there are updates + result = subprocess.run( + git_cmd + ["rev-list", f"HEAD..origin/{branch}", "--count"], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=True, + ) + commit_count = int(result.stdout.strip()) + + if commit_count == 0: + _invalidate_update_cache() + + # Even if origin is up to date, the fork may be behind upstream + if is_fork and branch == "main": + _m()._sync_with_upstream_if_needed(git_cmd, _m().PROJECT_ROOT) + + # Restore stash and switch back to original branch if we moved + if auto_stash_ref is not None: + _m()._restore_stashed_changes( + git_cmd, + _m().PROJECT_ROOT, + auto_stash_ref, + prompt_user=prompt_for_restore, + input_fn=gw_input_fn, + ) + if current_branch not in {branch, "HEAD"}: + subprocess.run( + git_cmd + ["checkout", current_branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=False, + ) + + # "No new commits" does not mean the managed interpreter is safe. + # uv can retain the same CPython patch while python-build-standalone + # refreshes the embedded SQLite underneath it. Keep the existing + # update-boundary hook active on this retry path too. + from hermes_cli.managed_uv import ensure_uv, update_managed_uv + + runtime_repairs = [] + update_managed_uv(repair_observer=runtime_repairs.append) + ensure_uv(repair_observer=runtime_repairs.append) + runtime_repaired = next( + (result for result in runtime_repairs if result.repaired), + None, + ) + + # A current checkout does NOT imply a healthy install: a previous + # dependency sync may have failed partway (classic on Windows, + # where a running gateway/desktop backend keeps .pyd files locked + # and uv/pip dies with access-denied, stranding the venv between + # versions). Probe the venv's core imports and repair if broken — + # otherwise "Already up to date!" gaslights the user while their + # install stays bricked. + healthy, detail = _venv_core_imports_healthy() + if not healthy: + print("⚠ Checkout is current, but the venv is unhealthy:") + print(f" {detail}") + print("→ Repairing Python dependencies...") + _write_update_incomplete_marker() + from hermes_cli.managed_uv import ensure_uv + + repair_uv = ensure_uv() + # A managed install whose venv is gone entirely (interrupted + # repair after the old venv was moved aside) needs the venv + # recreated before dependencies can be installed into it. + venv_python_missing = not ( + _m().PROJECT_ROOT + / "venv" + / ("Scripts" if _m()._is_windows() else "bin") + / ("python.exe" if _m()._is_windows() else "python") + ).exists() + if venv_python_missing and repair_uv: + print("→ Recreating virtual environment...") + subprocess.run( + [repair_uv, "venv", "venv"], + cwd=_m().PROJECT_ROOT, + check=False, + ) + if repair_uv: + repair_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")} + _m()._install_python_dependencies_with_optional_fallback( + [repair_uv, "pip"], env=repair_env, group="all" + ) + else: + _m()._install_python_dependencies_with_optional_fallback( + [_m().sys.executable, "-m", "pip"], group="all" + ) + _m()._clear_update_incomplete_marker() + healthy_after, detail_after = _venv_core_imports_healthy() + if healthy_after: + print("✓ Dependencies repaired!") + else: + print(f"⚠ Venv still unhealthy after repair: {detail_after}") + print(" Close all Hermes windows/gateways and re-run: hermes update") + else: + print("✓ Already up to date!") + if runtime_repaired is not None and not _m()._is_windows(): + print() + print( + "⚠ Restart required to finish the managed Python runtime repair." + ) + print( + " Any running Hermes gateways, Desktop backends, or other " + "long-lived processes still use the previous runtime." + ) + print( + " Restart each of them before removing the parked venv" + + ( + f": {runtime_repaired.backup_venv}" + if runtime_repaired.backup_venv is not None + else "." + ) + ) + _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + return + + print(f"→ Found {commit_count} new commit(s)") + + print("→ Pulling updates...") + update_succeeded = False + # Capture the pre-pull SHA so we can auto-roll-back if the new code + # has a syntax error in a critical-path file (PR #28452 incident: + # orphan merge-conflict markers in hermes_cli/config.py bricked + # every user who ran ``hermes update`` for the 7 minutes between + # the bad commit and the fix landing). + pre_pull_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) + try: + pull_result = subprocess.run( + git_cmd + ["pull", "--ff-only", "origin", branch], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if pull_result.returncode != 0: + # ff-only failed — local and remote have diverged (e.g. upstream + # force-pushed or rebase). Since local changes are already + # stashed, reset to match the remote exactly. + print( + " ⚠ Fast-forward not possible (history diverged), resetting to match remote..." + ) + reset_result = subprocess.run( + git_cmd + ["reset", "--hard", f"origin/{branch}"], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if reset_result.returncode != 0: + print(f"✗ Failed to reset to origin/{branch}.") + if reset_result.stderr.strip(): + print(f" {reset_result.stderr.strip()}") + print( + f" Try manually: git fetch origin && git reset --hard origin/{branch}" + ) + _m().sys.exit(1) + + # Post-pull syntax guard: validate critical-path files actually + # parse before declaring the update successful. If a bad commit + # made it through CI (e.g. admin-merge bypass of a failing + # ruff check), this catches it on the user side and rolls back + # so the CLI stays bootable. The user can then retry ``hermes + # update`` later once a fix lands upstream. + syntax_ok, failing_path, syntax_error = _validate_critical_files_syntax( + _m().PROJECT_ROOT + ) + if not syntax_ok: + print() + print("✗ Pulled code has a syntax error in a critical file:") + print(f" {failing_path}") + if syntax_error: + # py_compile errors can be multi-line; show the first + # ~6 lines so the user sees the actual SyntaxError text. + for line in str(syntax_error).splitlines()[:6]: + print(f" {line}") + if pre_pull_sha: + print() + print(f"→ Rolling back to {pre_pull_sha[:10]}...") + rollback_result = subprocess.run( + git_cmd + ["reset", "--hard", pre_pull_sha], + cwd=_m().PROJECT_ROOT, + capture_output=True, + text=True, encoding="utf-8", errors="replace", + ) + if rollback_result.returncode == 0: + print(" ✓ Rollback complete — your install is unchanged.") + print(" Try ``hermes update`` again later once a fix lands.") + else: + print(" ✗ Rollback failed. Recover manually with:") + print(f" cd {_m().PROJECT_ROOT} && git reset --hard {pre_pull_sha}") + if rollback_result.stderr.strip(): + print(f" ({rollback_result.stderr.strip().splitlines()[0]})") + else: + print() + print(" Could not capture pre-pull SHA — recover manually with:") + print(f" cd {_m().PROJECT_ROOT} && git reflog && git reset --hard ") + _m().sys.exit(1) + + update_succeeded = True + finally: + if auto_stash_ref is not None: + # Don't attempt stash restore if the code update itself failed — + # working tree is in an unknown state. + if not update_succeeded: + print( + f" ℹ️ Local changes preserved in stash (ref: {auto_stash_ref})" + ) + print(" Restore manually with: git stash apply") + elif discard_local_changes: + # Non-interactive update + user opted into discarding local + # source edits (updates.non_interactive_local_changes: + # discard). Throw the stash away instead of re-applying it. + _m()._discard_stashed_changes( + git_cmd, + _m().PROJECT_ROOT, + auto_stash_ref, + ) + else: + _m()._restore_stashed_changes( + git_cmd, + _m().PROJECT_ROOT, + auto_stash_ref, + prompt_user=prompt_for_restore, + input_fn=gw_input_fn, + ) + + _invalidate_update_cache() + + # Clear stale .pyc bytecode cache — prevents ImportError on gateway + # restart when updated source references names that didn't exist in + # the old bytecode (e.g. get_hermes_home added to hermes_constants). + removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) + if removed: + print( + f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" + ) + _m()._record_bytecode_fingerprint() + + # Fork upstream sync logic (only for main branch on forks) + if is_fork and branch == "main": + _m()._sync_with_upstream_if_needed(git_cmd, _m().PROJECT_ROOT) + + # Reinstall Python dependencies. Prefer .[all], but if one optional extra + # breaks on this machine, keep base deps and reinstall the remaining extras + # individually so update does not silently strip working capabilities. + # + # Drop the core-install breadcrumb BEFORE touching the venv. If the + # install is killed mid-flight (Ctrl-C, terminal close, WSL OOM), the + # marker survives and the next ``hermes`` launch finishes the install + # via ``_recover_from_interrupted_install``. Cleared after the core + # ``.[all]`` install completes — lazy refresh uses a separate marker. + _write_update_incomplete_marker() + print("→ Updating Python dependencies...") + from hermes_cli.managed_uv import ensure_uv, update_managed_uv + + # Keep managed uv current — runs `uv self update` if we already have one. + update_managed_uv() + + uv_bin = ensure_uv() + + pip_cmd = [_m().sys.executable, "-m", "pip"] + if not uv_bin: + uv_bin = _ensure_uv_for_termux(pip_cmd) + install_group = "all" + + if uv_bin: + uv_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")} + if _m()._is_termux_env(uv_env): + uv_env.pop("PYTHONPATH", None) + uv_env.pop("PYTHONHOME", None) + install_group = "termux-all" + print(" → Termux detected: using uv + curated termux-all optional profile...") + if _m()._is_termux_env(uv_env) and _is_android_python(): + print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...") + _install_psutil_android_compat([uv_bin, "pip"], env=uv_env) + _m()._install_python_dependencies_with_optional_fallback( + [uv_bin, "pip"], env=uv_env, group=install_group + ) + else: + # Use sys.executable to explicitly call the venv's pip module, + # avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu. + # Some environments lose pip inside the venv; bootstrap it back with + # ensurepip before trying the editable install. + pip_cmd = [_m().sys.executable, "-m", "pip"] + try: + subprocess.run( + pip_cmd + ["--version"], + cwd=_m().PROJECT_ROOT, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError: + subprocess.run( + [_m().sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], + cwd=_m().PROJECT_ROOT, + check=True, + ) + if _m()._is_termux_env(): + install_group = "termux-all" + print(" → Termux detected: using curated termux-all optional profile...") + if _m()._is_termux_env() and _is_android_python(): + print(" → Termux/Android detected: prebuilding psutil with Linux source path compatibility...") + _install_psutil_android_compat(pip_cmd) + _m()._install_python_dependencies_with_optional_fallback(pip_cmd, group=install_group) + + install_prefix = [uv_bin, "pip"] if uv_bin else pip_cmd + lazy_env = uv_env if uv_bin else None + + # Core ``.[all]`` install finished. Clear the generic core breadcrumb + # before the lazy-refresh phase — that phase uses its own marker so a + # later lazy failure cannot be "healed" by clearing the core marker + # based on a narrow 7-package import probe (#58004 review). + _m()._clear_update_incomplete_marker() + + # The update process is still the old Python interpreter process. Run + # one final cache/module refresh immediately before lazy backend + # refresh, which imports newly-pulled modules that may depend on fresh + # symbols in hermes_constants or lazy_deps. The dependency install + # above may also have regenerated bytecode from build-cache copies — + # this second sweep catches those stragglers (#60242, #65240). + removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) + if removed: + print( + f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" + ) + _m()._record_bytecode_fingerprint() + _m()._reload_updated_runtime_modules() + + # Upgrade pip before lazy refreshes — stale pip can fail source builds + # and leave partially-written packages (#57828). + _write_lazy_refresh_incomplete_marker() + _m()._upgrade_pip_before_lazy_refresh(install_prefix, env=lazy_env) + + # Lazy refresh can corrupt the venv when a backend install fails. + # Clear the lazy marker only when refresh/repair is confirmed healthy. + lazy_ok = _m()._refresh_active_lazy_features(install_prefix, env=lazy_env) + if lazy_ok: + _m()._clear_lazy_refresh_incomplete_marker() + else: + print( + " ⚠ Lazy-refresh recovery incomplete — run `hermes` again " + "to finish import-based venv repair." + ) + + # Heal the active memory provider's bridge packages last — the core + # reinstall + lazy refresh above may have stripped or downgraded + # plugin.yaml-declared deps that aren't in extras (#53272, #70636). + _m()._refresh_active_memory_provider_dependencies() + + node_failures = _update_node_dependencies() + _m()._build_web_ui(_m().PROJECT_ROOT / "web") + + # Rebuild the desktop app if the source tree changed since the last + # build. ``hermes desktop --build-only`` uses the content-hash stamp + # internally, so this is effectively a no-op when nothing changed. + # Only bother if the user has a desktop app installed (indicated by + # an existing packaged executable or desktop dist); people who have + # never run ``hermes desktop`` shouldn't be forced into a full + # Electron build by ``hermes update``. + desktop_dir = _m().PROJECT_ROOT / "apps" / "desktop" + has_desktop_app = _m()._desktop_packaged_executable(desktop_dir) is not None or _m()._desktop_dist_exists(desktop_dir) + if (desktop_dir / "package.json").exists() and _m()._resolve_node_runtime_npm() and has_desktop_app: + print("→ Checking if desktop app needs rebuilding...") + _desktop_build_cmd = [_m().sys.executable, "-m", "hermes_cli.main", "desktop", "--build-only"] + # Capture the (very loud) Electron/vite build output into + # update.log instead of streaming it to the terminal. On the rare + # nonzero exit, retry once after waiting again for the venv — this + # covers a still-settling rebuild window the first wait didn't fully + # catch — then surface the captured tail so the failure is + # debuggable. + # + # Start the build subprocess with the Hermes-managed Node on PATH: + # when `hermes update` runs inside the desktop updater chain + # (Desktop → hermes-setup → hermes update), the shell PATH + # customizations are lost, so a bare-PATH child would fail with + # `node: not found` before cmd_gui can self-heal. + from hermes_constants import with_hermes_node_path + + _build_env = with_hermes_node_path() + build_result = _m()._run_logged_subprocess(_desktop_build_cmd, cwd=_m().PROJECT_ROOT, env=_build_env) + if build_result.returncode != 0: + build_result = _m()._run_logged_subprocess(_desktop_build_cmd, cwd=_m().PROJECT_ROOT, env=_build_env) + if build_result.returncode != 0: + print(" ⚠ Desktop build failed (non-fatal; run `hermes desktop` to retry)") + tail = "\n".join((build_result.stdout or "").strip().splitlines()[-15:]) + if tail: + print(tail) + from hermes_constants import display_hermes_home as _dhh + print(f" Full build log: {_dhh()}/logs/update.log") + else: + print(" ✓ Desktop app up to date") + + print() + print("✓ Code updated!") + + # ── Post-update state.db integrity guard (#68474) ───────────────── + # Verify that state.db survived the update intact. If the live file + # is now corrupted (zeroed, missing header, integrity failure), + # automatically restore from the pre-update snapshot rather than + # letting the user discover silently that their sessions are gone. + try: + from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity + + _state_path = get_hermes_home() / "state.db" + if _state_path.exists(): + _state_ok = verify_sqlite_integrity( + _state_path, + check_header=True, + run_pragma=True, + ) + if _state_ok.get("valid"): + logger.debug( + "Post-update state.db integrity check: %s", + _state_ok.get("message"), + ) + else: + print() + print( + "⚠ state.db is corrupted after update: " + + _state_ok.get("message", "unknown error") + ) + _pre_snap_id = pre_update_snapshot_id + if _pre_snap_id: + _snap_state = ( + _quick_snapshot_root(get_hermes_home()) + / _pre_snap_id + / "state.db" + ) + if _snap_state.exists(): + _snap_ok = verify_sqlite_integrity( + _snap_state, check_header=True, run_pragma=True + ) + if _snap_ok.get("valid"): + try: + import shutil as _shutil + + _shutil.copy2(_snap_state, _state_path) + _restored_ok = verify_sqlite_integrity( + _state_path, + check_header=True, + run_pragma=True, + ) + if _restored_ok.get("valid"): + print( + " ✓ Auto-restored from pre-update " + f"snapshot ({_pre_snap_id})" + ) + else: + print( + " ✗ Auto-restore FAILED — restored " + "copy also failed integrity" + ) + except OSError as _exc: + print( + f" ✗ Auto-restore file copy failed: {_exc}" + ) + else: + print( + " ✗ Pre-update snapshot also failed integrity" + ) + else: + print( + " ⚠ Pre-update snapshot does not contain state.db" + ) + else: + print(" ⚠ No pre-update snapshot was taken") + print() + except Exception as exc: + logger.debug("Post-update state.db integrity check failed: %s", exc) + + # Seed the model-catalog disk cache from the freshly-pulled checkout. + # The repo ships the canonical catalog at + # website/static/api/model-catalog.json, and `git pull` just made it + # current — so copy it straight over ~/.hermes/cache/model_catalog.json + # instead of waiting on a network fetch (which can be bot-gated or hit a + # Portal hiccup). Keeps the model picker's curated/free lists in sync + # with the version the user just installed. Non-fatal on failure: the + # normal network refresh still applies on the next picker open. + try: + from hermes_cli.model_catalog import seed_cache_from_checkout + + if seed_cache_from_checkout(_m().PROJECT_ROOT): + print(" ✓ Model catalog cache refreshed from checkout") + except Exception as e: + logger.debug("Model catalog seed during update failed: %s", e) + + # Sync bundled skills (copies new, updates changed, respects user deletions) + try: + from tools.skills_sync import sync_skills + + print() + print("→ Syncing bundled skills...") + result = sync_skills(quiet=True) + if result["copied"]: + print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}") + if result.get("updated"): + print( + f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}" + ) + if result.get("user_modified"): + print(f" ~ {len(result['user_modified'])} user-modified (kept)") + print( + " → see them: hermes skills list-modified " + "(diff/reset to resume updates)" + ) + if result.get("cleaned"): + print(f" − {len(result['cleaned'])} removed from manifest") + if result.get("relocated"): + print( + f" → {len(result['relocated'])} moved to new upstream paths: " + f"{', '.join(result['relocated'])}" + ) + if not result["copied"] and not result.get("updated"): + print(" ✓ Skills are up to date") + except Exception as e: + logger.debug("Skills sync during update failed: %s", e) + + # Sync bundled skills to all profiles (including the active one). + # seed_profile_skills() uses subprocess with an explicit HERMES_HOME so + # it is not affected by sync_skills()'s module-level HERMES_HOME cache, + # which means the active profile is reliably synced regardless of whether + # the caller's HERMES_HOME env var points at the default or a named profile. + try: + from hermes_cli.profiles import ( + list_profiles, + seed_profile_skills, + ) + + all_profiles = list_profiles() + if all_profiles: + print() + print("→ Syncing bundled skills to all profiles...") + for p in all_profiles: + try: + r = seed_profile_skills(p.path, quiet=True) + if r and r.get("skipped_opt_out"): + status = "opted out (--no-skills)" + elif r: + copied = len(r.get("copied", [])) + updated = len(r.get("updated", [])) + modified = len(r.get("user_modified", [])) + parts = [] + if copied: + parts.append(f"+{copied} new") + if updated: + parts.append(f"↑{updated} updated") + if modified: + parts.append(f"~{modified} user-modified") + status = ", ".join(parts) if parts else "up to date" + else: + status = "sync failed" + print(f" {p.name}: {status}") + except Exception as pe: + print(f" {p.name}: error ({pe})") + except Exception: + pass # profiles module not available or no profiles + + # Backfill per-profile .env files for profiles created before the + # .env-seeding fix (#44792). Copies the default install's .env so + # those profiles keep the credentials they were effectively using. + try: + from hermes_cli.profiles import backfill_profile_envs + + backfilled = backfill_profile_envs(quiet=True) + if backfilled: + print() + print( + f"→ Seeded .env for {len(backfilled)} profile(s) " + f"(copied from default): {', '.join(backfilled)}" + ) + except Exception: + pass # profiles module not available or no profiles + + # Sync Honcho host blocks to all profiles + try: + from plugins.memory.honcho.cli import sync_honcho_profiles_quiet + + synced = sync_honcho_profiles_quiet() + if synced: + print(f"\n-> Honcho: synced {synced} profile(s)") + except Exception: + pass # honcho plugin not installed or not configured + + # Check for config migrations + print() + print("→ Checking configuration for new options...") + + from hermes_cli.config import ( + get_missing_env_vars, + get_missing_config_fields, + check_config_version, + migrate_config, + ) + + missing_env = get_missing_env_vars(required_only=True) + missing_config = get_missing_config_fields() + current_ver, latest_ver = check_config_version() + + has_new_options = bool(missing_env or missing_config) + version_bump_only = ( + not has_new_options and current_ver < latest_ver + ) + needs_migration = has_new_options or current_ver < latest_ver + + if version_bump_only: + # Nothing for the user to fill in — only the config format version + # changed (new defaults already merge in transparently). Asking + # "configure new options now?" here is misleading: saying yes just + # bumps the version and looks like a no-op (issue: ScottFive / + # Tt2021). Apply it silently and say what actually happened. + print() + print( + f" ℹ Updating config format (v{current_ver} → v{latest_ver})…" + ) + try: + migrate_config(interactive=False, quiet=True) + print(" ✓ Config format updated (no new settings to configure)") + except Exception as _mig_err: + print(f" ⚠️ Config format update failed: {_mig_err}") + print(" Run 'hermes config migrate' to retry.") + elif needs_migration: + print() + # Show WHAT changed, not just a count, so the user can make an + # informed yes/no decision (previously the prompt named nothing). + + if missing_env: + print( + f" ⚠️ {len(missing_env)} new required setting(s) need configuration" + ) + _print_items(missing_env, "New settings", "name") + if missing_config: + print(f" ℹ️ {len(missing_config)} new config option(s) available") + _print_items(missing_config, "New options", "key") + + print() + if assume_yes: + print( + " ℹ --yes: auto-applying config migration (skipping API-key prompts)." + ) + response = "y" + elif gateway_mode: + response = ( + _gateway_prompt( + "Would you like to configure new options now? [Y/n]", "n" + ) + .strip() + .lower() + ) + elif not (_m().sys.stdin.isatty() and _m().sys.stdout.isatty()): + print(" ℹ Non-interactive session — applying safe config migrations.") + response = "auto" + else: + try: + response = ( + input("Would you like to configure them now? [Y/n]: ") + .strip() + .lower() + ) + except EOFError: + response = "n" + + if response in {"", "y", "yes", "auto"}: + print() + # Gateway mode, --yes, and non-interactive update contexts + # (dashboard / web server actions) cannot prompt for API keys. + # Still run the non-interactive migration pass before restarting + # so new default config fields and version bumps are written + # before the freshly updated gateway validates config at startup. + interactive_migration = not ( + gateway_mode or assume_yes or response == "auto" + ) + results = migrate_config(interactive=interactive_migration, quiet=False) + + if results["env_added"] or results["config_added"]: + print() + print("✓ Configuration updated!") + if (gateway_mode or assume_yes or response == "auto") and missing_env: + print(" ℹ API keys require manual entry: hermes config migrate") + else: + print() + print("Skipped. Run 'hermes config migrate' later to configure.") + else: + print(" ✓ Configuration is up to date") + + # Safety net: config-version migrations have been observed to leave + # cron/jobs.json valid-but-empty, silently dropping every scheduled + # job (issue #34600). The desktop scheduler can also overwrite with + # its own small set, causing partial loss (issue #52144). If the + # live file now has fewer jobs than the pre-update snapshot, restore + # it and warn loudly. + try: + from hermes_cli.backup import restore_cron_jobs_if_emptied + + cron_restore = restore_cron_jobs_if_emptied(pre_update_snapshot_id) + if cron_restore: + print() + print( + " ⚠️ cron/jobs.json lost jobs during this update — " + f"restored {cron_restore['job_count']} job(s) from " + f"pre-update snapshot {cron_restore['snapshot_id']}." + ) + except Exception as exc: + # Never let the cron safety net break an otherwise-good update. + logger.debug("Cron jobs auto-restore check failed: %s", exc) + + print() + if node_failures: + print( + "⚠ Update partially complete — Node.js dependencies for " + f"{', '.join(node_failures)} did not refresh." + ) + print(" Code and Python deps are updated, but the dashboard/TUI may") + print(" be in a mixed state until the Node deps are rebuilt.") + else: + print("✓ Update complete!") + + # Search-index optimization notice (v23). Existing installs keep their + # working search index untouched on update; the compact v23 layout — + # which reclaims a large fraction of state.db on heavy users — is + # opt-in. Surface it here (the moment the user is already thinking + # about their install) with the exact command and the concrete size + # win. Show-once-ish: only when a legacy index is actually present. + try: + _print_fts_optimize_available_notice() + except Exception as e: + logger.debug("FTS optimize notice failed: %s", e) + + # Curator first-run heads-up. Only prints when curator is enabled AND + # has never run — i.e. the window where the ticker would otherwise + # have fired against a fresh skill library. Kept silent on steady + # state so we don't nag. + try: + _print_curator_first_run_notice() + except Exception as e: + logger.debug("Curator first-run notice failed: %s", e) + + # Most-recent curator run notice — show-once per run. Surfaces the + # rename map (`old-name → umbrella`) on the high-attention update + # surface so users learn about consolidations without having to + # check `hermes curator status`. Self-stamps after printing so it + # never repeats for the same run. + try: + _print_curator_recent_run_notice() + except Exception as e: + logger.debug("Curator recent-run notice failed: %s", e) + + # Repair RHEL-family root installs where /usr/local/bin isn't on PATH + # for non-login interactive shells. No-op on every other platform. + try: + _ensure_fhs_path_guard() + except Exception as e: + logger.debug("FHS PATH guard check failed: %s", e) + + # Self-heal the hermes-acp launcher for installs that predate it, so + # ACP hosts (Zed, JetBrains, Buzz) can resolve Hermes on PATH without + # a reinstall. No-op on Windows and when already present. + try: + _ensure_acp_launcher() + except Exception as e: + logger.debug("hermes-acp launcher self-heal failed: %s", e) + + # Refresh the cua-driver binary used by the Computer Use toolset. + # The upstream installer is gated on supported platforms and on the + # binary already being on PATH, so this is a no-op for users who + # don't have it. Tying the refresh to ``hermes update`` gives users a + # predictable cadence (matches when they pull new agent code) without + # adding startup latency or a per-launch GitHub API call. + try: + refresh_cua_driver = True + try: + from hermes_cli.config import load_config + + _update_cfg = (load_config() or {}).get("updates", {}) + if isinstance(_update_cfg, dict): + refresh_cua_driver = bool( + _update_cfg.get("refresh_cua_driver", True) + ) + except Exception as cfg_exc: + logger.debug("Could not read updates.refresh_cua_driver: %s", cfg_exc) + + if ( + refresh_cua_driver + and _m().sys.platform in ("darwin", "win32", "linux") + and shutil.which("cua-driver") + ): + from hermes_cli.tools_config import install_cua_driver + + print() + print("→ Refreshing cua-driver (Computer Use)...") + # require_confirmed_update: only run the (multi-minute, + # silent) upstream installer when the driver's native + # check-update verb positively reports a newer release. + # An indeterminate check (offline, rate-limited, old + # driver) keeps the installed version — `hermes update` + # must stay fast; `hermes computer-use install --upgrade` + # remains the force path. + install_cua_driver(upgrade=True, require_confirmed_update=True) + except Exception as e: + logger.debug("cua-driver refresh failed: %s", e) + + # Write exit code *before* the gateway restart attempt. + # When running as ``hermes update --gateway`` (spawned by the gateway's + # /update command), this process lives inside the gateway's systemd + # cgroup. A graceful SIGUSR1 restart keeps the drain loop alive long + # enough for the exit-code marker to be written below, but the + # fallback ``systemctl restart`` path (see below) kills everything in + # the cgroup (KillMode=mixed → SIGKILL to remaining processes), + # including us and the wrapping bash shell. The shell never reaches + # its ``printf $status > .update_exit_code`` epilogue, so the + # exit-code marker file would never be created. The new gateway's + # update watcher would then poll for 30 minutes and send a spurious + # timeout message. + # + # Writing the marker here — after git pull + pip install succeed but + # before we attempt the restart — ensures the new gateway sees it + # regardless of how we die. + if gateway_mode: + _exit_code_path = get_hermes_home() / ".update_exit_code" + try: + _exit_code_path.write_text("0", encoding="utf-8") + except OSError: + pass + + gateway_fleet_restart_incomplete = False + + # Auto-restart ALL gateways after update. + # The code update (git pull) is shared across all profiles, so every + # running gateway needs restarting to pick up the new code. + try: + from hermes_cli.gateway import ( + is_macos, + supports_systemd_services, + _ensure_user_systemd_env, + find_gateway_pids, + find_profile_gateway_processes, + _prepare_profile_gateway_update_restart, + _get_service_pids, + _graceful_restart_via_sigusr1, + _wait_for_gateway_exit, + ) + import signal as _signal + + _manage_cmd_cache: dict = {} + + def _resolve_manage_cmd(scope_: str, scope_cmd_: list, svc_name_: str): + """Resolve the command prefix for manage-units operations. + + Read-only systemctl calls (``is-active``, ``show``, + ``list-units``) work unprivileged, but manage-units verbs + (``reset-failed``, ``start``, ``restart``) on a *system* + service trigger a polkit ``org.freedesktop.systemd1.manage-units`` + authentication prompt when run as a non-root user. That + interactive prompt runs inside our captured subprocess with a + 10-15s timeout — the user sees the prompt flash and "exit + directly" before they can answer, and the resulting + TimeoutExpired used to be swallowed silently. + + Strategy: if root, plain systemctl. If not root, try + non-interactive sudo (``sudo -n``) — first a blanket probe, + then a targeted ``systemctl reset-failed`` probe so a + least-privilege sudoers entry scoped to + ``systemctl ... hermes-gateway*`` also qualifies + (``reset-failed`` is an idempotent no-op we run before every + privileged restart anyway). If neither works, return None — + the caller must SKIP the restart (without draining the + gateway first!) and tell the user how to restart manually. + ``--no-ask-password`` guarantees polkit can never hang a + captured subprocess on this path. + """ + if scope_ in _manage_cmd_cache: + return _manage_cmd_cache[scope_] + cmd = scope_cmd_ + ["--no-ask-password"] + if ( + scope_ == "system" + and hasattr(os, "geteuid") + and os.geteuid() != 0 # windows-footgun: ok — systemd path, Linux-only + ): + sudo_cmd = ["sudo", "-n"] + scope_cmd_ + ["--no-ask-password"] + sudo_ok = False + try: + _probe = subprocess.run( + ["sudo", "-n", "true"], + capture_output=True, + timeout=5, + ) + sudo_ok = _probe.returncode == 0 + if not sudo_ok: + # Blanket sudo refused — a targeted sudoers entry + # (NOPASSWD for systemctl ... hermes-gateway*) + # may still allow the exact commands we need. + _probe = subprocess.run( + sudo_cmd + ["reset-failed", svc_name_], + capture_output=True, + timeout=5, + ) + sudo_ok = _probe.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + sudo_ok = False + cmd = sudo_cmd if sudo_ok else None + _manage_cmd_cache[scope_] = cmd + return cmd + + # Drain budget for graceful SIGUSR1 restarts. The gateway drains + # for up to ``agent.restart_drain_timeout`` (default 60s) before + # exiting with code 75; we wait slightly longer so the drain + # completes before we fall back to a hard restart. On older + # systemd units without SIGUSR1 wiring this wait just times out + # and we fall back to ``systemctl restart`` (the old behaviour). + try: + from hermes_constants import ( + DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT as _DEFAULT_DRAIN, + ) + except Exception: + _DEFAULT_DRAIN = 60.0 + _cfg_drain = None + try: + from hermes_cli.config import load_config + + _cfg_agent = load_config().get("agent") or {} + _cfg_drain = _cfg_agent.get("restart_drain_timeout") + except Exception: + pass + try: + _drain_budget = ( + float(_cfg_drain) + if _cfg_drain is not None + else float(_DEFAULT_DRAIN) + ) + except (TypeError, ValueError): + _drain_budget = float(_DEFAULT_DRAIN) + # Add a 15s margin so the drain loop + final exit finish before + # we escalate to ``systemctl restart`` / SIGTERM. + _drain_budget = max(_drain_budget, 30.0) + 15.0 + + restarted_services = [] + failed_or_stale_units = [] + killed_pids = set() + relaunched_profiles = [] + externally_supervised_profiles = [] + + # --- Systemd services (Linux) --- + # Discover all hermes-gateway* units (default + profiles) + if supports_systemd_services(): + try: + _ensure_user_systemd_env() + except Exception: + pass + + for scope, scope_cmd in [ + ("user", ["systemctl", "--user"]), + ("system", ["systemctl"]), + ]: + try: + result = subprocess.run( + scope_cmd + + [ + "list-units", + "hermes-gateway*", + "--plain", + "--no-legend", + "--no-pager", + ], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=10, + ) + except FileNotFoundError: + continue + except subprocess.TimeoutExpired as exc: + # Discovery timeout — skip this scope, keep the other. + print( + f" ⚠ systemctl timed out listing {scope}-scope " + f"gateway units ({exc.cmd if exc.cmd else 'unknown command'}). " + f"Check the gateway with: hermes gateway status" + ) + continue + + def _restart_one_systemd_gateway_unit(svc_name: str) -> None: + # Check if active + check = subprocess.run( + scope_cmd + ["is-active", svc_name], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=5, + ) + if check.stdout.strip() != "active": + return + + # Resolve how we may run manage-units verbs + # (reset-failed/start/restart) for this scope. + # None ⇒ no non-interactive privilege path; we + # must avoid those verbs entirely or polkit will + # throw an interactive auth prompt inside our + # captured 10-15s subprocess (the user sees it + # flash and "exit directly" — reported June 2026). + _manage_cmd = _resolve_manage_cmd( + scope, scope_cmd, svc_name + ) + + # Prefer a graceful SIGUSR1 restart so in-flight + # agent runs drain instead of being SIGKILLed. + # The gateway's SIGUSR1 handler calls + # request_restart(via_service=True) → drain → + # exit; systemd's Restart=always respawns the unit. + _main_pid = 0 + try: + _show = subprocess.run( + scope_cmd + + [ + "show", + svc_name, + "--property=MainPID", + "--value", + ], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=5, + ) + _main_pid = int((_show.stdout or "").strip() or 0) + except ( + ValueError, + subprocess.TimeoutExpired, + FileNotFoundError, + ): + _main_pid = 0 + + _graceful_ok = False + if _main_pid > 0: + print( + f" → {svc_name}: draining (up to {int(_drain_budget)}s)..." + ) + _graceful_ok = _graceful_restart_via_sigusr1( + _main_pid, + drain_timeout=_drain_budget, + ) + + if _graceful_ok: + # Gateway exited after a planned restart. + # ``Restart=always`` means systemd WILL respawn + # the unit — but only after + # ``RestartSec`` (default 60s on our unit + # file). That 60s wait is a crash-loop guard, + # and is the right default when the gateway + # dies unexpectedly. For a voluntary restart + # on update, it's dead time the user watches. + # + # Shortcut it: ``reset-failed`` + ``start`` + # skips RestartSec entirely (we're manually + # initiating the unit, not waiting for + # systemd's auto-restart logic). Takes about + # as long as the process takes to come up + # (~1-3s on a warm box). + # + # If the unit is already active because + # RestartSec elapsed while we were draining, + # ``start`` is a no-op and we fall through to + # the poll below. Either way we collapse the + # 60s+ delay to a ~5s one. + # + # The shortcut needs manage-units privileges. + # Without them (system service, non-root, no + # passwordless sudo) skip it — systemd's own + # auto-restart still relaunches the unit after + # RestartSec, no privileges required. + if _manage_cmd is not None: + subprocess.run( + _manage_cmd + ["reset-failed", svc_name], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=10, + ) + subprocess.run( + _manage_cmd + ["start", svc_name], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=15, + ) + # Short poll: the gateway should be up + # within a few seconds now that we + # bypassed RestartSec. + if _wait_for_service_active( + scope_cmd, + svc_name, + timeout=10.0, + ): + restarted_services.append(svc_name) + return + # Passive poll: systemd's auto-restart fires + # after RestartSec regardless of privileges. + # This is the primary path when _manage_cmd is + # None, and the fallback when the explicit + # start didn't take. + _restart_sec = _service_restart_sec( + scope_cmd, + svc_name, + default=0.0, + ) + _post_drain_timeout = max( + 10.0, + _restart_sec + 10.0, + ) + if _manage_cmd is None and _restart_sec > 5.0: + print( + f" → {svc_name}: waiting for systemd " + f"auto-restart (~{int(_restart_sec)}s; " + "no root for an immediate restart)..." + ) + if _wait_for_service_active( + scope_cmd, + svc_name, + timeout=_post_drain_timeout, + ): + restarted_services.append(svc_name) + return + # Process exited but wasn't respawned (older + # unit without Restart=on-failure or + # RestartForceExitStatus=75). Fall through + # to systemctl start/restart. + print( + f" ⚠ {svc_name} drained but didn't relaunch — forcing restart" + ) + + # Forcing a restart requires manage-units + # privileges. Without a non-interactive path, + # running systemctl here would spawn a polkit + # auth prompt inside a captured 10-15s subprocess + # — it flashes and dies before the user can + # answer. Skip with clear instructions instead. + if _manage_cmd is None: + failed_or_stale_units.append(svc_name) + print( + f" ⚠ {svc_name} is a system service and restarting it needs root.\n" + f" Restart it manually to load the new version:\n" + f" sudo systemctl restart {svc_name}\n" + f" To let `hermes update` restart it automatically, allow\n" + f" passwordless sudo for systemctl, or run updates with sudo." + ) + return + + # Fallback: blunt systemctl restart. This is + # what the old code always did; we get here only + # when the graceful path failed (unit missing + # SIGUSR1 wiring, drain exceeded the budget, + # restart-policy mismatch). + # + # Always `reset-failed` first. If systemd's own + # auto-restart attempts already parked the unit + # in a failed state (transient CHDIR / OOM / + # filesystem race after our drain + exit-75), + # a plain `systemctl restart` can wedge against + # the RestartSec backoff and leave the unit + # dead. Clearing the failed state first makes + # the restart idempotent. Mirrors the recovery + # path in `hermes gateway restart` + # (`systemd_restart()`) as of PR #20949. + subprocess.run( + _manage_cmd + ["reset-failed", svc_name], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=10, + ) + restart = subprocess.run( + _manage_cmd + ["restart", svc_name], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=15, + ) + if restart.returncode == 0: + # Verify the service actually survived the + # restart. systemctl restart returns 0 even + # if the new process crashes immediately. + if _wait_for_service_active( + scope_cmd, + svc_name, + timeout=10.0, + ): + restarted_services.append(svc_name) + else: + # Retry once — transient startup failures + # (stale module cache, import race) often + # resolve on the second attempt. Again + # clear any failed state first so the + # retry isn't blocked by the previous + # crash. + print( + f" ⚠ {svc_name} died after restart, retrying..." + ) + subprocess.run( + _manage_cmd + ["reset-failed", svc_name], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=10, + ) + subprocess.run( + _manage_cmd + ["restart", svc_name], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=15, + ) + if _wait_for_service_active( + scope_cmd, + svc_name, + timeout=10.0, + ): + restarted_services.append(svc_name) + print(f" ✓ {svc_name} recovered on retry") + else: + failed_or_stale_units.append(svc_name) + _scope_flag = "--user " if scope == "user" else "" + _sudo_hint = "sudo " if scope == "system" else "" + print( + f" ✗ {svc_name} failed to stay running after restart.\n" + f" Check logs: {_sudo_hint}journalctl {_scope_flag}-u {svc_name} --since '2 min ago'\n" + f" Recover manually:\n" + f" {_sudo_hint}systemctl {_scope_flag}reset-failed {svc_name}\n" + f" {_sudo_hint}systemctl {_scope_flag}restart {svc_name}" + ) + else: + failed_or_stale_units.append(svc_name) + print( + f" ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}" + ) + + def _on_unit_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None: + # Isolate the timeout to this unit and keep going + # (#68523). A scope-wide handler used to abort every + # later gateway and leave the fleet on mixed code. + failed_or_stale_units.append(svc_name) + print( + f" ⚠ systemctl timed out restarting {svc_name} " + f"({exc.cmd if exc.cmd else 'unknown command'}); " + f"continuing with remaining gateways" + ) + + _for_each_systemd_gateway_unit( + result.stdout, + process_unit=_restart_one_systemd_gateway_unit, + on_unit_timeout=_on_unit_timeout, + ) + + # --- Launchd services (macOS) --- + if is_macos(): + try: + from hermes_cli.gateway import ( + launchd_restart, + get_launchd_label, + get_launchd_plist_path, + ) + + plist_path = get_launchd_plist_path() + if plist_path.exists(): + check = subprocess.run( + ["launchctl", "list", get_launchd_label()], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=5, + ) + if check.returncode == 0: + try: + launchd_restart() + restarted_services.append(get_launchd_label()) + except subprocess.CalledProcessError as e: + stderr = (getattr(e, "stderr", "") or "").strip() + print(f" ⚠ Gateway restart failed: {stderr}") + except (FileNotFoundError, subprocess.TimeoutExpired, ImportError): + pass + + # --- Manual (non-service) gateways --- + # Kill any remaining gateway processes not managed by a service. + # Exclude PIDs that belong to just-restarted services so we don't + # immediately kill the process that systemd/launchd just spawned. + service_pids = _get_service_pids() + manual_pids = find_gateway_pids( + exclude_pids=service_pids, all_profiles=True + ) + profile_processes = { + proc.pid: proc + for proc in find_profile_gateway_processes(exclude_pids=service_pids) + if proc.pid in manual_pids + } + for pid, proc in profile_processes.items(): + restart_mode = _prepare_profile_gateway_update_restart( + proc.profile, pid + ) + if restart_mode is None: + continue + # Prefer a graceful SIGUSR1 drain so in-flight agent runs + # finish before the watcher respawns the gateway. If the + # gateway doesn't support SIGUSR1 or doesn't exit within + # the drain budget, fall back to SIGTERM — the watcher + # still sees the exit and relaunches either way. + # Announce the drain first: this wait can hold for the full + # budget per gateway with no other output, and on surfaces + # that stream update progress (the desktop updater most of + # all) the silence reads as a hung update (#44515). + print( + f" → {proc.profile}: draining gateway PID {pid} " + f"(up to {int(_drain_budget)}s)..." + ) + drained = _graceful_restart_via_sigusr1( + pid, + drain_timeout=_drain_budget, + ) + if not drained: + try: + os.kill(pid, _signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + # Wait for the old process to fully exit before the watcher + # spawns the new gateway. Telegram holds the previous + # getUpdates long-poll session open on its servers for up to + # ~30s after the client disconnects. If the new gateway + # connects before that window expires it receives a 409 + # Conflict, which _handle_polling_conflict() recovers from + # via back-off retries — but a brief wait here reduces the + # chance of hitting that path at all, especially on fast + # machines where the watcher loop restarts in < 1s. + # We wait up to 5s for the process to exit (the OS-level + # close, not the Telegram server-side expiry), then let the + # watcher take over. The Telegram adapter's retry logic + # handles any remaining 409s if the server session is still + # live when the new gateway polls. + _wait_for_gateway_exit(timeout=5.0, force_after=None) + killed_pids.add(pid) + if restart_mode == "external-supervisor": + externally_supervised_profiles.append(proc.profile) + else: + relaunched_profiles.append(proc.profile) + + for pid in manual_pids: + if pid in profile_processes: + continue + try: + os.kill(pid, _signal.SIGTERM) + killed_pids.add(pid) + except (ProcessLookupError, PermissionError): + pass + + if restarted_services or killed_pids: + print() + for svc in restarted_services: + print(f" ✓ Restarted {svc}") + if relaunched_profiles: + names = ", ".join(relaunched_profiles) + print(f" ✓ Restarting manual gateway profile(s): {names}") + if externally_supervised_profiles: + names = ", ".join(externally_supervised_profiles) + print( + " ✓ Handed gateway profile(s) back to their external " + f"supervisor: {names}" + ) + unmapped_count = ( + len(killed_pids) + - len(relaunched_profiles) + - len(externally_supervised_profiles) + ) + if unmapped_count: + print(f" → Stopped {unmapped_count} manual gateway process(es)") + print(" Restart manually: hermes gateway run") + if unmapped_count > 1: + print( + " (or: hermes -p gateway run for each profile)" + ) + + if failed_or_stale_units: + gateway_fleet_restart_incomplete = True + if gateway_mode: + _exit_code_path = get_hermes_home() / ".update_exit_code" + try: + _exit_code_path.write_text("1", encoding="utf-8") + except OSError: + pass + _warn_incomplete_gateway_fleet_restart(failed_or_stale_units) + + if not restarted_services and not killed_pids: + # No gateways were running — nothing to do + pass + + # --- Post-restart survivor sweep ----------------------------- + # Issue #17648: some gateways ignore SIGTERM (stuck drain, + # blocked I/O, PID dead but zombie). The detached profile + # watchers wait 120s for the old PID to exit — if it never + # does, no respawn happens and the user keeps hitting + # ImportError against a stale sys.modules. Give the + # graceful paths a brief window to complete, then SIGKILL + # any remaining pre-update PIDs so the watcher / service + # manager can relaunch with fresh code. + try: + _time.sleep(3.0) + _service_pids_after = _get_service_pids() + _surviving = find_gateway_pids( + exclude_pids=_service_pids_after, + all_profiles=True, + ) + # Scope to PIDs we already tried to kill during this + # update (killed_pids). Anything new is a gateway that + # started AFTER our restart attempt — respecting user + # intent, we don't kill those. + _stuck = [pid for pid in _surviving if pid in killed_pids] + if _stuck: + print() + print( + f" ⚠ {len(_stuck)} gateway process(es) ignored SIGTERM — force-killing" + ) + from gateway.status import terminate_pid as _terminate_pid + for pid in _stuck: + try: + # Routes through taskkill /T /F on Windows, + # SIGKILL on POSIX — _signal.SIGKILL doesn't + # exist on Windows so the old raw os.kill call + # used to crash the entire update path. + _terminate_pid(pid, force=True) + except (ProcessLookupError, PermissionError, OSError): + pass + # Give the OS a beat to reap the processes so the + # watchers see them exit and respawn. + _time.sleep(1.5) + except Exception as _sweep_exc: + logger.debug("Post-restart survivor sweep failed: %s", _sweep_exc) + + except Exception as e: + logger.debug("Gateway restart during update failed: %s", e) + + _m()._resume_windows_gateways_after_update(_windows_gateway_resume) + + # Warn if legacy Hermes gateway unit files are still installed. + # When both hermes.service (from a pre-rename install) and the + # current hermes-gateway.service are enabled, they SIGTERM-fight + # for the same bot token (see PR #11909). Flagging here means + # every `hermes update` surfaces the issue until the user migrates. + try: + from hermes_cli.gateway import ( + has_legacy_hermes_units, + _find_legacy_hermes_units, + supports_systemd_services, + ) + + if supports_systemd_services() and has_legacy_hermes_units(): + print() + print("⚠ Legacy Hermes gateway unit(s) detected:") + for name, path, is_sys in _find_legacy_hermes_units(): + scope = "system" if is_sys else "user" + print(f" {path} ({scope} scope)") + print() + print(" These pre-rename units (hermes.service) fight the current") + print(" hermes-gateway.service for the bot token and cause SIGTERM") + print(" flap loops. Remove them with:") + print() + print(" hermes gateway migrate-legacy") + print() + print(" (add `sudo` if any are in system scope)") + except Exception as e: + logger.debug("Legacy unit check during update failed: %s", e) + + # Restart a managed dashboard through systemd, or stop stale manual + # dashboard processes. Raw-killing a systemd-owned dashboard PID makes + # systemd treat it as a clean stop, leaving the Cloudflare origin dead. + # Preserve the safety rule above: a failed Node refresh leaves the + # currently running dashboard untouched. + _finish_dashboard_update_cleanup(node_failures) + + print() + print("Tip: You can now select a provider and model:") + print(" hermes model # Select provider and model") + + if gateway_fleet_restart_incomplete: + # Code update itself succeeded, but at least one gateway still + # runs pre-update modules — surface that as a failed update so + # automation / operators do not treat the fleet as healthy. + _m().sys.exit(1) + + except subprocess.CalledProcessError as e: + if _m().sys.platform == "win32": + print(f"⚠ Git update failed: {e}") + print("→ Falling back to ZIP download...") + print() + _update_via_zip(args) + else: + print(f"✗ Update failed: {e}") + _m().sys.exit(1) + +# --- Hoisted from the body of _cmd_update_impl (self-contained, no closure state) --- + +def _print_items(items, label, key, fallback_key=None): + if not items: + return + print(f" {label}:") + shown = items[:8] + for it in shown: + if isinstance(it, dict): + name = it.get(key) or (fallback_key and it.get(fallback_key)) or "?" + desc = (it.get("description") or "").strip() + else: + # Defensive: some callers/mocks pass bare name strings. + name = str(it) + desc = "" + if desc: + print(f" • {name} — {desc}") + else: + print(f" • {name}") + extra = len(items) - len(shown) + if extra > 0: + print(f" … and {extra} more") + +def _wait_for_service_active( + scope_cmd_: list, + svc_name_: str, + timeout: float = 10.0, +) -> bool: + """Poll ``systemctl is-active`` until the unit reports active. + + systemd's Stopped -> Started transition after a graceful exit + (or a hard restart) is not instantaneous; a one-shot check + races that window and falsely reports the unit as down. + Poll every 0.5s up to ``timeout`` seconds before giving up. + """ + deadline = _time.monotonic() + max(timeout, 0.5) + while True: + try: + _verify = subprocess.run( + scope_cmd_ + ["is-active", svc_name_], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=5, + ) + if _verify.stdout.strip() == "active": + return True + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + if _time.monotonic() >= deadline: + return False + _time.sleep(0.5) + +def _service_restart_sec( + scope_cmd_: list, + svc_name_: str, + default: float = 0.0, +) -> float: + """Read the unit's ``RestartUSec`` (RestartSec) in seconds. + + After a graceful exit-75, systemd waits ``RestartSec`` before + respawning the unit. Callers that poll for ``is-active`` + must use a timeout >= ``RestartSec`` + transition slack, or + they'll give up *during* the cooldown window and wrongly + conclude the unit didn't relaunch. + """ + try: + _show = subprocess.run( + scope_cmd_ + + [ + "show", + svc_name_, + "--property=RestartUSec", + "--value", + ], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + timeout=5, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return default + raw = (_show.stdout or "").strip() + # systemd emits values like "30s", "100ms", "1min 30s", or + # "infinity". Parse conservatively; on any miss return default. + if not raw or raw == "infinity": + return default + total = 0.0 + matched = False + for part in raw.split(): + for _suf, _mult in ( + ("ms", 0.001), + ("us", 0.000001), + ("min", 60.0), + ("s", 1.0), + ): + if part.endswith(_suf): + try: + total += float(part[: -len(_suf)]) * _mult + matched = True + except ValueError: + pass + break + return total if matched else default diff --git a/tests/hermes_cli/test_update_modified_notice.py b/tests/hermes_cli/test_update_modified_notice.py index f29802069858c..ec5453260ac58 100644 --- a/tests/hermes_cli/test_update_modified_notice.py +++ b/tests/hermes_cli/test_update_modified_notice.py @@ -18,6 +18,7 @@ import re from pathlib import Path import hermes_cli.main as main_mod +import hermes_cli.update_cmd as update_mod _COUNT_RE = re.compile(r"user-modified \(kept\)") @@ -25,7 +26,13 @@ _HINT_RE = re.compile(r"hermes skills list-modified") def _source_lines() -> list[str]: - return Path(main_mod.__file__).read_text(encoding="utf-8").splitlines() + # The update pipeline was extracted to hermes_cli/update_cmd.py + # (main.py decomposition); scan both homes of the notice. + return [ + line + for mod in (main_mod, update_mod) + for line in Path(mod.__file__).read_text(encoding="utf-8").splitlines() + ] def test_every_user_modified_notice_points_at_list_modified():