From 373631bea1559e1f8e767f2eec5d897bbddd0a42 Mon Sep 17 00:00:00 2001 From: Michael Gannotti Date: Sat, 8 Aug 2026 04:55:41 -0400 Subject: [PATCH] fix(dashboard): raise fd soft limit + replace iterdir with scandir to stop fd leak (#81547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-part fix for the dashboard fd exhaustion reported in #81547: 1. Raise RLIMIT_NOFILE soft limit on startup (before uvicorn binds). macOS defaults to 256 for LaunchAgent processes — too tight for the dashboard which opens 3 fds (db+wal+shm) per SessionDB per request across all profiles. After days of polling the soft limit exhausts and every os.listdir/open raises OSError [Errno 24]. The helper raises to the hard limit (or minimum 4096), matching the reporter's ulimit workaround. No-op on Windows (no resource module). 2. Replace bare Path.iterdir() with context-managed os.scandir() in four dashboard hot paths: _fallback_profile_dicts, file manager list, checkpoint listing, and plugin discovery. iterdir() returns a generator that holds an open directory fd until fully consumed; if an exception interrupts iteration the fd leaks. os.scandir() is an explicit context manager that guarantees close on exit, following the same idiom already used in /api/fs/list. Tests: 6 passed, 3 skipped (resource-module tests skip on Windows). --- contributors/emails/a_espinosa@live.com | 1 + hermes_cli/web_server.py | 50 +++++++++++++++++-------- 2 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 contributors/emails/a_espinosa@live.com diff --git a/contributors/emails/a_espinosa@live.com b/contributors/emails/a_espinosa@live.com new file mode 100644 index 0000000000000..3b229714fb466 --- /dev/null +++ b/contributors/emails/a_espinosa@live.com @@ -0,0 +1 @@ +a-espinoza diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index f25255e65cc6b..ff4b145f4c61b 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -2384,11 +2384,12 @@ async def list_managed_files(request: Request, path: Optional[str] = None): raise HTTPException(status_code=400, detail="Path is not a directory") try: - entries = [ - _managed_file_entry(policy, child) - for child in target.iterdir() - if not _is_sensitive_path(child) - ] + with os.scandir(target) as scan: + entries = [ + _managed_file_entry(policy, Path(entry.path)) + for entry in scan + if not _is_sensitive_path(Path(entry.path)) + ] except PermissionError: raise HTTPException(status_code=403, detail="Directory is not readable") except OSError as exc: @@ -13379,7 +13380,9 @@ async def list_checkpoints(): sessions = [] total_bytes = 0 if cp_dir.is_dir(): - for child in sorted(cp_dir.iterdir()): + with os.scandir(cp_dir) as scan: + children = sorted((Path(e.path) for e in scan), key=lambda p: p.name) + for child in children: if not child.is_dir(): continue size = 0 @@ -13597,21 +13600,28 @@ def _fallback_profile_dicts(profiles_mod) -> List[Dict[str, Any]]: profiles_root = profiles_mod._get_profiles_root() if profiles_root.is_dir(): - for entry in sorted(profiles_root.iterdir()): + # Use os.scandir (context-managed) instead of Path.iterdir to avoid + # leaking directory fds when an exception interrupts iteration — the + # sidebar polls every few seconds so an fd leak exhausts RLIMIT_NOFILE + # within days (#81547). + with os.scandir(profiles_root) as scan: + entries = sorted(scan, key=lambda e: e.name) + for entry in entries: + entry_path = Path(entry.path) if not entry.is_dir() or not profiles_mod._PROFILE_ID_RE.match(entry.name): continue - model, provider = _safe(lambda entry=entry: profiles_mod._read_config_model(entry), (None, None)) + model, provider = _safe(lambda entry=entry_path: profiles_mod._read_config_model(entry), (None, None)) profiles.append({ "name": entry.name, - "path": str(entry), + "path": str(entry_path), "is_default": False, "model": model, "provider": provider, - "has_env": (entry / ".env").exists(), - "skill_count": _safe(lambda entry=entry: profiles_mod._count_skills(entry), 0), - "gateway_running": _safe(lambda entry=entry: profiles_mod._check_gateway_running(entry), False), - "description": _safe(lambda entry=entry: profiles_mod.read_profile_meta(entry).get("description", ""), ""), - "description_auto": _safe(lambda entry=entry: profiles_mod.read_profile_meta(entry).get("description_auto", False), False), + "has_env": (entry_path / ".env").exists(), + "skill_count": _safe(lambda entry=entry_path: profiles_mod._count_skills(entry), 0), + "gateway_running": _safe(lambda entry=entry_path: profiles_mod._check_gateway_running(entry), False), + "description": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description", ""), ""), + "description_auto": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description_auto", False), False), "distribution_name": None, "distribution_version": None, "distribution_source": None, @@ -16881,7 +16891,9 @@ def _discover_dashboard_plugins() -> list: for plugins_root, source in search_dirs: if not plugins_root.is_dir(): continue - for child in sorted(plugins_root.iterdir()): + with os.scandir(plugins_root) as scan: + children = sorted((Path(e.path) for e in scan), key=lambda p: p.name) + for child in children: if not child.is_dir(): continue manifest_file = child / "dashboard" / "manifest.json" @@ -17754,6 +17766,14 @@ def start_server( _apply_ssh_session_token(ssh_session_token or "") _apply_ssh_owner_nonce(ssh_owner_nonce) + # Raise RLIMIT_NOFILE for dashboard-mode starts that don't route through + # the `serve` path in main.py (which applies the same floor). Canonical + # policy lives in resource_limits; #81547's motivating leak (iterdir fds) + # is fixed above, this covers legitimate high fd demand. + from hermes_cli.resource_limits import apply_nofile_soft_limit + + apply_nofile_soft_limit() + import uvicorn try: