fix(dashboard): raise fd soft limit + replace iterdir with scandir to stop fd leak (#81547)

Two-part fix for the dashboard fd exhaustion reported in #81547:

1. Raise RLIMIT_NOFILE soft limit on startup (before uvicorn binds).
   macOS defaults to 256 for LaunchAgent processes — too tight for the
   dashboard which opens 3 fds (db+wal+shm) per SessionDB per request
   across all profiles. After days of polling the soft limit exhausts
   and every os.listdir/open raises OSError [Errno 24]. The helper raises
   to the hard limit (or minimum 4096), matching the reporter's ulimit
   workaround. No-op on Windows (no resource module).

2. Replace bare Path.iterdir() with context-managed os.scandir() in four
   dashboard hot paths: _fallback_profile_dicts, file manager list,
   checkpoint listing, and plugin discovery. iterdir() returns a
   generator that holds an open directory fd until fully consumed; if
   an exception interrupts iteration the fd leaks. os.scandir() is an
   explicit context manager that guarantees close on exit, following
   the same idiom already used in /api/fs/list.

Tests: 6 passed, 3 skipped (resource-module tests skip on Windows).
This commit is contained in:
Michael Gannotti 2026-08-08 04:55:41 -04:00 committed by Teknium
parent 91de3beb72
commit 373631bea1
2 changed files with 36 additions and 15 deletions

View File

@ -0,0 +1 @@
a-espinoza

View File

@ -2384,11 +2384,12 @@ async def list_managed_files(request: Request, path: Optional[str] = None):
raise HTTPException(status_code=400, detail="Path is not a directory")
try:
entries = [
_managed_file_entry(policy, child)
for child in target.iterdir()
if not _is_sensitive_path(child)
]
with os.scandir(target) as scan:
entries = [
_managed_file_entry(policy, Path(entry.path))
for entry in scan
if not _is_sensitive_path(Path(entry.path))
]
except PermissionError:
raise HTTPException(status_code=403, detail="Directory is not readable")
except OSError as exc:
@ -13379,7 +13380,9 @@ async def list_checkpoints():
sessions = []
total_bytes = 0
if cp_dir.is_dir():
for child in sorted(cp_dir.iterdir()):
with os.scandir(cp_dir) as scan:
children = sorted((Path(e.path) for e in scan), key=lambda p: p.name)
for child in children:
if not child.is_dir():
continue
size = 0
@ -13597,21 +13600,28 @@ def _fallback_profile_dicts(profiles_mod) -> List[Dict[str, Any]]:
profiles_root = profiles_mod._get_profiles_root()
if profiles_root.is_dir():
for entry in sorted(profiles_root.iterdir()):
# Use os.scandir (context-managed) instead of Path.iterdir to avoid
# leaking directory fds when an exception interrupts iteration — the
# sidebar polls every few seconds so an fd leak exhausts RLIMIT_NOFILE
# within days (#81547).
with os.scandir(profiles_root) as scan:
entries = sorted(scan, key=lambda e: e.name)
for entry in entries:
entry_path = Path(entry.path)
if not entry.is_dir() or not profiles_mod._PROFILE_ID_RE.match(entry.name):
continue
model, provider = _safe(lambda entry=entry: profiles_mod._read_config_model(entry), (None, None))
model, provider = _safe(lambda entry=entry_path: profiles_mod._read_config_model(entry), (None, None))
profiles.append({
"name": entry.name,
"path": str(entry),
"path": str(entry_path),
"is_default": False,
"model": model,
"provider": provider,
"has_env": (entry / ".env").exists(),
"skill_count": _safe(lambda entry=entry: profiles_mod._count_skills(entry), 0),
"gateway_running": _safe(lambda entry=entry: profiles_mod._check_gateway_running(entry), False),
"description": _safe(lambda entry=entry: profiles_mod.read_profile_meta(entry).get("description", ""), ""),
"description_auto": _safe(lambda entry=entry: profiles_mod.read_profile_meta(entry).get("description_auto", False), False),
"has_env": (entry_path / ".env").exists(),
"skill_count": _safe(lambda entry=entry_path: profiles_mod._count_skills(entry), 0),
"gateway_running": _safe(lambda entry=entry_path: profiles_mod._check_gateway_running(entry), False),
"description": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description", ""), ""),
"description_auto": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description_auto", False), False),
"distribution_name": None,
"distribution_version": None,
"distribution_source": None,
@ -16881,7 +16891,9 @@ def _discover_dashboard_plugins() -> list:
for plugins_root, source in search_dirs:
if not plugins_root.is_dir():
continue
for child in sorted(plugins_root.iterdir()):
with os.scandir(plugins_root) as scan:
children = sorted((Path(e.path) for e in scan), key=lambda p: p.name)
for child in children:
if not child.is_dir():
continue
manifest_file = child / "dashboard" / "manifest.json"
@ -17754,6 +17766,14 @@ def start_server(
_apply_ssh_session_token(ssh_session_token or "")
_apply_ssh_owner_nonce(ssh_owner_nonce)
# Raise RLIMIT_NOFILE for dashboard-mode starts that don't route through
# the `serve` path in main.py (which applies the same floor). Canonical
# policy lives in resource_limits; #81547's motivating leak (iterdir fds)
# is fixed above, this covers legitimate high fd demand.
from hermes_cli.resource_limits import apply_nofile_soft_limit
apply_nofile_soft_limit()
import uvicorn
try: