fix(dashboard): raise fd soft limit + replace iterdir with scandir to stop fd leak (#81547)
Two-part fix for the dashboard fd exhaustion reported in #81547: 1. Raise RLIMIT_NOFILE soft limit on startup (before uvicorn binds). macOS defaults to 256 for LaunchAgent processes — too tight for the dashboard which opens 3 fds (db+wal+shm) per SessionDB per request across all profiles. After days of polling the soft limit exhausts and every os.listdir/open raises OSError [Errno 24]. The helper raises to the hard limit (or minimum 4096), matching the reporter's ulimit workaround. No-op on Windows (no resource module). 2. Replace bare Path.iterdir() with context-managed os.scandir() in four dashboard hot paths: _fallback_profile_dicts, file manager list, checkpoint listing, and plugin discovery. iterdir() returns a generator that holds an open directory fd until fully consumed; if an exception interrupts iteration the fd leaks. os.scandir() is an explicit context manager that guarantees close on exit, following the same idiom already used in /api/fs/list. Tests: 6 passed, 3 skipped (resource-module tests skip on Windows).
This commit is contained in:
parent
91de3beb72
commit
373631bea1
|
|
@ -0,0 +1 @@
|
|||
a-espinoza
|
||||
|
|
@ -2384,11 +2384,12 @@ async def list_managed_files(request: Request, path: Optional[str] = None):
|
|||
raise HTTPException(status_code=400, detail="Path is not a directory")
|
||||
|
||||
try:
|
||||
entries = [
|
||||
_managed_file_entry(policy, child)
|
||||
for child in target.iterdir()
|
||||
if not _is_sensitive_path(child)
|
||||
]
|
||||
with os.scandir(target) as scan:
|
||||
entries = [
|
||||
_managed_file_entry(policy, Path(entry.path))
|
||||
for entry in scan
|
||||
if not _is_sensitive_path(Path(entry.path))
|
||||
]
|
||||
except PermissionError:
|
||||
raise HTTPException(status_code=403, detail="Directory is not readable")
|
||||
except OSError as exc:
|
||||
|
|
@ -13379,7 +13380,9 @@ async def list_checkpoints():
|
|||
sessions = []
|
||||
total_bytes = 0
|
||||
if cp_dir.is_dir():
|
||||
for child in sorted(cp_dir.iterdir()):
|
||||
with os.scandir(cp_dir) as scan:
|
||||
children = sorted((Path(e.path) for e in scan), key=lambda p: p.name)
|
||||
for child in children:
|
||||
if not child.is_dir():
|
||||
continue
|
||||
size = 0
|
||||
|
|
@ -13597,21 +13600,28 @@ def _fallback_profile_dicts(profiles_mod) -> List[Dict[str, Any]]:
|
|||
|
||||
profiles_root = profiles_mod._get_profiles_root()
|
||||
if profiles_root.is_dir():
|
||||
for entry in sorted(profiles_root.iterdir()):
|
||||
# Use os.scandir (context-managed) instead of Path.iterdir to avoid
|
||||
# leaking directory fds when an exception interrupts iteration — the
|
||||
# sidebar polls every few seconds so an fd leak exhausts RLIMIT_NOFILE
|
||||
# within days (#81547).
|
||||
with os.scandir(profiles_root) as scan:
|
||||
entries = sorted(scan, key=lambda e: e.name)
|
||||
for entry in entries:
|
||||
entry_path = Path(entry.path)
|
||||
if not entry.is_dir() or not profiles_mod._PROFILE_ID_RE.match(entry.name):
|
||||
continue
|
||||
model, provider = _safe(lambda entry=entry: profiles_mod._read_config_model(entry), (None, None))
|
||||
model, provider = _safe(lambda entry=entry_path: profiles_mod._read_config_model(entry), (None, None))
|
||||
profiles.append({
|
||||
"name": entry.name,
|
||||
"path": str(entry),
|
||||
"path": str(entry_path),
|
||||
"is_default": False,
|
||||
"model": model,
|
||||
"provider": provider,
|
||||
"has_env": (entry / ".env").exists(),
|
||||
"skill_count": _safe(lambda entry=entry: profiles_mod._count_skills(entry), 0),
|
||||
"gateway_running": _safe(lambda entry=entry: profiles_mod._check_gateway_running(entry), False),
|
||||
"description": _safe(lambda entry=entry: profiles_mod.read_profile_meta(entry).get("description", ""), ""),
|
||||
"description_auto": _safe(lambda entry=entry: profiles_mod.read_profile_meta(entry).get("description_auto", False), False),
|
||||
"has_env": (entry_path / ".env").exists(),
|
||||
"skill_count": _safe(lambda entry=entry_path: profiles_mod._count_skills(entry), 0),
|
||||
"gateway_running": _safe(lambda entry=entry_path: profiles_mod._check_gateway_running(entry), False),
|
||||
"description": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description", ""), ""),
|
||||
"description_auto": _safe(lambda entry=entry_path: profiles_mod.read_profile_meta(entry).get("description_auto", False), False),
|
||||
"distribution_name": None,
|
||||
"distribution_version": None,
|
||||
"distribution_source": None,
|
||||
|
|
@ -16881,7 +16891,9 @@ def _discover_dashboard_plugins() -> list:
|
|||
for plugins_root, source in search_dirs:
|
||||
if not plugins_root.is_dir():
|
||||
continue
|
||||
for child in sorted(plugins_root.iterdir()):
|
||||
with os.scandir(plugins_root) as scan:
|
||||
children = sorted((Path(e.path) for e in scan), key=lambda p: p.name)
|
||||
for child in children:
|
||||
if not child.is_dir():
|
||||
continue
|
||||
manifest_file = child / "dashboard" / "manifest.json"
|
||||
|
|
@ -17754,6 +17766,14 @@ def start_server(
|
|||
_apply_ssh_session_token(ssh_session_token or "")
|
||||
_apply_ssh_owner_nonce(ssh_owner_nonce)
|
||||
|
||||
# Raise RLIMIT_NOFILE for dashboard-mode starts that don't route through
|
||||
# the `serve` path in main.py (which applies the same floor). Canonical
|
||||
# policy lives in resource_limits; #81547's motivating leak (iterdir fds)
|
||||
# is fixed above, this covers legitimate high fd demand.
|
||||
from hermes_cli.resource_limits import apply_nofile_soft_limit
|
||||
|
||||
apply_nofile_soft_limit()
|
||||
|
||||
import uvicorn
|
||||
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in New Issue