diff --git a/manager/ffmpeg.py b/manager/ffmpeg.py index 437d9145..079398b0 100644 --- a/manager/ffmpeg.py +++ b/manager/ffmpeg.py @@ -86,7 +86,10 @@ def _mark_installed(source_url): def _install_btbn(url): - tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_") + try: + tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_", dir=REPO_ROOT) + except (OSError, FileNotFoundError): + tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_") try: archive = os.path.join(tmp, os.path.basename(url)) download(url, archive, label="ffmpeg") @@ -111,7 +114,10 @@ def _install_btbn(url): def _install_mac(detection): arch = "arm64" if detection["arch"] == "arm64" else "amd64" - tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_") + try: + tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_", dir=REPO_ROOT) + except (OSError, FileNotFoundError): + tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_") try: os.makedirs(bin_dir(), exist_ok=True) for tool in ("ffmpeg", "ffprobe"): diff --git a/manager/util.py b/manager/util.py index cc944c1c..e89f9692 100644 --- a/manager/util.py +++ b/manager/util.py @@ -179,6 +179,26 @@ def clean_env(extra=None): for var in _SCRUB_VARS: env.pop(var, None) env.setdefault("UV_PYTHON_INSTALL_DIR", os.path.join(REPO_ROOT, ".uv", "python")) + # Launchers (e.g. Stability Matrix) sometimes scrub PATH down to node/venv + # only. Put standard Unix utilities back so /bin/sh, nvidia-smi helpers, and + # npm script shells keep working. Skip on Windows — System32 is already on PATH. + if not IS_WINDOWS: + sane_path_dirs = [ + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin", + "/usr/bin", + "/sbin", + "/bin", + ] + current_path = env.get("PATH", "") + path_parts = [p for p in current_path.split(os.pathsep) if p] if current_path else [] + env["PATH"] = os.pathsep.join( + sane_path_dirs + [p for p in path_parts if p not in sane_path_dirs] + ) + env["SHELL"] = "/bin/sh" + env["npm_config_script_shell"] = "/bin/sh" + env["NPM_CONFIG_SCRIPT_SHELL"] = "/bin/sh" if extra: env.update(extra) return env diff --git a/ui/cron/fileServer.ts b/ui/cron/fileServer.ts index 9ec26cbe..28b144bf 100644 --- a/ui/cron/fileServer.ts +++ b/ui/cron/fileServer.ts @@ -514,25 +514,67 @@ function startServer(publicPort: number, upstreamPort: number): void { // request timeout would kill them mid-transfer. server.requestTimeout = 0; + server.on('error', err => { + console.error('File server failed to bind:', err); + process.exit(1); + }); + + // Cluster workers share this listen via the primary handle (do not set + // exclusive/SO_REUSEPORT here — that races with an already-bound port and + // produces endless EADDRINUSE worker respawns under Stability Matrix). server.listen(publicPort); } // --------------------------------------------------------------------------- // Supervisor: spawn Next.js on an ephemeral loopback port, then serve // --------------------------------------------------------------------------- -function getFreePort(): Promise { +function getFreePort(excludePort?: number): Promise { return new Promise((resolve, reject) => { const probe = net.createServer(); probe.on('error', reject); probe.listen(0, UPSTREAM_HOST, () => { const port = (probe.address() as net.AddressInfo).port; - probe.close(err => (err ? reject(err) : resolve(port))); + probe.close(err => { + if (err) { + reject(err); + return; + } + if (excludePort && port === excludePort) { + resolve(getFreePort(excludePort)); + return; + } + resolve(port); + }); }); }); } +/** Fail fast if another process (often an SM orphan) already owns the UI port. */ +function assertPortAvailable(port: number): Promise { + return new Promise((resolve, reject) => { + const tester = net.createServer(); + tester.once('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'EADDRINUSE') { + reject( + new Error( + `Port ${port} is already in use. Stop the leftover AI Toolkit process ` + + `(Stability Matrix Stop can leave orphans), then launch again.`, + ), + ); + } else { + reject(err); + } + }); + tester.once('listening', () => { + tester.close(closeErr => (closeErr ? reject(closeErr) : resolve())); + }); + tester.listen(port); + }); +} + async function primaryMain(): Promise { - const upstreamPort = await getFreePort(); + await assertPortAvailable(PUBLIC_PORT); + const upstreamPort = await getFreePort(PUBLIC_PORT); const nextBin = require.resolve('next/dist/bin/next'); const nextArgs = isDev ? ['dev', '--turbopack'] : ['start']; @@ -608,11 +650,22 @@ async function primaryMain(): Promise { for (let i = 0; i < numWorkers; i++) { cluster.fork(workerEnv); } - cluster.on('exit', worker => { - if (!shuttingDown) { - console.warn(`File server worker ${worker.id} died, restarting`); - cluster.fork(workerEnv); + // Cap respawns so a bind failure (EADDRINUSE) cannot fork-bomb the machine + // when Stability Matrix leaves a previous instance holding the port. + let workerDeaths = 0; + const maxWorkerDeaths = numWorkers * 3; + cluster.on('exit', (worker, code, signal) => { + if (shuttingDown) return; + workerDeaths += 1; + if (workerDeaths > maxWorkerDeaths) { + console.error( + `File server workers keep dying (last code=${code} signal=${signal}); shutting down`, + ); + shutdown(1); + return; } + console.warn(`File server worker ${worker.id} died, restarting`); + cluster.fork(workerEnv); }); } diff --git a/ui/package.json b/ui/package.json index 0b72a0b8..43ef3732 100644 --- a/ui/package.json +++ b/ui/package.json @@ -5,7 +5,8 @@ "scripts": { "dev": "concurrently -k -n WORKER,UI \"ts-node-dev --project tsconfig.worker.json --respawn --watch cron --transpile-only cron/worker.ts\" \"ts-node-dev --project tsconfig.worker.json --respawn --watch cron --transpile-only cron/fileServer.ts dev --port 3000\"", "build": "tsc -p tsconfig.worker.json && next build", - "start": "concurrently --restart-tries -1 --restart-after 1000 -n WORKER,UI \"node dist/cron/worker.js\" \"node dist/cron/fileServer.js start --port 8675\"", + "prestart": "node ./scripts/ensure-port-free.js 8675", + "start": "concurrently --restart-tries 3 --restart-after 1000 -n WORKER,UI \"node dist/cron/worker.js\" \"node dist/cron/fileServer.js start --port 8675\"", "install_deps": "npm install --no-save --no-audit --no-fund", "db_build_start": "npm run update_db && npm run build && npm run start", "build_and_start": "npm run install_deps && npm run db_build_start", @@ -54,5 +55,12 @@ "optionalDependencies": { "macstats": "^4.2.0" }, - "prettier": "prettier-basic" + "prettier": "prettier-basic", + "allowScripts": { + "sqlite3@6.0.1": true, + "prisma@6.3.1": true, + "@prisma/engines@6.3.1": true, + "@prisma/client@6.3.1": true, + "sharp@0.34.5": true + } } diff --git a/ui/scripts/ensure-port-free.js b/ui/scripts/ensure-port-free.js new file mode 100644 index 00000000..1eca5e15 --- /dev/null +++ b/ui/scripts/ensure-port-free.js @@ -0,0 +1,163 @@ +#!/usr/bin/env node +/** + * Free the AI Toolkit UI port before `npm start`. + * + * Stability Matrix's Linux stop path often leaves orphaned concurrently / + * cluster workers holding :8675, which then causes EADDRINUSE crash loops on + * the next launch. This clears listeners on the target port that belong to + * this package (or any node process bound to it). + */ +const { execSync } = require('child_process'); +const path = require('path'); + +const port = parseInt(process.argv[2] || '8675', 10); +if (!port || Number.isNaN(port)) { + console.error('usage: ensure-port-free.js '); + process.exit(1); +} + +// Windows Process Job Objects / taskkill are handled by the launcher; this +// helper targets Linux orphan trees (ss + /proc) left after a hard stop. +if (process.platform === 'win32') { + process.exit(0); +} + +const toolkitRoot = path.resolve(__dirname, '..', '..'); + +function pidsOnPort(p) { + try { + const out = execSync(`ss -ltnp '( sport = :${p} )'`, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const pids = new Set(); + for (const match of out.matchAll(/pid=(\d+)/g)) { + pids.add(match[1]); + } + return [...pids]; + } catch { + return []; + } +} + +function cmdline(pid) { + try { + return execSync(`ps -p ${pid} -o args=`, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + } catch { + return ''; + } +} + +function cwdOf(pid) { + try { + return execSync(`readlink -f /proc/${pid}/cwd`, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + } catch { + return ''; + } +} + +function belongsToToolkit(pid, args) { + if (args.includes(toolkitRoot)) return true; + const cwd = cwdOf(pid); + if (cwd === toolkitRoot || cwd.startsWith(toolkitRoot + path.sep)) return true; + // Workers often have a short argv (`node dist/cron/worker.js`) with cwd in ui/. + if ( + /dist\/cron\/(worker|fileServer)\.js|next-server|concurrently/.test(args) && + (cwd.endsWith('/ui') || cwd.includes('/Packages/ai-toolkit') || cwd.includes('/ai-toolkit')) + ) { + return true; + } + // Orphans reparented to systemd may lose a readable cwd; still match the + // canonical short argv this package uses. + if (/(?:^|[\/\s])dist\/cron\/(worker|fileServer)\.js(?:\s|$)/.test(args)) { + return true; + } + return false; +} + +function shouldKill(pid) { + const args = cmdline(pid); + if (!args) return false; + if (belongsToToolkit(pid, args)) return true; + // Last resort: any node still bound to our UI port after a failed SM stop. + return /node/.test(args); +} + +const pids = pidsOnPort(port); +if (!pids.length) { + process.exit(0); +} + +const victims = pids.filter(shouldKill); +if (!victims.length) { + console.error( + `Port ${port} is in use by non-AI-Toolkit process(es): ${pids.join(', ')}. ` + + 'Stop that process or change the UI port.', + ); + process.exit(1); +} + +console.warn(`Freeing port ${port} (stopping leftover PIDs: ${victims.join(', ')})`); +try { + execSync(`kill -TERM ${victims.join(' ')}`, { stdio: 'ignore' }); +} catch { + /* already gone */ +} +const deadline = Date.now() + 3000; +while (Date.now() < deadline && pidsOnPort(port).some(p => victims.includes(p))) { + try { + execSync('sleep 0.1', { stdio: 'ignore' }); + } catch { + break; + } +} +const still = pidsOnPort(port).filter(p => victims.includes(p) || shouldKill(p)); +if (still.length) { + try { + execSync(`kill -KILL ${still.join(' ')}`, { stdio: 'ignore' }); + } catch { + /* already gone */ + } +} + +// Also sweep orphaned workers that no longer hold the port but still thrash CPU. +try { + const all = execSync('ps -eo pid=,args=', { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const extra = []; + for (const line of all.split('\n')) { + const m = line.trim().match(/^(\d+)\s+(.*)$/); + if (!m) continue; + const [, pid, args] = m; + if (String(process.pid) === pid || String(process.ppid) === pid) continue; + if (!/dist\/cron\/(worker|fileServer)\.js|concurrently|next-server/.test(args)) { + continue; + } + if (!belongsToToolkit(pid, args)) continue; + extra.push(pid); + } + if (extra.length) { + console.warn(`Stopping leftover AI Toolkit workers: ${extra.join(', ')}`); + try { + execSync(`kill -KILL ${extra.join(' ')}`, { stdio: 'ignore' }); + } catch { + /* already gone */ + } + } +} catch { + /* ignore */ +} + +if (pidsOnPort(port).length) { + console.error(`Port ${port} is still busy after cleanup.`); + process.exit(1); +} +process.exit(0); diff --git a/ui/src/app/api/gpu/route.ts b/ui/src/app/api/gpu/route.ts index 643357e7..c75c0ea4 100644 --- a/ui/src/app/api/gpu/route.ts +++ b/ui/src/app/api/gpu/route.ts @@ -1,11 +1,68 @@ import { NextResponse } from 'next/server'; -import { exec } from 'child_process'; +import { exec, execFile, execSync } from 'child_process'; import { promisify } from 'util'; +import fs from 'fs'; import os from 'os'; +import path from 'path'; import { cached } from '@/server/apiCache'; import { loadMacstats } from '@/server/macstats'; const execAsync = promisify(exec); +const execFileAsync = promisify(execFile); + +/** + * Resolve nvidia-smi even when PATH is scrubbed (Stability Matrix / launcher + * envs often keep only node+venv dirs and drop /usr/bin). + */ +function resolveNvidiaSmi(isWindows: boolean): string | null { + const fromEnv = process.env.NVIDIA_SMI; + if (fromEnv && fs.existsSync(fromEnv)) return fromEnv; + + const candidates = isWindows + ? [ + path.join( + process.env['ProgramFiles'] || 'C:\\Program Files', + 'NVIDIA Corporation', + 'NVSMI', + 'nvidia-smi.exe', + ), + path.join( + process.env['ProgramW6432'] || 'C:\\Program Files', + 'NVIDIA Corporation', + 'NVSMI', + 'nvidia-smi.exe', + ), + ] + : ['/usr/bin/nvidia-smi', '/bin/nvidia-smi', '/usr/local/bin/nvidia-smi']; + + for (const candidate of candidates) { + try { + if (fs.existsSync(candidate)) return candidate; + } catch { + // ignore + } + } + + // Last resort: honor PATH (normal desktop installs). + try { + const whichCmd = isWindows ? 'where nvidia-smi' : 'command -v nvidia-smi'; + const stdout = execSync(whichCmd, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + env: { + ...process.env, + PATH: [process.env.PATH || '', isWindows ? '' : '/usr/local/bin:/usr/bin:/bin'] + .filter(Boolean) + .join(path.delimiter), + }, + }); + const hit = stdout.trim().split(/\r?\n/)[0]; + if (hit && fs.existsSync(hit)) return hit; + } catch { + // not found + } + return null; +} interface MacGpuResult { name: string; @@ -171,31 +228,34 @@ export async function GET() { } async function checkNvidiaSmi(isWindows: boolean): Promise { + const smi = resolveNvidiaSmi(isWindows); + if (!smi) return false; try { - if (isWindows) { - // Check if nvidia-smi is available on Windows - // It's typically located in C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi.exe - // but we'll just try to run it directly as it may be in PATH - await execAsync('nvidia-smi -L'); - } else { - // Linux/macOS check - await execAsync('which nvidia-smi'); - } + await execFileAsync(smi, ['-L'], { timeout: 8000 }); return true; - } catch (error) { + } catch { return false; } } async function getGpuStats(isWindows: boolean) { - // Command is the same for both platforms, but the path might be different - const command = - 'nvidia-smi --query-gpu=index,name,driver_version,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,power.draw,power.limit,clocks.current.graphics,clocks.current.memory,fan.speed --format=csv,noheader,nounits'; + const smi = resolveNvidiaSmi(isWindows); + if (!smi) { + throw new Error('nvidia-smi not found'); + } - // Execute command - const { stdout } = await execAsync(command, { - env: { ...process.env, CUDA_DEVICE_ORDER: 'PCI_BUS_ID' }, - }); + const { stdout } = await execFileAsync( + smi, + [ + '--query-gpu=index,name,driver_version,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,power.draw,power.limit,clocks.current.graphics,clocks.current.memory,fan.speed', + '--format=csv,noheader,nounits', + ], + { + timeout: 8000, + env: { ...process.env, CUDA_DEVICE_ORDER: 'PCI_BUS_ID' }, + encoding: 'utf8', + }, + ); // Parse CSV output const gpus = stdout