Merge 3869e19b23 into 0f788923ae

manager/ffmpeg.py

@@ -86,7 +86,10 @@ def _mark_installed(source_url):
 
 
 def _install_btbn(url):
-    tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_")
+    try:
+        tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_", dir=REPO_ROOT)
+    except (OSError, FileNotFoundError):
+        tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_")
     try:
         archive = os.path.join(tmp, os.path.basename(url))
         download(url, archive, label="ffmpeg")
@@ -111,7 +114,10 @@ def _install_btbn(url):
 
 def _install_mac(detection):
     arch = "arm64" if detection["arch"] == "arm64" else "amd64"
-    tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_")
+    try:
+        tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_", dir=REPO_ROOT)
+    except (OSError, FileNotFoundError):
+        tmp = tempfile.mkdtemp(prefix="aitk_ffmpeg_")
     try:
         os.makedirs(bin_dir(), exist_ok=True)
         for tool in ("ffmpeg", "ffprobe"):

manager/util.py

@@ -179,6 +179,26 @@ def clean_env(extra=None):
     for var in _SCRUB_VARS:
         env.pop(var, None)
     env.setdefault("UV_PYTHON_INSTALL_DIR", os.path.join(REPO_ROOT, ".uv", "python"))
+    # Launchers (e.g. Stability Matrix) sometimes scrub PATH down to node/venv
+    # only. Put standard Unix utilities back so /bin/sh, nvidia-smi helpers, and
+    # npm script shells keep working. Skip on Windows — System32 is already on PATH.
+    if not IS_WINDOWS:
+        sane_path_dirs = [
+            "/usr/local/sbin",
+            "/usr/local/bin",
+            "/usr/sbin",
+            "/usr/bin",
+            "/sbin",
+            "/bin",
+        ]
+        current_path = env.get("PATH", "")
+        path_parts = [p for p in current_path.split(os.pathsep) if p] if current_path else []
+        env["PATH"] = os.pathsep.join(
+            sane_path_dirs + [p for p in path_parts if p not in sane_path_dirs]
+        )
+        env["SHELL"] = "/bin/sh"
+        env["npm_config_script_shell"] = "/bin/sh"
+        env["NPM_CONFIG_SCRIPT_SHELL"] = "/bin/sh"
     if extra:
         env.update(extra)
     return env

ui/cron/fileServer.ts

@@ -514,25 +514,67 @@ function startServer(publicPort: number, upstreamPort: number): void {
   // request timeout would kill them mid-transfer.
   server.requestTimeout = 0;
 
+  server.on('error', err => {
+    console.error('File server failed to bind:', err);
+    process.exit(1);
+  });
+
+  // Cluster workers share this listen via the primary handle (do not set
+  // exclusive/SO_REUSEPORT here — that races with an already-bound port and
+  // produces endless EADDRINUSE worker respawns under Stability Matrix).
   server.listen(publicPort);
 }
 
 // ---------------------------------------------------------------------------
 // Supervisor: spawn Next.js on an ephemeral loopback port, then serve
 // ---------------------------------------------------------------------------
-function getFreePort(): Promise<number> {
+function getFreePort(excludePort?: number): Promise<number> {
   return new Promise((resolve, reject) => {
     const probe = net.createServer();
     probe.on('error', reject);
     probe.listen(0, UPSTREAM_HOST, () => {
       const port = (probe.address() as net.AddressInfo).port;
-      probe.close(err => (err ? reject(err) : resolve(port)));
+      probe.close(err => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        if (excludePort && port === excludePort) {
+          resolve(getFreePort(excludePort));
+          return;
+        }
+        resolve(port);
+      });
     });
   });
 }
 
+/** Fail fast if another process (often an SM orphan) already owns the UI port. */
+function assertPortAvailable(port: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const tester = net.createServer();
+    tester.once('error', (err: NodeJS.ErrnoException) => {
+      if (err.code === 'EADDRINUSE') {
+        reject(
+          new Error(
+            `Port ${port} is already in use. Stop the leftover AI Toolkit process ` +
+              `(Stability Matrix Stop can leave orphans), then launch again.`,
+          ),
+        );
+      } else {
+        reject(err);
+      }
+    });
+    tester.once('listening', () => {
+      tester.close(closeErr => (closeErr ? reject(closeErr) : resolve()));
+    });
+    tester.listen(port);
+  });
+}
+
 async function primaryMain(): Promise<void> {
-  const upstreamPort = await getFreePort();
+  await assertPortAvailable(PUBLIC_PORT);
+  const upstreamPort = await getFreePort(PUBLIC_PORT);
 
   const nextBin = require.resolve('next/dist/bin/next');
   const nextArgs = isDev ? ['dev', '--turbopack'] : ['start'];
@@ -608,11 +650,22 @@ async function primaryMain(): Promise<void> {
   for (let i = 0; i < numWorkers; i++) {
     cluster.fork(workerEnv);
   }
-  cluster.on('exit', worker => {
-    if (!shuttingDown) {
-      console.warn(`File server worker ${worker.id} died, restarting`);
-      cluster.fork(workerEnv);
+  // Cap respawns so a bind failure (EADDRINUSE) cannot fork-bomb the machine
+  // when Stability Matrix leaves a previous instance holding the port.
+  let workerDeaths = 0;
+  const maxWorkerDeaths = numWorkers * 3;
+  cluster.on('exit', (worker, code, signal) => {
+    if (shuttingDown) return;
+    workerDeaths += 1;
+    if (workerDeaths > maxWorkerDeaths) {
+      console.error(
+        `File server workers keep dying (last code=${code} signal=${signal}); shutting down`,
+      );
+      shutdown(1);
+      return;
     }
+    console.warn(`File server worker ${worker.id} died, restarting`);
+    cluster.fork(workerEnv);
   });
 }
 

ui/package.json

@@ -5,7 +5,8 @@
   "scripts": {
     "dev": "concurrently -k -n WORKER,UI \"ts-node-dev --project tsconfig.worker.json --respawn --watch cron --transpile-only cron/worker.ts\" \"ts-node-dev --project tsconfig.worker.json --respawn --watch cron --transpile-only cron/fileServer.ts dev --port 3000\"",
     "build": "tsc -p tsconfig.worker.json && next build",
-    "start": "concurrently --restart-tries -1 --restart-after 1000 -n WORKER,UI \"node dist/cron/worker.js\" \"node dist/cron/fileServer.js start --port 8675\"",
+    "prestart": "node ./scripts/ensure-port-free.js 8675",
+    "start": "concurrently --restart-tries 3 --restart-after 1000 -n WORKER,UI \"node dist/cron/worker.js\" \"node dist/cron/fileServer.js start --port 8675\"",
     "install_deps": "npm install --no-save --no-audit --no-fund",
     "db_build_start": "npm run update_db && npm run build && npm run start",
     "build_and_start": "npm run install_deps && npm run db_build_start",
@@ -54,5 +55,12 @@
   "optionalDependencies": {
     "macstats": "^4.2.0"
   },
-  "prettier": "prettier-basic"
+  "prettier": "prettier-basic",
+  "allowScripts": {
+    "sqlite3@6.0.1": true,
+    "prisma@6.3.1": true,
+    "@prisma/engines@6.3.1": true,
+    "@prisma/client@6.3.1": true,
+    "sharp@0.34.5": true
+  }
 }

ui/scripts/ensure-port-free.js

@@ -0,0 +1,163 @@
+#!/usr/bin/env node
+/**
+ * Free the AI Toolkit UI port before `npm start`.
+ *
+ * Stability Matrix's Linux stop path often leaves orphaned concurrently /
+ * cluster workers holding :8675, which then causes EADDRINUSE crash loops on
+ * the next launch. This clears listeners on the target port that belong to
+ * this package (or any node process bound to it).
+ */
+const { execSync } = require('child_process');
+const path = require('path');
+
+const port = parseInt(process.argv[2] || '8675', 10);
+if (!port || Number.isNaN(port)) {
+  console.error('usage: ensure-port-free.js <port>');
+  process.exit(1);
+}
+
+// Windows Process Job Objects / taskkill are handled by the launcher; this
+// helper targets Linux orphan trees (ss + /proc) left after a hard stop.
+if (process.platform === 'win32') {
+  process.exit(0);
+}
+
+const toolkitRoot = path.resolve(__dirname, '..', '..');
+
+function pidsOnPort(p) {
+  try {
+    const out = execSync(`ss -ltnp '( sport = :${p} )'`, {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    const pids = new Set();
+    for (const match of out.matchAll(/pid=(\d+)/g)) {
+      pids.add(match[1]);
+    }
+    return [...pids];
+  } catch {
+    return [];
+  }
+}
+
+function cmdline(pid) {
+  try {
+    return execSync(`ps -p ${pid} -o args=`, {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+  } catch {
+    return '';
+  }
+}
+
+function cwdOf(pid) {
+  try {
+    return execSync(`readlink -f /proc/${pid}/cwd`, {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+  } catch {
+    return '';
+  }
+}
+
+function belongsToToolkit(pid, args) {
+  if (args.includes(toolkitRoot)) return true;
+  const cwd = cwdOf(pid);
+  if (cwd === toolkitRoot || cwd.startsWith(toolkitRoot + path.sep)) return true;
+  // Workers often have a short argv (`node dist/cron/worker.js`) with cwd in ui/.
+  if (
+    /dist\/cron\/(worker|fileServer)\.js|next-server|concurrently/.test(args) &&
+    (cwd.endsWith('/ui') || cwd.includes('/Packages/ai-toolkit') || cwd.includes('/ai-toolkit'))
+  ) {
+    return true;
+  }
+  // Orphans reparented to systemd may lose a readable cwd; still match the
+  // canonical short argv this package uses.
+  if (/(?:^|[\/\s])dist\/cron\/(worker|fileServer)\.js(?:\s|$)/.test(args)) {
+    return true;
+  }
+  return false;
+}
+
+function shouldKill(pid) {
+  const args = cmdline(pid);
+  if (!args) return false;
+  if (belongsToToolkit(pid, args)) return true;
+  // Last resort: any node still bound to our UI port after a failed SM stop.
+  return /node/.test(args);
+}
+
+const pids = pidsOnPort(port);
+if (!pids.length) {
+  process.exit(0);
+}
+
+const victims = pids.filter(shouldKill);
+if (!victims.length) {
+  console.error(
+    `Port ${port} is in use by non-AI-Toolkit process(es): ${pids.join(', ')}. ` +
+      'Stop that process or change the UI port.',
+  );
+  process.exit(1);
+}
+
+console.warn(`Freeing port ${port} (stopping leftover PIDs: ${victims.join(', ')})`);
+try {
+  execSync(`kill -TERM ${victims.join(' ')}`, { stdio: 'ignore' });
+} catch {
+  /* already gone */
+}
+const deadline = Date.now() + 3000;
+while (Date.now() < deadline && pidsOnPort(port).some(p => victims.includes(p))) {
+  try {
+    execSync('sleep 0.1', { stdio: 'ignore' });
+  } catch {
+    break;
+  }
+}
+const still = pidsOnPort(port).filter(p => victims.includes(p) || shouldKill(p));
+if (still.length) {
+  try {
+    execSync(`kill -KILL ${still.join(' ')}`, { stdio: 'ignore' });
+  } catch {
+    /* already gone */
+  }
+}
+
+// Also sweep orphaned workers that no longer hold the port but still thrash CPU.
+try {
+  const all = execSync('ps -eo pid=,args=', {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'ignore'],
+  });
+  const extra = [];
+  for (const line of all.split('\n')) {
+    const m = line.trim().match(/^(\d+)\s+(.*)$/);
+    if (!m) continue;
+    const [, pid, args] = m;
+    if (String(process.pid) === pid || String(process.ppid) === pid) continue;
+    if (!/dist\/cron\/(worker|fileServer)\.js|concurrently|next-server/.test(args)) {
+      continue;
+    }
+    if (!belongsToToolkit(pid, args)) continue;
+    extra.push(pid);
+  }
+  if (extra.length) {
+    console.warn(`Stopping leftover AI Toolkit workers: ${extra.join(', ')}`);
+    try {
+      execSync(`kill -KILL ${extra.join(' ')}`, { stdio: 'ignore' });
+    } catch {
+      /* already gone */
+    }
+  }
+} catch {
+  /* ignore */
+}
+
+if (pidsOnPort(port).length) {
+  console.error(`Port ${port} is still busy after cleanup.`);
+  process.exit(1);
+}
+process.exit(0);

ui/src/app/api/gpu/route.ts

@@ -1,11 +1,68 @@
 import { NextResponse } from 'next/server';
-import { exec } from 'child_process';
+import { exec, execFile, execSync } from 'child_process';
 import { promisify } from 'util';
+import fs from 'fs';
 import os from 'os';
+import path from 'path';
 import { cached } from '@/server/apiCache';
 import { loadMacstats } from '@/server/macstats';
 
 const execAsync = promisify(exec);
+const execFileAsync = promisify(execFile);
+
+/**
+ * Resolve nvidia-smi even when PATH is scrubbed (Stability Matrix / launcher
+ * envs often keep only node+venv dirs and drop /usr/bin).
+ */
+function resolveNvidiaSmi(isWindows: boolean): string | null {
+  const fromEnv = process.env.NVIDIA_SMI;
+  if (fromEnv && fs.existsSync(fromEnv)) return fromEnv;
+
+  const candidates = isWindows
+    ? [
+        path.join(
+          process.env['ProgramFiles'] || 'C:\\Program Files',
+          'NVIDIA Corporation',
+          'NVSMI',
+          'nvidia-smi.exe',
+        ),
+        path.join(
+          process.env['ProgramW6432'] || 'C:\\Program Files',
+          'NVIDIA Corporation',
+          'NVSMI',
+          'nvidia-smi.exe',
+        ),
+      ]
+    : ['/usr/bin/nvidia-smi', '/bin/nvidia-smi', '/usr/local/bin/nvidia-smi'];
+
+  for (const candidate of candidates) {
+    try {
+      if (fs.existsSync(candidate)) return candidate;
+    } catch {
+      // ignore
+    }
+  }
+
+  // Last resort: honor PATH (normal desktop installs).
+  try {
+    const whichCmd = isWindows ? 'where nvidia-smi' : 'command -v nvidia-smi';
+    const stdout = execSync(whichCmd, {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+      env: {
+        ...process.env,
+        PATH: [process.env.PATH || '', isWindows ? '' : '/usr/local/bin:/usr/bin:/bin']
+          .filter(Boolean)
+          .join(path.delimiter),
+      },
+    });
+    const hit = stdout.trim().split(/\r?\n/)[0];
+    if (hit && fs.existsSync(hit)) return hit;
+  } catch {
+    // not found
+  }
+  return null;
+}
 
 interface MacGpuResult {
   name: string;
@@ -171,31 +228,34 @@ export async function GET() {
 }
 
 async function checkNvidiaSmi(isWindows: boolean): Promise<boolean> {
+  const smi = resolveNvidiaSmi(isWindows);
+  if (!smi) return false;
   try {
-    if (isWindows) {
-      // Check if nvidia-smi is available on Windows
-      // It's typically located in C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi.exe
-      // but we'll just try to run it directly as it may be in PATH
-      await execAsync('nvidia-smi -L');
-    } else {
-      // Linux/macOS check
-      await execAsync('which nvidia-smi');
-    }
+    await execFileAsync(smi, ['-L'], { timeout: 8000 });
     return true;
-  } catch (error) {
+  } catch {
     return false;
   }
 }
 
 async function getGpuStats(isWindows: boolean) {
-  // Command is the same for both platforms, but the path might be different
-  const command =
-    'nvidia-smi --query-gpu=index,name,driver_version,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,power.draw,power.limit,clocks.current.graphics,clocks.current.memory,fan.speed --format=csv,noheader,nounits';
+  const smi = resolveNvidiaSmi(isWindows);
+  if (!smi) {
+    throw new Error('nvidia-smi not found');
+  }
 
-  // Execute command
-  const { stdout } = await execAsync(command, {
-    env: { ...process.env, CUDA_DEVICE_ORDER: 'PCI_BUS_ID' },
-  });
+  const { stdout } = await execFileAsync(
+    smi,
+    [
+      '--query-gpu=index,name,driver_version,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,power.draw,power.limit,clocks.current.graphics,clocks.current.memory,fan.speed',
+      '--format=csv,noheader,nounits',
+    ],
+    {
+      timeout: 8000,
+      env: { ...process.env, CUDA_DEVICE_ORDER: 'PCI_BUS_ID' },
+      encoding: 'utf8',
+    },
+  );
 
   // Parse CSV output
   const gpus = stdout