Fix several cases of silent crashing on UI (#922)

Co-authored-by: Jaret Burkett <jaretburkett@gmail.com>

ui/cron/actions/startJob.ts

@@ -7,6 +7,12 @@ import { TOOLKIT_ROOT, getTrainingFolder, getHFToken } from '../paths';
 import { resolvePythonPath } from '../pythonPath';
 const isWindows = process.platform === 'win32';
 
+const appendJobLog = (logPath: string, message: string) => {
+  fs.appendFile(logPath, message, error => {
+    if (error) console.error('Error writing to job log:', error);
+  });
+};
+
 const startAndWatchJob = (job: Job) => {
   // starts and watches the job asynchronously
   return new Promise<void>(async (resolve, reject) => {
@@ -73,6 +79,7 @@ const startAndWatchJob = (job: Job) => {
       CUDA_DEVICE_ORDER: 'PCI_BUS_ID',
       CUDA_VISIBLE_DEVICES: `${job.gpu_ids}`,
       IS_AI_TOOLKIT_UI: '1',
+      PYTHONUNBUFFERED: '1', // write Python output immediately so it is not lost on a crash
     };
 
     // HF_TOKEN
@@ -81,10 +88,12 @@ const startAndWatchJob = (job: Job) => {
       additionalEnv.HF_TOKEN = hfToken;
     }
 
-    // Add the --log argument to the command
-    const args = [runFilePath, configPath, '--log', logPath];
+    const args = [runFilePath, configPath];
 
+    let logFd: number | null = null;
     try {
+      // Capture errors that occur before run.py can initialize file logging.
+      logFd = fs.openSync(logPath, 'a');
       let subprocess;
 
       if (isWindows) {
@@ -97,13 +106,13 @@ const startAndWatchJob = (job: Job) => {
           cwd: TOOLKIT_ROOT,
           detached: true,
           windowsHide: true,
-          stdio: 'ignore', // don't tie stdio to parent
+          stdio: ['ignore', logFd, logFd], // don't tie stdio to parent; log fd passed as stdout and stderr
         });
       } else {
         // For non-Windows platforms, fully detach and ignore stdio so it survives daemon-like
         subprocess = spawn(pythonPath, args, {
           detached: true,
-          stdio: 'ignore',
+          stdio: ['ignore', logFd, logFd], // don't tie stdio to parent; log fd passed as stdout and stderr
           env: {
             ...process.env,
             ...additionalEnv,
@@ -112,6 +121,38 @@ const startAndWatchJob = (job: Job) => {
         });
       }
 
+      // Handle failures where the child process could not be started.
+      subprocess.once('error', error => {
+        const message = `Error launching job process: ${error.message}`;
+        console.error(message);
+        appendJobLog(logPath, `${message}\n`);
+        void prisma.job
+          .update({
+            where: { id: jobID },
+            data: { status: 'error', info: message, pid: null },
+          })
+          .catch(updateError => {
+            console.error('Error updating job after process launch failure:', updateError);
+          });
+      });
+
+      // Record abnormal termination and repair jobs Python could not update itself.
+      subprocess.once('exit', (code, signal) => {
+        if (code === 0) return;
+
+        const result = signal ? `signal ${signal}` : `exit code ${code}`;
+        const message = `Job process terminated with ${result}.`;
+        appendJobLog(logPath, `\n${message}\n`);
+        void prisma.job
+          .updateMany({
+            where: { id: jobID, status: 'running' },
+            data: { status: 'error', info: message, pid: null },
+          })
+          .catch(updateError => {
+            console.error('Error updating job after abnormal process exit:', updateError);
+          });
+      });
+
       // Save the PID to the database and a file for future management (stop/inspect)
       const pid = subprocess.pid ?? null;
       if (pid != null) {
@@ -131,11 +172,12 @@ const startAndWatchJob = (job: Job) => {
         subprocess.unref();
       }
 
-      // (No stdout/stderr listeners — logging should go to --log handled by your Python)
-      // (No monitoring loop — the whole point is to let it live past this worker)
+      // The child remains independent; these listeners only record failures
+      // while the worker is alive.
     } catch (error: any) {
       // Handle any exceptions during process launch
       console.error('Error launching process:', error);
+      appendJobLog(logPath, `Error launching job process: ${error?.message || 'Unknown error'}\n`);
 
       await prisma.job.update({
         where: { id: jobID },
@@ -145,6 +187,10 @@ const startAndWatchJob = (job: Job) => {
         },
       });
       return;
+    } finally {
+      if (logFd !== null) {
+        fs.closeSync(logFd);
+      }
     }
     // Resolve the promise immediately after starting the process
     resolve();