Only pull the new log bytes from the ui to improve performance

ui/src/app/api/jobs/[jobID]/log/route.ts

@@ -22,14 +22,58 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
   const logPath = path.join(jobFolder, 'log.txt');
 
   if (!fs.existsSync(logPath)) {
-    return NextResponse.json({ log: '' });
+    return NextResponse.json({ log: '', offset: 0, reset: true });
   }
-  let log = '';
+
+  const MAX_LINES = 2000;
+  // Client sends the byte offset it has already consumed so we only return new
+  // content. `offset` omitted (or NaN) => initial load / full tail.
+  const offsetParam = request.nextUrl.searchParams.get('offset');
+  const offset = offsetParam === null ? NaN : parseInt(offsetParam, 10);
+
+  const readRange = (fd: number, start: number, end: number): string => {
+    const length = end - start;
+    if (length <= 0) return '';
+    const buffer = Buffer.alloc(length);
+    fs.readSync(fd, buffer, 0, length, start);
+    return buffer.toString('utf-8');
+  };
+
   try {
-    log = fs.readFileSync(logPath, 'utf-8');
+    const stats = fs.statSync(logPath);
+    const size = stats.size;
+    // If the client's offset is past the current end, the log was reset/truncated
+    // (e.g. a fresh run overwrote it) — fall back to a fresh tail load.
+    const isReset = Number.isNaN(offset) || offset > size;
+
+    const fd = fs.openSync(logPath, 'r');
+    try {
+      if (isReset) {
+        // Read only the tail of the file to avoid loading huge logs into memory.
+        // Assume an average line length so we grab enough bytes to cover MAX_LINES.
+        const start = Math.max(0, size - MAX_LINES * 512);
+        let log = readRange(fd, start, size);
+        // Drop a partial first line if we started mid-file.
+        if (start > 0) {
+          const newlineIdx = log.indexOf('\n');
+          if (newlineIdx !== -1) {
+            log = log.slice(newlineIdx + 1);
+          }
+        }
+        const lines = log.split('\n');
+        if (lines.length > MAX_LINES) {
+          log = lines.slice(-MAX_LINES).join('\n');
+        }
+        return NextResponse.json({ log, offset: size, reset: true });
+      }
+      // Incremental: return only the bytes appended since the last offset.
+      const log = readRange(fd, offset, size);
+      return NextResponse.json({ log, offset: size, reset: false });
+    } finally {
+      fs.closeSync(fd);
+    }
   } catch (error) {
     console.error('Error reading log file:', error);
-    log = 'Error reading log file';
+    return NextResponse.json({ log: 'Error reading log file', offset: 0, reset: true });
   }
-  return NextResponse.json({ log: log });
 }

ui/src/hooks/useJobLog.tsx

@@ -17,6 +17,9 @@ const clean = (text: string): string => {
 export default function useJobLog(jobID: string, reloadInterval: null | number = null) {
   const [log, setLog] = useState<string>('');
   const didInitialLoadRef = useRef(false);
+  // Byte offset into the log file that we've already consumed. Sent to the
+  // server so it only returns newly appended content.
+  const offsetRef = useRef<number | null>(null);
   const [status, setStatus] = useState<'idle' | 'loading' | 'success' | 'error' | 'refreshing'>('idle');
 
   const refresh = () => {
@@ -25,13 +28,19 @@ export default function useJobLog(jobID: string, reloadInterval: null | number =
       loadStatus = 'refreshing';
     }
     setStatus(loadStatus);
+    const offset = offsetRef.current;
     apiClient
-      .get(`/api/jobs/${jobID}/log`)
+      .get(`/api/jobs/${jobID}/log`, offset !== null ? { params: { offset } } : undefined)
       .then(res => res.data)
       .then(data => {
-        if (data.log) {
-          let cleanLog = clean(data.log);
+        offsetRef.current = data.offset ?? null;
+        const cleanLog = clean(data.log ?? '');
+        if (data.reset) {
+          // Log was reset/truncated (or initial load) — replace everything.
           setLog(cleanLog);
+        } else if (cleanLog) {
+          // Incremental — append only the new content.
+          setLog(prev => prev + cleanLog);
         }
         setStatus('success');
         didInitialLoadRef.current = true;
@@ -43,6 +52,10 @@ export default function useJobLog(jobID: string, reloadInterval: null | number =
   };
 
   useEffect(() => {
+    // New job — start fresh.
+    offsetRef.current = null;
+    didInitialLoadRef.current = false;
+    setLog('');
     refresh();
 
     if (reloadInterval) {