feat(benchmark): live telemetry during benchmark runs (#1082)

Phase 1 of the live benchmark-run experience. Replaces the opaque (and
in sync mode, simulated) progress bar with a real-time run view driven by
actual host telemetry.

- Async run path: the UI now dispatches to the queue worker and keys off
  SSE instead of faking stage progress with client-side timers.
- New BenchmarkTelemetrySampler broadcasts per-core CPU load, CPU temp
  (best-effort, hidden when unavailable), and disk MB/s at 1 Hz over a new
  benchmark-telemetry SSE channel. Runs in the orchestration process, never
  the sysbench container, so it cannot affect scores.
- BenchmarkProgress carries the ordered stage plan + index so the frontend
  renders a live stage rail.
- AI benchmark streams /api/generate for live tokens/sec and true TTFT; the
  scored numbers still come from Ollama's authoritative final eval fields.
- Frontend: useBenchmarkRun hook owns both subscriptions; self-contained SVG
  components (StageRail, CoreGrid, Sparkline, LiveReadout) + BenchmarkRunView,
  styled in the desert palette. No chart library added.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

admin/app/services/benchmark_service.ts

@@ -21,11 +21,13 @@ import type {
   RepositorySubmission,
   RepositorySubmitResponse,
   RepositoryStats,
+  BenchmarkStageDescriptor,
 } from '../../types/benchmark.js'
 import { randomUUID, createHmac } from 'node:crypto'
 import { DockerService } from './docker_service.js'
 import { SERVICE_NAMES } from '../../constants/service_names.js'
 import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
+import { BenchmarkTelemetrySampler } from './benchmark_telemetry.js'
 import Dockerode from 'dockerode'
 
 // HMAC secret for signing submissions to the benchmark repository
@@ -67,9 +69,28 @@ const REFERENCE_SCORES = {
 export class BenchmarkService {
   private currentBenchmarkId: string | null = null
   private currentStatus: BenchmarkStatus = 'idle'
+  private currentStages: BenchmarkStageDescriptor[] = []
+  private telemetry: BenchmarkTelemetrySampler | null = null
 
   constructor(private dockerService: DockerService) {}
 
+  /**
+   * Build the ordered stage plan for a run so the live UI can render a rail of
+   * pending/active/complete nodes. Mirrors exactly the stages `_runBenchmark`
+   * emits for the given type.
+   */
+  private _buildStageList(type: BenchmarkType, includeAI: boolean): BenchmarkStageDescriptor[] {
+    const stages: BenchmarkStatus[] = ['detecting_hardware']
+    if (type === 'full' || type === 'system') {
+      stages.push('running_cpu', 'running_memory', 'running_disk_read', 'running_disk_write')
+    }
+    if (includeAI && (type === 'full' || type === 'ai')) {
+      stages.push('running_ai')
+    }
+    stages.push('calculating_score')
+    return stages.map((status) => ({ status, label: this._getStageLabel(status) }))
+  }
+
   /**
    * Run a full benchmark suite
    */
@@ -357,6 +378,14 @@ export class BenchmarkService {
     }
 
     this.currentBenchmarkId = randomUUID()
+    this.currentStages = this._buildStageList(type, includeAI)
+
+    // Start live host-telemetry sampling for the duration of the run. This runs
+    // in the orchestration process, not inside the sysbench container, so it
+    // cannot influence the scored numbers.
+    this.telemetry = new BenchmarkTelemetrySampler(this.currentBenchmarkId)
+    this.telemetry.start()
+
     this._updateStatus('starting', 'Starting benchmark...')
 
     try {
@@ -425,6 +454,10 @@ export class BenchmarkService {
       this.currentStatus = 'idle'
       this.currentBenchmarkId = null
       throw error
+    } finally {
+      this.telemetry?.stop()
+      this.telemetry = null
+      this.currentStages = []
     }
   }
 
@@ -487,49 +520,102 @@ export class BenchmarkService {
       throw new Error(`Model does not exist and failed to download: ${modelResponse.message}`)
     }
 
-    // Run inference benchmark
+    // Run inference benchmark. We stream the response so the live UI can show
+    // tokens/sec climbing and stamp the true time-to-first-token, but the SCORED
+    // numbers still come from Ollama's authoritative final eval_count /
+    // eval_duration / prompt_eval_duration, exactly as the non-streaming path did.
     const startTime = Date.now()
+    let firstTokenAt: number | null = null
+    let streamedTokens = 0
+    let responseText = ''
+    let finalEvalCount = 0
+    let finalEvalDuration = 0
+    let finalPromptEvalDuration = 0
 
-      const response = await axios.post(
+    const response = await axios.post(
-        `${ollamaAPIURL}/api/generate`,
+      `${ollamaAPIURL}/api/generate`,
-        {
+      {
-          model: AI_BENCHMARK_MODEL,
+        model: AI_BENCHMARK_MODEL,
-          prompt: AI_BENCHMARK_PROMPT,
+        prompt: AI_BENCHMARK_PROMPT,
-          stream: false,
+        stream: true,
-        },
+      },
-        { timeout: 120000 }
+      { timeout: 120000, responseType: 'stream' }
-      )
+    )
 
-      const endTime = Date.now()
+    await new Promise<void>((resolve, reject) => {
-      const totalTime = (endTime - startTime) / 1000 // seconds
+      let buffer = ''
-
+      let lastEmit = 0
-      // Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds)
+      response.data.on('data', (chunk: Buffer) => {
-      if (response.data.eval_count && response.data.eval_duration) {
+        buffer += chunk.toString('utf8')
-        const tokenCount = response.data.eval_count
+        // Ollama streams newline-delimited JSON; process each complete line.
-        const evalDurationSeconds = response.data.eval_duration / 1e9
+        let newlineIdx: number
-        const tokensPerSecond = tokenCount / evalDurationSeconds
+        while ((newlineIdx = buffer.indexOf('\n')) >= 0) {
-
+          const line = buffer.slice(0, newlineIdx).trim()
-        // Time to first token from prompt_eval_duration
+          buffer = buffer.slice(newlineIdx + 1)
-        const ttft = response.data.prompt_eval_duration
+          if (!line) continue
-          ? response.data.prompt_eval_duration / 1e6 // Convert to ms
+          try {
-          : (totalTime * 1000) / 2 // Estimate if not available
+            const obj = JSON.parse(line)
-
+            if (typeof obj.response === 'string' && obj.response.length > 0) {
-        return {
+              if (firstTokenAt === null) {
-          ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
+                firstTokenAt = Date.now()
-          ai_model_used: AI_BENCHMARK_MODEL,
+                this.telemetry?.setStageMetric('tokens_per_sec', 0, firstTokenAt - startTime)
-          ai_time_to_first_token: Math.round(ttft * 100) / 100,
+              }
+              responseText += obj.response
+              streamedTokens++
+              const now = Date.now()
+              if (firstTokenAt && now - lastEmit >= 400) {
+                const elapsed = (now - firstTokenAt) / 1000
+                const rate = elapsed > 0 ? streamedTokens / elapsed : 0
+                this.telemetry?.setStageMetric(
+                  'tokens_per_sec',
+                  Math.round(rate * 10) / 10,
+                  firstTokenAt - startTime
+                )
+                lastEmit = now
+              }
+            }
+            if (obj.done) {
+              finalEvalCount = obj.eval_count || 0
+              finalEvalDuration = obj.eval_duration || 0
+              finalPromptEvalDuration = obj.prompt_eval_duration || 0
+            }
+          } catch {
+            // Ignore a partial or non-JSON line; the next chunk completes it.
+          }
         }
-      }
+      })
+      response.data.on('end', () => resolve())
+      response.data.on('error', (err: Error) => reject(err))
+    })
 
-      // Fallback calculation
+    const totalTime = (Date.now() - startTime) / 1000 // seconds
-      const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100
+
-      const tokensPerSecond = estimatedTokens / totalTime
+    // Authoritative scored numbers (identical to the previous non-streaming path).
+    if (finalEvalCount && finalEvalDuration) {
+      const evalDurationSeconds = finalEvalDuration / 1e9
+      const tokensPerSecond = finalEvalCount / evalDurationSeconds
+      const ttft = finalPromptEvalDuration
+        ? finalPromptEvalDuration / 1e6 // Convert ns to ms
+        : (totalTime * 1000) / 2 // Estimate if not available
 
       return {
         ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
         ai_model_used: AI_BENCHMARK_MODEL,
-        ai_time_to_first_token: Math.round((totalTime * 1000) / 2),
+        ai_time_to_first_token: Math.round(ttft * 100) / 100,
       }
+    }
+
+    // Fallback if Ollama didn't return eval metrics: use the streamed counts.
+    const estimatedTokens = streamedTokens || responseText.split(' ').length * 1.3 || 100
+    const tokensPerSecond = totalTime > 0 ? estimatedTokens / totalTime : 0
+
+    return {
+      ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
+      ai_model_used: AI_BENCHMARK_MODEL,
+      ai_time_to_first_token: firstTokenAt
+        ? Math.round(firstTokenAt - startTime)
+        : Math.round((totalTime * 1000) / 2),
+    }
     } catch (error) {
       throw new Error(`AI benchmark failed: ${error.message}`)
     }
@@ -789,6 +875,7 @@ export class BenchmarkService {
    */
   private _updateStatus(status: BenchmarkStatus, message: string) {
     this.currentStatus = status
+    this.telemetry?.setStage(status)
 
     const progress: BenchmarkProgress = {
       status,
@@ -796,6 +883,9 @@ export class BenchmarkService {
       message,
       current_stage: this._getStageLabel(status),
       timestamp: new Date().toISOString(),
+      stages: this.currentStages,
+      stage_index: this.currentStages.findIndex((s) => s.status === status),
+      stage_count: this.currentStages.length,
     }
 
     transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_PROGRESS, {

admin/app/services/benchmark_telemetry.ts

@@ -0,0 +1,94 @@
+import transmit from '@adonisjs/transmit/services/main'
+import si from 'systeminformation'
+import logger from '@adonisjs/core/services/logger'
+import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
+import type { BenchmarkStatus, BenchmarkTelemetry } from '../../types/benchmark.js'
+
+const SAMPLE_INTERVAL_MS = 1000
+
+/**
+ * Samples host telemetry (per-core CPU load, temperature, disk throughput) on a
+ * timer and broadcasts it over SSE while a benchmark runs. Read-only: the sampler
+ * runs in the orchestration process, never inside the sysbench container, so it
+ * cannot influence the scored numbers.
+ *
+ * Signals come from `systeminformation`, which reads the host's /proc and /sys
+ * (not namespaced for these counters), so this works from inside a container too.
+ * Temperature legitimately isn't available on every host (VMs, some hwmon-less
+ * boards) and is reported as null rather than faked.
+ */
+export class BenchmarkTelemetrySampler {
+  private timer: NodeJS.Timeout | null = null
+  private readonly benchmarkId: string | null
+  private status: BenchmarkStatus = 'starting'
+  private startedAt = Date.now()
+  private stageMetric: BenchmarkTelemetry['stage_metric'] | undefined
+  private sampling = false
+
+  constructor(benchmarkId: string | null) {
+    this.benchmarkId = benchmarkId
+  }
+
+  start() {
+    if (this.timer) return
+    this.startedAt = Date.now()
+    // Prime systeminformation's per-second deltas and emit an initial frame,
+    // then continue on the interval.
+    void this._sample()
+    this.timer = setInterval(() => void this._sample(), SAMPLE_INTERVAL_MS)
+  }
+
+  /** Tag subsequent frames with the current stage; clears any prior in-test metric. */
+  setStage(status: BenchmarkStatus) {
+    this.status = status
+    this.stageMetric = undefined
+  }
+
+  /** Inject an in-test metric (e.g. live AI tokens/sec) into subsequent frames. */
+  setStageMetric(kind: NonNullable<BenchmarkTelemetry['stage_metric']>['kind'], value: number, ttftMs?: number) {
+    this.stageMetric = { kind, value, ...(ttftMs !== undefined ? { ttft_ms: ttftMs } : {}) }
+  }
+
+  stop() {
+    if (this.timer) {
+      clearInterval(this.timer)
+      this.timer = null
+    }
+  }
+
+  private async _sample() {
+    // Skip if the previous sample is still in flight (currentLoad can take a
+    // couple hundred ms); never let samples pile up.
+    if (this.sampling) return
+    this.sampling = true
+    try {
+      const [load, temp, fs] = await Promise.all([
+        si.currentLoad().catch(() => null),
+        si.cpuTemperature().catch(() => null),
+        si.fsStats().catch(() => null),
+      ])
+
+      const overall = load ? Math.max(0, Math.round(load.currentLoad)) : 0
+      const perCore = load?.cpus?.map((c) => Math.max(0, Math.round(c.load))) ?? []
+      const tempC = temp && typeof temp.main === 'number' && temp.main > 0 ? Math.round(temp.main) : null
+      const readMb = fs && typeof fs.rx_sec === 'number' && fs.rx_sec >= 0 ? Number((fs.rx_sec / 1e6).toFixed(1)) : 0
+      const writeMb = fs && typeof fs.wx_sec === 'number' && fs.wx_sec >= 0 ? Number((fs.wx_sec / 1e6).toFixed(1)) : 0
+
+      const payload: BenchmarkTelemetry = {
+        benchmark_id: this.benchmarkId,
+        status: this.status,
+        t: Date.now() - this.startedAt,
+        cpu: { overall, per_core: perCore },
+        temp_c: tempC,
+        disk: { read_mb_s: readMb, write_mb_s: writeMb },
+        ...(this.stageMetric ? { stage_metric: this.stageMetric } : {}),
+      }
+
+      transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_TELEMETRY, payload)
+    } catch (err) {
+      logger.debug(`[BenchmarkTelemetry] sample failed: ${err.message}`)
+    } finally {
+      this.sampling = false
+    }
+  }
+}

admin/constants/broadcast.ts

@@ -1,6 +1,7 @@
 
 export const BROADCAST_CHANNELS = {
     BENCHMARK_PROGRESS: 'benchmark-progress',
+    BENCHMARK_TELEMETRY: 'benchmark-telemetry',
     OLLAMA_MODEL_DOWNLOAD: 'ollama-model-download',
     SERVICE_INSTALLATION: 'service-installation',
     SERVICE_UPDATES: 'service-updates',

admin/inertia/components/benchmark/BenchmarkRunView.tsx

@@ -0,0 +1,167 @@
+import { useEffect, useRef, useState } from 'react'
+import { IconCpu, IconServer, IconRobot, IconTemperature, IconDatabase, IconChartBar } from '@tabler/icons-react'
+import StageRail from './StageRail'
+import CoreGrid from './CoreGrid'
+import Sparkline from './Sparkline'
+import LiveReadout from './LiveReadout'
+import type { BenchmarkRunHook } from '~/hooks/useBenchmarkRun'
+import type { BenchmarkStatus } from '../../../types/benchmark'
+
+function formatElapsed(ms: number): string {
+  const s = Math.floor(ms / 1000)
+  return `${Math.floor(s / 60)}:${(s % 60).toString().padStart(2, '0')}`
+}
+
+function StageHero({ run }: { run: BenchmarkRunHook }) {
+  const status = run.status
+  const heroCard = 'bg-desert-white rounded-lg p-6 border border-desert-stone-light h-full min-h-[240px] flex flex-col'
+  const title = 'text-sm font-semibold text-desert-green uppercase tracking-wide mb-4 flex items-center gap-2'
+
+  if (status === 'running_ai') {
+    return (
+      <div className={heroCard}>
+        <div className={title}><IconRobot className="w-4 h-4" /> AI Inference</div>
+        <div className="flex-1 flex flex-col justify-center gap-4">
+          <LiveReadout
+            value={run.aiTokensPerSec}
+            unit="tok/s"
+            label="Tokens per Second"
+            size="lg"
+            sub={run.aiTtftMs !== null ? `First token in ${Math.round(run.aiTtftMs)} ms` : 'Waiting for first token...'}
+          />
+          <div className="text-desert-green">
+            <Sparkline data={run.aiTokHistory} height={64} />
+          </div>
+        </div>
+      </div>
+    )
+  }
+
+  if (status === 'running_disk_read' || status === 'running_disk_write') {
+    const isRead = status === 'running_disk_read'
+    return (
+      <div className={heroCard}>
+        <div className={title}><IconServer className="w-4 h-4" /> {isRead ? 'Disk Read' : 'Disk Write'}</div>
+        <div className="flex-1 flex flex-col justify-center gap-4">
+          <LiveReadout
+            value={isRead ? run.diskReadMbs : run.diskWriteMbs}
+            unit="MB/s"
+            label="System disk activity"
+            size="lg"
+          />
+          <div className="text-desert-olive">
+            <Sparkline data={isRead ? run.diskReadHistory : run.diskWriteHistory} height={64} />
+          </div>
+        </div>
+      </div>
+    )
+  }
+
+  if (status === 'calculating_score') {
+    return (
+      <div className={heroCard}>
+        <div className={title}><IconChartBar className="w-4 h-4" /> Compiling Report</div>
+        <div className="flex-1 flex items-center justify-center">
+          <div className="flex items-center gap-3 text-desert-green">
+            <div className="animate-spin h-6 w-6 border-2 border-desert-green border-t-transparent rounded-full" />
+            <span className="text-lg font-medium">Calculating your NOMAD Score...</span>
+          </div>
+        </div>
+      </div>
+    )
+  }
+
+  if (status === 'detecting_hardware' || status === 'starting' || status === null) {
+    return (
+      <div className={heroCard}>
+        <div className={title}><IconCpu className="w-4 h-4" /> Identifying Hardware</div>
+        <div className="flex-1 flex items-center justify-center">
+          <div className="flex items-center gap-3 text-desert-stone-dark">
+            <div className="animate-spin h-6 w-6 border-2 border-desert-green border-t-transparent rounded-full" />
+            <span className="text-lg font-medium">Detecting your system...</span>
+          </div>
+        </div>
+      </div>
+    )
+  }
+
+  // CPU + memory stages: the core grid is the centrepiece.
+  return (
+    <div className={heroCard}>
+      <div className={title}>
+        <IconCpu className="w-4 h-4" /> {status === 'running_memory' ? 'Memory Throughput' : 'CPU Load'}
+      </div>
+      <div className="flex-1 flex flex-col justify-center">
+        <CoreGrid loads={run.perCore} />
+      </div>
+    </div>
+  )
+}
+
+/** Always-on host vitals, so something is moving between and during stages. */
+function VitalsStrip({ run }: { run: BenchmarkRunHook }) {
+  const card = 'bg-desert-white rounded-lg p-4 border border-desert-stone-light'
+  return (
+    <div className="space-y-4">
+      <div className={card}>
+        <div className="flex items-center gap-2 text-xs font-semibold text-desert-stone-dark uppercase tracking-wide mb-2">
+          <IconCpu className="w-4 h-4" /> CPU Load
+        </div>
+        <LiveReadout value={run.cpuOverall} unit="%" label="Overall" />
+        <div className="text-desert-green mt-2">
+          <Sparkline data={run.cpuHistory} height={36} max={100} />
+        </div>
+      </div>
+
+      {run.tempC !== null && (
+        <div className={card}>
+          <div className="flex items-center gap-2 text-xs font-semibold text-desert-stone-dark uppercase tracking-wide mb-2">
+            <IconTemperature className="w-4 h-4" /> Temperature
+          </div>
+          <LiveReadout value={run.tempC} unit="°C" label="CPU package" />
+        </div>
+      )}
+
+      <div className={card}>
+        <div className="flex items-center gap-2 text-xs font-semibold text-desert-stone-dark uppercase tracking-wide mb-2">
+          <IconDatabase className="w-4 h-4" /> Disk
+        </div>
+        <div className="grid grid-cols-2 gap-3">
+          <LiveReadout value={run.diskReadMbs} unit="MB/s" label="Read" />
+          <LiveReadout value={run.diskWriteMbs} unit="MB/s" label="Write" />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default function BenchmarkRunView({ run }: { run: BenchmarkRunHook }) {
+  // Elapsed clock, driven locally so it ticks even between telemetry frames.
+  const startedAt = useRef<number>(Date.now())
+  const [, force] = useState(0)
+  useEffect(() => {
+    const id = setInterval(() => force((n) => n + 1), 1000)
+    return () => clearInterval(id)
+  }, [])
+
+  const activeStatus: BenchmarkStatus | null = run.status
+
+  return (
+    <div className="space-y-6">
+      <StageRail
+        stages={run.stages}
+        stageIndex={run.stageIndex}
+        status={activeStatus}
+        progressPercent={run.progressPercent}
+        message={run.message || 'Running benchmark...'}
+        elapsedLabel={formatElapsed(Date.now() - startedAt.current)}
+      />
+      <div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
+        <div className="lg:col-span-2">
+          <StageHero run={run} />
+        </div>
+        <VitalsStrip run={run} />
+      </div>
+    </div>
+  )
+}

admin/inertia/components/benchmark/CoreGrid.tsx

@@ -0,0 +1,59 @@
+import classNames from '~/lib/classNames'
+
+interface CoreGridProps {
+  /** Per-thread load, 0-100. */
+  loads: number[]
+}
+
+const MAX_CELLS = 64
+
+// Colour ramp by load: idle olive -> warm orange -> hot red. Cells transition
+// smoothly as load climbs, so a single-thread pass lights one cell and an
+// all-threads pass lights the whole grid.
+function cellClass(load: number): string {
+  if (load >= 80) return 'bg-desert-red'
+  if (load >= 55) return 'bg-desert-orange'
+  if (load >= 25) return 'bg-desert-olive'
+  if (load >= 8) return 'bg-desert-green'
+  return 'bg-desert-green/15'
+}
+
+export default function CoreGrid({ loads }: CoreGridProps) {
+  // Fold very high core counts down to MAX_CELLS by averaging groups.
+  let cells = loads
+  let grouped = 1
+  if (loads.length > MAX_CELLS) {
+    grouped = Math.ceil(loads.length / MAX_CELLS)
+    cells = []
+    for (let i = 0; i < loads.length; i += grouped) {
+      const slice = loads.slice(i, i + grouped)
+      cells.push(slice.reduce((a, b) => a + b, 0) / slice.length)
+    }
+  }
+
+  const cols = Math.min(cells.length, 16) || 1
+
+  return (
+    <div className="space-y-2">
+      <div
+        className="grid gap-1.5"
+        style={{ gridTemplateColumns: `repeat(${cols}, minmax(0, 1fr))` }}
+      >
+        {cells.map((load, i) => (
+          <div
+            key={i}
+            title={`Thread ${i + 1}: ${Math.round(load)}%`}
+            className={classNames(
+              'aspect-square rounded-sm transition-colors duration-300',
+              cellClass(load)
+            )}
+          />
+        ))}
+      </div>
+      <div className="text-xs text-desert-stone-dark font-mono">
+        {loads.length} thread{loads.length === 1 ? '' : 's'}
+        {grouped > 1 ? ` (${grouped}/cell)` : ''}
+      </div>
+    </div>
+  )
+}

admin/inertia/components/benchmark/LiveReadout.tsx

@@ -0,0 +1,30 @@
+import classNames from '~/lib/classNames'
+
+interface LiveReadoutProps {
+  value: number | null
+  unit?: string
+  label: string
+  /** Optional secondary line, e.g. a TTFT badge. */
+  sub?: string
+  size?: 'md' | 'lg'
+  className?: string
+}
+
+/**
+ * Big monospace numeric readout for live metrics (tokens/sec, MB/s, %).
+ * Uses tabular figures so the digits don't jitter as the value updates.
+ */
+export default function LiveReadout({ value, unit, label, sub, size = 'md', className = 'text-desert-green' }: LiveReadoutProps) {
+  const display = value === null ? '--' : value >= 100 ? Math.round(value).toString() : value.toFixed(1)
+
+  return (
+    <div className="flex flex-col">
+      <div className={classNames('font-bold font-mono tabular-nums leading-none', size === 'lg' ? 'text-5xl' : 'text-3xl', className)}>
+        {display}
+        {unit && <span className="text-base font-semibold text-desert-stone-dark ml-1">{unit}</span>}
+      </div>
+      <div className="text-sm text-desert-stone-dark mt-1">{label}</div>
+      {sub && <div className="text-xs text-desert-stone-dark/80 font-mono mt-0.5">{sub}</div>}
+    </div>
+  )
+}

admin/inertia/components/benchmark/Sparkline.tsx

@@ -0,0 +1,56 @@
+interface SparklineProps {
+  data: number[]
+  height?: number
+  /** Fixed y-axis max; when unset the sparkline auto-scales to the data. */
+  max?: number
+  fill?: boolean
+  /** Tailwind text-color class drives both stroke and fill via currentColor. */
+  className?: string
+}
+
+/**
+ * Minimal self-contained SVG sparkline (no chart library). Colour comes from the
+ * parent's text color so it inherits the desert palette.
+ */
+export default function Sparkline({ data, height = 48, max, fill = true, className = 'text-desert-green' }: SparklineProps) {
+  const width = 240
+  const hi = Math.max(max ?? 0, ...data, 1)
+
+  let line = ''
+  if (data.length >= 2) {
+    line = data
+      .map((v, i) => {
+        const x = (i / (data.length - 1)) * width
+        const y = height - (Math.min(Math.max(v, 0), hi) / hi) * height
+        return `${i === 0 ? 'M' : 'L'}${x.toFixed(1)} ${y.toFixed(1)}`
+      })
+      .join(' ')
+  }
+
+  return (
+    <svg
+      viewBox={`0 0 ${width} ${height}`}
+      width="100%"
+      height={height}
+      preserveAspectRatio="none"
+      className={className}
+      role="img"
+      aria-hidden="true"
+    >
+      {fill && line && (
+        <path d={`${line} L${width} ${height} L0 ${height} Z`} className="fill-current opacity-10" stroke="none" />
+      )}
+      {line && (
+        <path
+          d={line}
+          fill="none"
+          className="stroke-current"
+          strokeWidth={2}
+          strokeLinejoin="round"
+          strokeLinecap="round"
+          vectorEffect="non-scaling-stroke"
+        />
+      )}
+    </svg>
+  )
+}

admin/inertia/components/benchmark/StageRail.tsx

@@ -0,0 +1,93 @@
+import { IconCheck } from '@tabler/icons-react'
+import classNames from '~/lib/classNames'
+import type { BenchmarkStageDescriptor, BenchmarkStatus } from '../../../types/benchmark'
+
+interface StageRailProps {
+  stages: BenchmarkStageDescriptor[]
+  stageIndex: number
+  status: BenchmarkStatus | null
+  progressPercent: number
+  message: string
+  elapsedLabel: string
+}
+
+type NodeState = 'pending' | 'active' | 'complete' | 'error'
+
+export default function StageRail({ stages, stageIndex, status, progressPercent, message, elapsedLabel }: StageRailProps) {
+  const done = status === 'completed'
+  const pct = done ? 100 : Math.min(100, Math.max(0, progressPercent))
+
+  const stateFor = (i: number): NodeState => {
+    if (done || (stageIndex >= 0 && i < stageIndex)) return 'complete'
+    if (i === stageIndex) return status === 'error' ? 'error' : 'active'
+    return 'pending'
+  }
+
+  return (
+    <div className="space-y-4">
+      <div className="overflow-x-auto">
+        <div className="flex items-start gap-1 min-w-max pb-1">
+          {stages.map((stage, i) => {
+            const state = stateFor(i)
+            const nodeClass = {
+              complete: 'bg-desert-green border-desert-green text-desert-white',
+              active: 'border-desert-green text-desert-green animate-pulse',
+              error: 'border-desert-red text-desert-red',
+              pending: 'border-desert-stone-light text-desert-stone',
+            }[state]
+            return (
+              <div key={stage.status} className="flex items-start">
+                <div className="flex flex-col items-center gap-2 w-24">
+                  <div
+                    className={classNames(
+                      'flex items-center justify-center w-8 h-8 rounded-full border-2 transition-colors',
+                      nodeClass
+                    )}
+                  >
+                    {state === 'complete' ? (
+                      <IconCheck className="w-5 h-5" />
+                    ) : (
+                      <span className="text-xs font-mono font-bold">{i + 1}</span>
+                    )}
+                  </div>
+                  <span
+                    className={classNames(
+                      'text-[11px] leading-tight text-center',
+                      state === 'active' ? 'text-desert-green font-semibold' : 'text-desert-stone-dark'
+                    )}
+                  >
+                    {stage.label}
+                  </span>
+                </div>
+                {i < stages.length - 1 && (
+                  <div
+                    className={classNames(
+                      'h-0.5 w-6 mt-4 transition-colors',
+                      state === 'complete' ? 'bg-desert-green' : 'bg-desert-stone-light'
+                    )}
+                  />
+                )}
+              </div>
+            )
+          })}
+        </div>
+      </div>
+
+      <div>
+        <div className="flex items-center justify-between text-sm text-desert-stone-dark mb-1">
+          <span>{message}</span>
+          <span className="font-mono">{elapsedLabel}</span>
+        </div>
+        <div className="w-full bg-desert-stone-lighter rounded-full h-3 overflow-hidden">
+          <div
+            className={classNames(
+              'h-full transition-all duration-700 ease-out',
+              status === 'error' ? 'bg-desert-red' : 'bg-desert-green'
+            )}
+            style={{ width: `${pct}%` }}
+          />
+        </div>
+      </div>
+    </div>
+  )
+}

admin/inertia/hooks/useBenchmarkRun.ts

@@ -0,0 +1,124 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { useTransmit } from 'react-adonis-transmit'
+import { BROADCAST_CHANNELS } from '../../constants/broadcast'
+import type {
+  BenchmarkProgress,
+  BenchmarkTelemetry,
+  BenchmarkStatus,
+  BenchmarkStageDescriptor,
+} from '../../types/benchmark'
+
+// Rolling window length for the sparklines (~1 minute at 1 Hz).
+const RING = 60
+
+export type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string }
+
+type LiveState = {
+  perCore: number[]
+  cpuOverall: number
+  cpuHistory: number[]
+  tempC: number | null
+  diskReadMbs: number
+  diskWriteMbs: number
+  diskReadHistory: number[]
+  diskWriteHistory: number[]
+  aiTokensPerSec: number | null
+  aiTokHistory: number[]
+  aiTtftMs: number | null
+}
+
+const EMPTY_LIVE: LiveState = {
+  perCore: [],
+  cpuOverall: 0,
+  cpuHistory: [],
+  tempC: null,
+  diskReadMbs: 0,
+  diskWriteMbs: 0,
+  diskReadHistory: [],
+  diskWriteHistory: [],
+  aiTokensPerSec: null,
+  aiTokHistory: [],
+  aiTtftMs: null,
+}
+
+const pushRing = (arr: number[], v: number) => {
+  const next = arr.length >= RING ? arr.slice(arr.length - RING + 1) : arr.slice()
+  next.push(v)
+  return next
+}
+
+/**
+ * Owns the two benchmark SSE subscriptions (stage progress + high-rate
+ * telemetry) and exposes a flat, render-ready view of a live run. All history
+ * buffers are kept as fresh arrays so memoized children re-render on change.
+ */
+export function useBenchmarkRun(opts?: { onFinished?: (status: 'completed' | 'error', message: string) => void }) {
+  const { subscribe } = useTransmit()
+  const onFinishedRef = useRef(opts?.onFinished)
+  onFinishedRef.current = opts?.onFinished
+
+  const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
+  const [live, setLive] = useState<LiveState>(EMPTY_LIVE)
+
+  const reset = useCallback(() => {
+    setProgress(null)
+    setLive(EMPTY_LIVE)
+  }, [])
+
+  useEffect(() => {
+    const unsubProgress = subscribe(
+      BROADCAST_CHANNELS.BENCHMARK_PROGRESS,
+      (data: BenchmarkProgressWithID) => {
+        setProgress(data)
+        if (data.status === 'completed' || data.status === 'error') {
+          onFinishedRef.current?.(data.status, data.message)
+        }
+      }
+    )
+
+    const unsubTelemetry = subscribe(
+      BROADCAST_CHANNELS.BENCHMARK_TELEMETRY,
+      (data: BenchmarkTelemetry) => {
+        setLive((prev) => {
+          const isTok = data.stage_metric?.kind === 'tokens_per_sec'
+          return {
+            perCore: data.cpu.per_core,
+            cpuOverall: data.cpu.overall,
+            cpuHistory: pushRing(prev.cpuHistory, data.cpu.overall),
+            tempC: data.temp_c,
+            diskReadMbs: data.disk.read_mb_s,
+            diskWriteMbs: data.disk.write_mb_s,
+            diskReadHistory: pushRing(prev.diskReadHistory, data.disk.read_mb_s),
+            diskWriteHistory: pushRing(prev.diskWriteHistory, data.disk.write_mb_s),
+            aiTokensPerSec: isTok ? data.stage_metric!.value : prev.aiTokensPerSec,
+            aiTokHistory: isTok ? pushRing(prev.aiTokHistory, data.stage_metric!.value) : prev.aiTokHistory,
+            aiTtftMs: isTok && data.stage_metric!.ttft_ms !== undefined ? data.stage_metric!.ttft_ms : prev.aiTtftMs,
+          }
+        })
+      }
+    )
+
+    return () => {
+      unsubProgress()
+      unsubTelemetry()
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [subscribe])
+
+  const status: BenchmarkStatus | null = progress?.status ?? null
+  const stages: BenchmarkStageDescriptor[] = progress?.stages ?? []
+  const stageIndex = progress?.stage_index ?? -1
+
+  return {
+    progress,
+    status,
+    stages,
+    stageIndex,
+    message: progress?.message ?? '',
+    progressPercent: progress?.progress ?? 0,
+    ...live,
+    reset,
+  }
+}
+
+export type BenchmarkRunHook = ReturnType<typeof useBenchmarkRun>

admin/inertia/pages/settings/benchmark.tsx

@@ -1,5 +1,5 @@
 import { Head, Link, usePage } from '@inertiajs/react'
-import { useState, useEffect, useRef } from 'react'
+import { useState } from 'react'
 import SettingsLayout from '~/layouts/SettingsLayout'
 import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
 import CircularGauge from '~/components/systeminfo/CircularGauge'
@@ -17,15 +17,13 @@ import {
   IconChevronDown,
   IconClock,
 } from '@tabler/icons-react'
-import { useTransmit } from 'react-adonis-transmit'
+import { BenchmarkStatus } from '../../../types/benchmark'
-import { BenchmarkProgress, BenchmarkStatus } from '../../../types/benchmark'
 import BenchmarkResult from '#models/benchmark_result'
 import api from '~/lib/api'
 import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
 import { SERVICE_NAMES } from '../../../constants/service_names'
-import { BROADCAST_CHANNELS } from '../../../constants/broadcast'
+import { useBenchmarkRun } from '~/hooks/useBenchmarkRun'
-
+import BenchmarkRunView from '~/components/benchmark/BenchmarkRunView'
-type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string }
 
 export default function BenchmarkPage(props: {
   benchmark: {
@@ -35,12 +33,10 @@ export default function BenchmarkPage(props: {
   }
 }) {
   const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props
-  const { subscribe } = useTransmit()
   const queryClient = useQueryClient()
   const aiInstalled = useServiceInstalledStatus(SERVICE_NAMES.OLLAMA)
-  const [progress, setProgress] = useState<BenchmarkProgressWithID | null>(null)
   const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle')
-  const refetchLatestRef = useRef<(() => void) | null>(null)
+  const [errorMsg, setErrorMsg] = useState<string | null>(null)
   const [showDetails, setShowDetails] = useState(false)
   const [showHistory, setShowHistory] = useState(false)
   const [showAIRequiredAlert, setShowAIRequiredAlert] = useState(false)
@@ -61,7 +57,18 @@ export default function BenchmarkPage(props: {
     },
     initialData: props.benchmark.latestResult,
   })
-  refetchLatestRef.current = refetchLatest
+
+  // Live run state: owns the progress + telemetry SSE subscriptions.
+  const run = useBenchmarkRun({
+    onFinished: (status, message) => {
+      setIsRunning(false)
+      if (status === 'completed') {
+        refetchLatest()
+      } else {
+        setErrorMsg(message || 'Benchmark failed')
+      }
+    },
+  })
 
   // Fetch all benchmark results for history
   const { data: benchmarkHistory } = useQuery({
@@ -75,55 +82,26 @@ export default function BenchmarkPage(props: {
     },
   })
 
-  // Run benchmark mutation (uses sync mode by default for simpler local dev)
+  // Run benchmark mutation (async: dispatched to the queue worker; live progress
+  // and completion arrive over SSE via useBenchmarkRun).
   const runBenchmark = useMutation({
     mutationFn: async (type: 'full' | 'system' | 'ai') => {
+      setErrorMsg(null)
+      run.reset()
       setIsRunning(true)
-      setProgress({
+      return await api.runBenchmark(type)
-        status: 'starting',
-        progress: 5,
-        message: 'Starting benchmark... This takes 2-5 minutes.',
-        current_stage: 'Starting',
-        benchmark_id: '',
-        timestamp: new Date().toISOString(),
-      })
-
-      // Use sync mode - runs inline without needing Redis/queue worker
-      return await api.runBenchmark(type, true)
     },
     onSuccess: (data) => {
-      if (data?.success) {
+      // Dispatch only confirms the job started; the 'completed'/'error' SSE event
-        setProgress({
+      // drives the rest (see useBenchmarkRun's onFinished).
-          status: 'completed',
+      if (!data?.success) {
-          progress: 100,
+        setIsRunning(false)
-          message: 'Benchmark completed!',
+        setErrorMsg('Failed to start benchmark')
-          current_stage: 'Complete',
-          benchmark_id: data.benchmark_id,
-          timestamp: new Date().toISOString(),
-        })
-        refetchLatest()
-      } else {
-        setProgress({
-          status: 'error',
-          progress: 0,
-          message: 'Benchmark failed',
-          current_stage: 'Error',
-          benchmark_id: '',
-          timestamp: new Date().toISOString(),
-        })
       }
-      setIsRunning(false)
     },
     onError: (error) => {
-      setProgress({
-        status: 'error',
-        progress: 0,
-        message: error.message || 'Benchmark failed',
-        current_stage: 'Error',
-        benchmark_id: '',
-        timestamp: new Date().toISOString(),
-      })
       setIsRunning(false)
+      setErrorMsg(error.message || 'Failed to start benchmark')
     },
   })
 
@@ -204,120 +182,6 @@ export default function BenchmarkPage(props: {
     runBenchmark.mutate('full')
   }
 
-  // Simulate progress during sync benchmark (since we don't get SSE updates)
-  useEffect(() => {
-    if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return
-
-    const stages: {
-      status: BenchmarkStatus
-      progress: number
-      message: string
-      label: string
-      duration: number
-    }[] = [
-      {
-        status: 'detecting_hardware',
-        progress: 10,
-        message: 'Detecting system hardware...',
-        label: 'Detecting Hardware',
-        duration: 2000,
-      },
-      {
-        status: 'running_cpu',
-        progress: 25,
-        message: 'Running CPU benchmark (30s)...',
-        label: 'CPU Benchmark',
-        duration: 32000,
-      },
-      {
-        status: 'running_memory',
-        progress: 40,
-        message: 'Running memory benchmark...',
-        label: 'Memory Benchmark',
-        duration: 8000,
-      },
-      {
-        status: 'running_disk_read',
-        progress: 55,
-        message: 'Running disk read benchmark (30s)...',
-        label: 'Disk Read Test',
-        duration: 35000,
-      },
-      {
-        status: 'running_disk_write',
-        progress: 70,
-        message: 'Running disk write benchmark (30s)...',
-        label: 'Disk Write Test',
-        duration: 35000,
-      },
-      {
-        status: 'downloading_ai_model',
-        progress: 80,
-        message: 'Downloading AI benchmark model (first run only)...',
-        label: 'Downloading AI Model',
-        duration: 5000,
-      },
-      {
-        status: 'running_ai',
-        progress: 85,
-        message: 'Running AI inference benchmark...',
-        label: 'AI Inference Test',
-        duration: 15000,
-      },
-      {
-        status: 'calculating_score',
-        progress: 95,
-        message: 'Calculating NOMAD score...',
-        label: 'Calculating Score',
-        duration: 2000,
-      },
-    ]
-
-    let currentStage = 0
-    const advanceStage = () => {
-      if (currentStage < stages.length && isRunning) {
-        const stage = stages[currentStage]
-        setProgress({
-          status: stage.status,
-          progress: stage.progress,
-          message: stage.message,
-          current_stage: stage.label,
-          benchmark_id: '',
-          timestamp: new Date().toISOString(),
-        })
-        currentStage++
-      }
-    }
-
-    // Start the first stage after a short delay
-    const timers: NodeJS.Timeout[] = []
-    let elapsed = 1000
-    stages.forEach((stage) => {
-      timers.push(setTimeout(() => advanceStage(), elapsed))
-      elapsed += stage.duration
-    })
-
-    return () => {
-      timers.forEach((t) => clearTimeout(t))
-    }
-  }, [isRunning])
-
-  // Listen for benchmark progress via SSE (backup for async mode)
-  useEffect(() => {
-    const unsubscribe = subscribe(BROADCAST_CHANNELS.BENCHMARK_PROGRESS, (data: BenchmarkProgressWithID) => {
-      setProgress(data)
-      if (data.status === 'completed' || data.status === 'error') {
-        setIsRunning(false)
-        refetchLatestRef.current?.()
-      }
-    })
-
-    return () => {
-      unsubscribe()
-    }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [subscribe])
-
   const formatBytes = (bytes: number) => {
     const gb = bytes / (1024 * 1024 * 1024)
     return `${gb.toFixed(1)} GB`
@@ -329,25 +193,6 @@ export default function BenchmarkPage(props: {
     return 'text-red-600'
   }
 
-  const getProgressPercent = () => {
-    if (!progress) return 0
-    const stages: Record<BenchmarkStatus, number> = {
-      idle: 0,
-      starting: 5,
-      detecting_hardware: 10,
-      running_cpu: 25,
-      running_memory: 40,
-      running_disk_read: 55,
-      running_disk_write: 70,
-      downloading_ai_model: 80,
-      running_ai: 85,
-      calculating_score: 95,
-      completed: 100,
-      error: 0,
-    }
-    return stages[progress.status] || 0
-  }
-
   // Calculate AI score from tokens per second (normalized to 0-100)
   // Reference: 30 tok/s = 50 score, 60 tok/s = 100 score
   const getAIScore = (tokensPerSecond: number | null): number => {
@@ -375,33 +220,19 @@ export default function BenchmarkPage(props: {
               Run Benchmark
             </h2>
 
-            <div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm">
+            {isRunning ? (
-              {isRunning ? (
+              <BenchmarkRunView run={run} />
-                <div className="space-y-4">
+            ) : (
-                  <div className="flex items-center gap-3">
+              <div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm">
-                    <div className="animate-spin h-6 w-6 border-2 border-desert-green border-t-transparent rounded-full" />
-                    <span className="text-lg font-medium">
-                      {progress?.current_stage || 'Running benchmark...'}
-                    </span>
-                  </div>
-                  <div className="w-full bg-desert-stone-lighter rounded-full h-4 overflow-hidden">
-                    <div
-                      className="bg-desert-green h-full transition-all duration-500"
-                      style={{ width: `${getProgressPercent()}%` }}
-                    />
-                  </div>
-                  <p className="text-sm text-desert-stone-dark">{progress?.message}</p>
-                </div>
-              ) : (
                 <div className="space-y-6">
-                  {progress?.status === 'error' && (
+                  {errorMsg && (
                     <Alert
                       type="error"
                       title="Benchmark Failed"
-                      message={progress.message}
+                      message={errorMsg}
                       variant="bordered"
                       dismissible
-                      onDismiss={() => setProgress(null)}
+                      onDismiss={() => setErrorMsg(null)}
                     />
                   )}
                   {showAIRequiredAlert && (
@@ -469,8 +300,8 @@ export default function BenchmarkPage(props: {
                     </p>
                   )}
                 </div>
-              )}
+              </div>
-            </div>
+            )}
           </section>
 
           {/* Results Section */}

admin/types/benchmark.ts

@@ -64,6 +64,20 @@ export type BenchmarkSettings = {
   last_benchmark_run: string | null
 }
 
+// A single stage in the ordered run plan (drives the frontend stage rail)
+export type BenchmarkStageDescriptor = {
+  status: BenchmarkStatus
+  label: string
+}
+
+// The raw metric produced when a stage finishes, surfaced to the live UI
+export type BenchmarkPartialResult = {
+  status: BenchmarkStatus
+  label: string
+  value: number
+  unit: string
+}
+
 // Progress update for real-time feedback
 export type BenchmarkProgress = {
   status: BenchmarkStatus
@@ -71,6 +85,35 @@ export type BenchmarkProgress = {
   message: string
   current_stage: string
   timestamp: string
+  // The ordered stage plan for this run + where we are in it. Optional so old
+  // clients / payloads without these fields still render.
+  stages?: BenchmarkStageDescriptor[]
+  stage_index?: number
+  stage_count?: number
+  // Raw result of the stage that just completed (fills the "results so far" strip)
+  partial_result?: BenchmarkPartialResult
+}
+
+// High-rate live telemetry sample broadcast during a run (1-2 Hz)
+export type BenchmarkTelemetry = {
+  benchmark_id: string | null
+  status: BenchmarkStatus
+  t: number // ms since run start
+  cpu: {
+    overall: number // 0-100
+    per_core: number[] // 0-100 per host thread
+  }
+  temp_c: number | null // null when host sensors are unavailable
+  disk: {
+    read_mb_s: number
+    write_mb_s: number
+  }
+  // In-test metric injected by the active stage (e.g. live AI tokens/sec)
+  stage_metric?: {
+    kind: 'tokens_per_sec' | 'events_per_sec' | 'mib_s'
+    value: number
+    ttft_ms?: number
+  }
 }
 
 // API request types