diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index 710b7ad..99bf420 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -21,11 +21,13 @@ import type { RepositorySubmission, RepositorySubmitResponse, RepositoryStats, + BenchmarkStageDescriptor, } from '../../types/benchmark.js' import { randomUUID, createHmac } from 'node:crypto' import { DockerService } from './docker_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' +import { BenchmarkTelemetrySampler } from './benchmark_telemetry.js' import Dockerode from 'dockerode' // HMAC secret for signing submissions to the benchmark repository @@ -67,9 +69,28 @@ const REFERENCE_SCORES = { export class BenchmarkService { private currentBenchmarkId: string | null = null private currentStatus: BenchmarkStatus = 'idle' + private currentStages: BenchmarkStageDescriptor[] = [] + private telemetry: BenchmarkTelemetrySampler | null = null constructor(private dockerService: DockerService) {} + /** + * Build the ordered stage plan for a run so the live UI can render a rail of + * pending/active/complete nodes. Mirrors exactly the stages `_runBenchmark` + * emits for the given type. + */ + private _buildStageList(type: BenchmarkType, includeAI: boolean): BenchmarkStageDescriptor[] { + const stages: BenchmarkStatus[] = ['detecting_hardware'] + if (type === 'full' || type === 'system') { + stages.push('running_cpu', 'running_memory', 'running_disk_read', 'running_disk_write') + } + if (includeAI && (type === 'full' || type === 'ai')) { + stages.push('running_ai') + } + stages.push('calculating_score') + return stages.map((status) => ({ status, label: this._getStageLabel(status) })) + } + /** * Run a full benchmark suite */ @@ -357,6 +378,14 @@ export class BenchmarkService { } this.currentBenchmarkId = randomUUID() + this.currentStages = this._buildStageList(type, includeAI) + + // Start live host-telemetry sampling for the duration of the run. This runs + // in the orchestration process, not inside the sysbench container, so it + // cannot influence the scored numbers. + this.telemetry = new BenchmarkTelemetrySampler(this.currentBenchmarkId) + this.telemetry.start() + this._updateStatus('starting', 'Starting benchmark...') try { @@ -425,6 +454,10 @@ export class BenchmarkService { this.currentStatus = 'idle' this.currentBenchmarkId = null throw error + } finally { + this.telemetry?.stop() + this.telemetry = null + this.currentStages = [] } } @@ -487,49 +520,102 @@ export class BenchmarkService { throw new Error(`Model does not exist and failed to download: ${modelResponse.message}`) } - // Run inference benchmark + // Run inference benchmark. We stream the response so the live UI can show + // tokens/sec climbing and stamp the true time-to-first-token, but the SCORED + // numbers still come from Ollama's authoritative final eval_count / + // eval_duration / prompt_eval_duration, exactly as the non-streaming path did. const startTime = Date.now() + let firstTokenAt: number | null = null + let streamedTokens = 0 + let responseText = '' + let finalEvalCount = 0 + let finalEvalDuration = 0 + let finalPromptEvalDuration = 0 - const response = await axios.post( - `${ollamaAPIURL}/api/generate`, - { - model: AI_BENCHMARK_MODEL, - prompt: AI_BENCHMARK_PROMPT, - stream: false, - }, - { timeout: 120000 } - ) + const response = await axios.post( + `${ollamaAPIURL}/api/generate`, + { + model: AI_BENCHMARK_MODEL, + prompt: AI_BENCHMARK_PROMPT, + stream: true, + }, + { timeout: 120000, responseType: 'stream' } + ) - const endTime = Date.now() - const totalTime = (endTime - startTime) / 1000 // seconds - - // Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds) - if (response.data.eval_count && response.data.eval_duration) { - const tokenCount = response.data.eval_count - const evalDurationSeconds = response.data.eval_duration / 1e9 - const tokensPerSecond = tokenCount / evalDurationSeconds - - // Time to first token from prompt_eval_duration - const ttft = response.data.prompt_eval_duration - ? response.data.prompt_eval_duration / 1e6 // Convert to ms - : (totalTime * 1000) / 2 // Estimate if not available - - return { - ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, - ai_model_used: AI_BENCHMARK_MODEL, - ai_time_to_first_token: Math.round(ttft * 100) / 100, + await new Promise((resolve, reject) => { + let buffer = '' + let lastEmit = 0 + response.data.on('data', (chunk: Buffer) => { + buffer += chunk.toString('utf8') + // Ollama streams newline-delimited JSON; process each complete line. + let newlineIdx: number + while ((newlineIdx = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, newlineIdx).trim() + buffer = buffer.slice(newlineIdx + 1) + if (!line) continue + try { + const obj = JSON.parse(line) + if (typeof obj.response === 'string' && obj.response.length > 0) { + if (firstTokenAt === null) { + firstTokenAt = Date.now() + this.telemetry?.setStageMetric('tokens_per_sec', 0, firstTokenAt - startTime) + } + responseText += obj.response + streamedTokens++ + const now = Date.now() + if (firstTokenAt && now - lastEmit >= 400) { + const elapsed = (now - firstTokenAt) / 1000 + const rate = elapsed > 0 ? streamedTokens / elapsed : 0 + this.telemetry?.setStageMetric( + 'tokens_per_sec', + Math.round(rate * 10) / 10, + firstTokenAt - startTime + ) + lastEmit = now + } + } + if (obj.done) { + finalEvalCount = obj.eval_count || 0 + finalEvalDuration = obj.eval_duration || 0 + finalPromptEvalDuration = obj.prompt_eval_duration || 0 + } + } catch { + // Ignore a partial or non-JSON line; the next chunk completes it. + } } - } + }) + response.data.on('end', () => resolve()) + response.data.on('error', (err: Error) => reject(err)) + }) - // Fallback calculation - const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100 - const tokensPerSecond = estimatedTokens / totalTime + const totalTime = (Date.now() - startTime) / 1000 // seconds + + // Authoritative scored numbers (identical to the previous non-streaming path). + if (finalEvalCount && finalEvalDuration) { + const evalDurationSeconds = finalEvalDuration / 1e9 + const tokensPerSecond = finalEvalCount / evalDurationSeconds + const ttft = finalPromptEvalDuration + ? finalPromptEvalDuration / 1e6 // Convert ns to ms + : (totalTime * 1000) / 2 // Estimate if not available return { ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, ai_model_used: AI_BENCHMARK_MODEL, - ai_time_to_first_token: Math.round((totalTime * 1000) / 2), + ai_time_to_first_token: Math.round(ttft * 100) / 100, } + } + + // Fallback if Ollama didn't return eval metrics: use the streamed counts. + const estimatedTokens = streamedTokens || responseText.split(' ').length * 1.3 || 100 + const tokensPerSecond = totalTime > 0 ? estimatedTokens / totalTime : 0 + + return { + ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, + ai_model_used: AI_BENCHMARK_MODEL, + ai_time_to_first_token: firstTokenAt + ? Math.round(firstTokenAt - startTime) + : Math.round((totalTime * 1000) / 2), + } } catch (error) { throw new Error(`AI benchmark failed: ${error.message}`) } @@ -789,6 +875,7 @@ export class BenchmarkService { */ private _updateStatus(status: BenchmarkStatus, message: string) { this.currentStatus = status + this.telemetry?.setStage(status) const progress: BenchmarkProgress = { status, @@ -796,6 +883,9 @@ export class BenchmarkService { message, current_stage: this._getStageLabel(status), timestamp: new Date().toISOString(), + stages: this.currentStages, + stage_index: this.currentStages.findIndex((s) => s.status === status), + stage_count: this.currentStages.length, } transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_PROGRESS, { diff --git a/admin/app/services/benchmark_telemetry.ts b/admin/app/services/benchmark_telemetry.ts new file mode 100644 index 0000000..36346d7 --- /dev/null +++ b/admin/app/services/benchmark_telemetry.ts @@ -0,0 +1,94 @@ +import transmit from '@adonisjs/transmit/services/main' +import si from 'systeminformation' +import logger from '@adonisjs/core/services/logger' +import { BROADCAST_CHANNELS } from '../../constants/broadcast.js' +import type { BenchmarkStatus, BenchmarkTelemetry } from '../../types/benchmark.js' + +const SAMPLE_INTERVAL_MS = 1000 + +/** + * Samples host telemetry (per-core CPU load, temperature, disk throughput) on a + * timer and broadcasts it over SSE while a benchmark runs. Read-only: the sampler + * runs in the orchestration process, never inside the sysbench container, so it + * cannot influence the scored numbers. + * + * Signals come from `systeminformation`, which reads the host's /proc and /sys + * (not namespaced for these counters), so this works from inside a container too. + * Temperature legitimately isn't available on every host (VMs, some hwmon-less + * boards) and is reported as null rather than faked. + */ +export class BenchmarkTelemetrySampler { + private timer: NodeJS.Timeout | null = null + private readonly benchmarkId: string | null + private status: BenchmarkStatus = 'starting' + private startedAt = Date.now() + private stageMetric: BenchmarkTelemetry['stage_metric'] | undefined + private sampling = false + + constructor(benchmarkId: string | null) { + this.benchmarkId = benchmarkId + } + + start() { + if (this.timer) return + this.startedAt = Date.now() + // Prime systeminformation's per-second deltas and emit an initial frame, + // then continue on the interval. + void this._sample() + this.timer = setInterval(() => void this._sample(), SAMPLE_INTERVAL_MS) + } + + /** Tag subsequent frames with the current stage; clears any prior in-test metric. */ + setStage(status: BenchmarkStatus) { + this.status = status + this.stageMetric = undefined + } + + /** Inject an in-test metric (e.g. live AI tokens/sec) into subsequent frames. */ + setStageMetric(kind: NonNullable['kind'], value: number, ttftMs?: number) { + this.stageMetric = { kind, value, ...(ttftMs !== undefined ? { ttft_ms: ttftMs } : {}) } + } + + stop() { + if (this.timer) { + clearInterval(this.timer) + this.timer = null + } + } + + private async _sample() { + // Skip if the previous sample is still in flight (currentLoad can take a + // couple hundred ms); never let samples pile up. + if (this.sampling) return + this.sampling = true + try { + const [load, temp, fs] = await Promise.all([ + si.currentLoad().catch(() => null), + si.cpuTemperature().catch(() => null), + si.fsStats().catch(() => null), + ]) + + const overall = load ? Math.max(0, Math.round(load.currentLoad)) : 0 + const perCore = load?.cpus?.map((c) => Math.max(0, Math.round(c.load))) ?? [] + const tempC = temp && typeof temp.main === 'number' && temp.main > 0 ? Math.round(temp.main) : null + const readMb = fs && typeof fs.rx_sec === 'number' && fs.rx_sec >= 0 ? Number((fs.rx_sec / 1e6).toFixed(1)) : 0 + const writeMb = fs && typeof fs.wx_sec === 'number' && fs.wx_sec >= 0 ? Number((fs.wx_sec / 1e6).toFixed(1)) : 0 + + const payload: BenchmarkTelemetry = { + benchmark_id: this.benchmarkId, + status: this.status, + t: Date.now() - this.startedAt, + cpu: { overall, per_core: perCore }, + temp_c: tempC, + disk: { read_mb_s: readMb, write_mb_s: writeMb }, + ...(this.stageMetric ? { stage_metric: this.stageMetric } : {}), + } + + transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_TELEMETRY, payload) + } catch (err) { + logger.debug(`[BenchmarkTelemetry] sample failed: ${err.message}`) + } finally { + this.sampling = false + } + } +} diff --git a/admin/constants/broadcast.ts b/admin/constants/broadcast.ts index 55c3048..75db1fa 100644 --- a/admin/constants/broadcast.ts +++ b/admin/constants/broadcast.ts @@ -1,6 +1,7 @@ export const BROADCAST_CHANNELS = { BENCHMARK_PROGRESS: 'benchmark-progress', + BENCHMARK_TELEMETRY: 'benchmark-telemetry', OLLAMA_MODEL_DOWNLOAD: 'ollama-model-download', SERVICE_INSTALLATION: 'service-installation', SERVICE_UPDATES: 'service-updates', diff --git a/admin/inertia/components/benchmark/BenchmarkRunView.tsx b/admin/inertia/components/benchmark/BenchmarkRunView.tsx new file mode 100644 index 0000000..a361560 --- /dev/null +++ b/admin/inertia/components/benchmark/BenchmarkRunView.tsx @@ -0,0 +1,167 @@ +import { useEffect, useRef, useState } from 'react' +import { IconCpu, IconServer, IconRobot, IconTemperature, IconDatabase, IconChartBar } from '@tabler/icons-react' +import StageRail from './StageRail' +import CoreGrid from './CoreGrid' +import Sparkline from './Sparkline' +import LiveReadout from './LiveReadout' +import type { BenchmarkRunHook } from '~/hooks/useBenchmarkRun' +import type { BenchmarkStatus } from '../../../types/benchmark' + +function formatElapsed(ms: number): string { + const s = Math.floor(ms / 1000) + return `${Math.floor(s / 60)}:${(s % 60).toString().padStart(2, '0')}` +} + +function StageHero({ run }: { run: BenchmarkRunHook }) { + const status = run.status + const heroCard = 'bg-desert-white rounded-lg p-6 border border-desert-stone-light h-full min-h-[240px] flex flex-col' + const title = 'text-sm font-semibold text-desert-green uppercase tracking-wide mb-4 flex items-center gap-2' + + if (status === 'running_ai') { + return ( +
+
AI Inference
+
+ +
+ +
+
+
+ ) + } + + if (status === 'running_disk_read' || status === 'running_disk_write') { + const isRead = status === 'running_disk_read' + return ( +
+
{isRead ? 'Disk Read' : 'Disk Write'}
+
+ +
+ +
+
+
+ ) + } + + if (status === 'calculating_score') { + return ( +
+
Compiling Report
+
+
+
+ Calculating your NOMAD Score... +
+
+
+ ) + } + + if (status === 'detecting_hardware' || status === 'starting' || status === null) { + return ( +
+
Identifying Hardware
+
+
+
+ Detecting your system... +
+
+
+ ) + } + + // CPU + memory stages: the core grid is the centrepiece. + return ( +
+
+ {status === 'running_memory' ? 'Memory Throughput' : 'CPU Load'} +
+
+ +
+
+ ) +} + +/** Always-on host vitals, so something is moving between and during stages. */ +function VitalsStrip({ run }: { run: BenchmarkRunHook }) { + const card = 'bg-desert-white rounded-lg p-4 border border-desert-stone-light' + return ( +
+
+
+ CPU Load +
+ +
+ +
+
+ + {run.tempC !== null && ( +
+
+ Temperature +
+ +
+ )} + +
+
+ Disk +
+
+ + +
+
+
+ ) +} + +export default function BenchmarkRunView({ run }: { run: BenchmarkRunHook }) { + // Elapsed clock, driven locally so it ticks even between telemetry frames. + const startedAt = useRef(Date.now()) + const [, force] = useState(0) + useEffect(() => { + const id = setInterval(() => force((n) => n + 1), 1000) + return () => clearInterval(id) + }, []) + + const activeStatus: BenchmarkStatus | null = run.status + + return ( +
+ +
+
+ +
+ +
+
+ ) +} diff --git a/admin/inertia/components/benchmark/CoreGrid.tsx b/admin/inertia/components/benchmark/CoreGrid.tsx new file mode 100644 index 0000000..b7626cb --- /dev/null +++ b/admin/inertia/components/benchmark/CoreGrid.tsx @@ -0,0 +1,59 @@ +import classNames from '~/lib/classNames' + +interface CoreGridProps { + /** Per-thread load, 0-100. */ + loads: number[] +} + +const MAX_CELLS = 64 + +// Colour ramp by load: idle olive -> warm orange -> hot red. Cells transition +// smoothly as load climbs, so a single-thread pass lights one cell and an +// all-threads pass lights the whole grid. +function cellClass(load: number): string { + if (load >= 80) return 'bg-desert-red' + if (load >= 55) return 'bg-desert-orange' + if (load >= 25) return 'bg-desert-olive' + if (load >= 8) return 'bg-desert-green' + return 'bg-desert-green/15' +} + +export default function CoreGrid({ loads }: CoreGridProps) { + // Fold very high core counts down to MAX_CELLS by averaging groups. + let cells = loads + let grouped = 1 + if (loads.length > MAX_CELLS) { + grouped = Math.ceil(loads.length / MAX_CELLS) + cells = [] + for (let i = 0; i < loads.length; i += grouped) { + const slice = loads.slice(i, i + grouped) + cells.push(slice.reduce((a, b) => a + b, 0) / slice.length) + } + } + + const cols = Math.min(cells.length, 16) || 1 + + return ( +
+
+ {cells.map((load, i) => ( +
+ ))} +
+
+ {loads.length} thread{loads.length === 1 ? '' : 's'} + {grouped > 1 ? ` (${grouped}/cell)` : ''} +
+
+ ) +} diff --git a/admin/inertia/components/benchmark/LiveReadout.tsx b/admin/inertia/components/benchmark/LiveReadout.tsx new file mode 100644 index 0000000..cf8ddf1 --- /dev/null +++ b/admin/inertia/components/benchmark/LiveReadout.tsx @@ -0,0 +1,30 @@ +import classNames from '~/lib/classNames' + +interface LiveReadoutProps { + value: number | null + unit?: string + label: string + /** Optional secondary line, e.g. a TTFT badge. */ + sub?: string + size?: 'md' | 'lg' + className?: string +} + +/** + * Big monospace numeric readout for live metrics (tokens/sec, MB/s, %). + * Uses tabular figures so the digits don't jitter as the value updates. + */ +export default function LiveReadout({ value, unit, label, sub, size = 'md', className = 'text-desert-green' }: LiveReadoutProps) { + const display = value === null ? '--' : value >= 100 ? Math.round(value).toString() : value.toFixed(1) + + return ( +
+
+ {display} + {unit && {unit}} +
+
{label}
+ {sub &&
{sub}
} +
+ ) +} diff --git a/admin/inertia/components/benchmark/Sparkline.tsx b/admin/inertia/components/benchmark/Sparkline.tsx new file mode 100644 index 0000000..e5e85e5 --- /dev/null +++ b/admin/inertia/components/benchmark/Sparkline.tsx @@ -0,0 +1,56 @@ +interface SparklineProps { + data: number[] + height?: number + /** Fixed y-axis max; when unset the sparkline auto-scales to the data. */ + max?: number + fill?: boolean + /** Tailwind text-color class drives both stroke and fill via currentColor. */ + className?: string +} + +/** + * Minimal self-contained SVG sparkline (no chart library). Colour comes from the + * parent's text color so it inherits the desert palette. + */ +export default function Sparkline({ data, height = 48, max, fill = true, className = 'text-desert-green' }: SparklineProps) { + const width = 240 + const hi = Math.max(max ?? 0, ...data, 1) + + let line = '' + if (data.length >= 2) { + line = data + .map((v, i) => { + const x = (i / (data.length - 1)) * width + const y = height - (Math.min(Math.max(v, 0), hi) / hi) * height + return `${i === 0 ? 'M' : 'L'}${x.toFixed(1)} ${y.toFixed(1)}` + }) + .join(' ') + } + + return ( + + ) +} diff --git a/admin/inertia/components/benchmark/StageRail.tsx b/admin/inertia/components/benchmark/StageRail.tsx new file mode 100644 index 0000000..bff4e74 --- /dev/null +++ b/admin/inertia/components/benchmark/StageRail.tsx @@ -0,0 +1,93 @@ +import { IconCheck } from '@tabler/icons-react' +import classNames from '~/lib/classNames' +import type { BenchmarkStageDescriptor, BenchmarkStatus } from '../../../types/benchmark' + +interface StageRailProps { + stages: BenchmarkStageDescriptor[] + stageIndex: number + status: BenchmarkStatus | null + progressPercent: number + message: string + elapsedLabel: string +} + +type NodeState = 'pending' | 'active' | 'complete' | 'error' + +export default function StageRail({ stages, stageIndex, status, progressPercent, message, elapsedLabel }: StageRailProps) { + const done = status === 'completed' + const pct = done ? 100 : Math.min(100, Math.max(0, progressPercent)) + + const stateFor = (i: number): NodeState => { + if (done || (stageIndex >= 0 && i < stageIndex)) return 'complete' + if (i === stageIndex) return status === 'error' ? 'error' : 'active' + return 'pending' + } + + return ( +
+
+
+ {stages.map((stage, i) => { + const state = stateFor(i) + const nodeClass = { + complete: 'bg-desert-green border-desert-green text-desert-white', + active: 'border-desert-green text-desert-green animate-pulse', + error: 'border-desert-red text-desert-red', + pending: 'border-desert-stone-light text-desert-stone', + }[state] + return ( +
+
+
+ {state === 'complete' ? ( + + ) : ( + {i + 1} + )} +
+ + {stage.label} + +
+ {i < stages.length - 1 && ( +
+ )} +
+ ) + })} +
+
+ +
+
+ {message} + {elapsedLabel} +
+
+
+
+
+
+ ) +} diff --git a/admin/inertia/hooks/useBenchmarkRun.ts b/admin/inertia/hooks/useBenchmarkRun.ts new file mode 100644 index 0000000..9e774bc --- /dev/null +++ b/admin/inertia/hooks/useBenchmarkRun.ts @@ -0,0 +1,124 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { useTransmit } from 'react-adonis-transmit' +import { BROADCAST_CHANNELS } from '../../constants/broadcast' +import type { + BenchmarkProgress, + BenchmarkTelemetry, + BenchmarkStatus, + BenchmarkStageDescriptor, +} from '../../types/benchmark' + +// Rolling window length for the sparklines (~1 minute at 1 Hz). +const RING = 60 + +export type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string } + +type LiveState = { + perCore: number[] + cpuOverall: number + cpuHistory: number[] + tempC: number | null + diskReadMbs: number + diskWriteMbs: number + diskReadHistory: number[] + diskWriteHistory: number[] + aiTokensPerSec: number | null + aiTokHistory: number[] + aiTtftMs: number | null +} + +const EMPTY_LIVE: LiveState = { + perCore: [], + cpuOverall: 0, + cpuHistory: [], + tempC: null, + diskReadMbs: 0, + diskWriteMbs: 0, + diskReadHistory: [], + diskWriteHistory: [], + aiTokensPerSec: null, + aiTokHistory: [], + aiTtftMs: null, +} + +const pushRing = (arr: number[], v: number) => { + const next = arr.length >= RING ? arr.slice(arr.length - RING + 1) : arr.slice() + next.push(v) + return next +} + +/** + * Owns the two benchmark SSE subscriptions (stage progress + high-rate + * telemetry) and exposes a flat, render-ready view of a live run. All history + * buffers are kept as fresh arrays so memoized children re-render on change. + */ +export function useBenchmarkRun(opts?: { onFinished?: (status: 'completed' | 'error', message: string) => void }) { + const { subscribe } = useTransmit() + const onFinishedRef = useRef(opts?.onFinished) + onFinishedRef.current = opts?.onFinished + + const [progress, setProgress] = useState(null) + const [live, setLive] = useState(EMPTY_LIVE) + + const reset = useCallback(() => { + setProgress(null) + setLive(EMPTY_LIVE) + }, []) + + useEffect(() => { + const unsubProgress = subscribe( + BROADCAST_CHANNELS.BENCHMARK_PROGRESS, + (data: BenchmarkProgressWithID) => { + setProgress(data) + if (data.status === 'completed' || data.status === 'error') { + onFinishedRef.current?.(data.status, data.message) + } + } + ) + + const unsubTelemetry = subscribe( + BROADCAST_CHANNELS.BENCHMARK_TELEMETRY, + (data: BenchmarkTelemetry) => { + setLive((prev) => { + const isTok = data.stage_metric?.kind === 'tokens_per_sec' + return { + perCore: data.cpu.per_core, + cpuOverall: data.cpu.overall, + cpuHistory: pushRing(prev.cpuHistory, data.cpu.overall), + tempC: data.temp_c, + diskReadMbs: data.disk.read_mb_s, + diskWriteMbs: data.disk.write_mb_s, + diskReadHistory: pushRing(prev.diskReadHistory, data.disk.read_mb_s), + diskWriteHistory: pushRing(prev.diskWriteHistory, data.disk.write_mb_s), + aiTokensPerSec: isTok ? data.stage_metric!.value : prev.aiTokensPerSec, + aiTokHistory: isTok ? pushRing(prev.aiTokHistory, data.stage_metric!.value) : prev.aiTokHistory, + aiTtftMs: isTok && data.stage_metric!.ttft_ms !== undefined ? data.stage_metric!.ttft_ms : prev.aiTtftMs, + } + }) + } + ) + + return () => { + unsubProgress() + unsubTelemetry() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [subscribe]) + + const status: BenchmarkStatus | null = progress?.status ?? null + const stages: BenchmarkStageDescriptor[] = progress?.stages ?? [] + const stageIndex = progress?.stage_index ?? -1 + + return { + progress, + status, + stages, + stageIndex, + message: progress?.message ?? '', + progressPercent: progress?.progress ?? 0, + ...live, + reset, + } +} + +export type BenchmarkRunHook = ReturnType diff --git a/admin/inertia/pages/settings/benchmark.tsx b/admin/inertia/pages/settings/benchmark.tsx index 8565cd0..85b8881 100644 --- a/admin/inertia/pages/settings/benchmark.tsx +++ b/admin/inertia/pages/settings/benchmark.tsx @@ -1,5 +1,5 @@ import { Head, Link, usePage } from '@inertiajs/react' -import { useState, useEffect, useRef } from 'react' +import { useState } from 'react' import SettingsLayout from '~/layouts/SettingsLayout' import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' import CircularGauge from '~/components/systeminfo/CircularGauge' @@ -17,15 +17,13 @@ import { IconChevronDown, IconClock, } from '@tabler/icons-react' -import { useTransmit } from 'react-adonis-transmit' -import { BenchmarkProgress, BenchmarkStatus } from '../../../types/benchmark' +import { BenchmarkStatus } from '../../../types/benchmark' import BenchmarkResult from '#models/benchmark_result' import api from '~/lib/api' import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import { SERVICE_NAMES } from '../../../constants/service_names' -import { BROADCAST_CHANNELS } from '../../../constants/broadcast' - -type BenchmarkProgressWithID = BenchmarkProgress & { benchmark_id: string } +import { useBenchmarkRun } from '~/hooks/useBenchmarkRun' +import BenchmarkRunView from '~/components/benchmark/BenchmarkRunView' export default function BenchmarkPage(props: { benchmark: { @@ -35,12 +33,10 @@ export default function BenchmarkPage(props: { } }) { const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props - const { subscribe } = useTransmit() const queryClient = useQueryClient() const aiInstalled = useServiceInstalledStatus(SERVICE_NAMES.OLLAMA) - const [progress, setProgress] = useState(null) const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle') - const refetchLatestRef = useRef<(() => void) | null>(null) + const [errorMsg, setErrorMsg] = useState(null) const [showDetails, setShowDetails] = useState(false) const [showHistory, setShowHistory] = useState(false) const [showAIRequiredAlert, setShowAIRequiredAlert] = useState(false) @@ -61,7 +57,18 @@ export default function BenchmarkPage(props: { }, initialData: props.benchmark.latestResult, }) - refetchLatestRef.current = refetchLatest + + // Live run state: owns the progress + telemetry SSE subscriptions. + const run = useBenchmarkRun({ + onFinished: (status, message) => { + setIsRunning(false) + if (status === 'completed') { + refetchLatest() + } else { + setErrorMsg(message || 'Benchmark failed') + } + }, + }) // Fetch all benchmark results for history const { data: benchmarkHistory } = useQuery({ @@ -75,55 +82,26 @@ export default function BenchmarkPage(props: { }, }) - // Run benchmark mutation (uses sync mode by default for simpler local dev) + // Run benchmark mutation (async: dispatched to the queue worker; live progress + // and completion arrive over SSE via useBenchmarkRun). const runBenchmark = useMutation({ mutationFn: async (type: 'full' | 'system' | 'ai') => { + setErrorMsg(null) + run.reset() setIsRunning(true) - setProgress({ - status: 'starting', - progress: 5, - message: 'Starting benchmark... This takes 2-5 minutes.', - current_stage: 'Starting', - benchmark_id: '', - timestamp: new Date().toISOString(), - }) - - // Use sync mode - runs inline without needing Redis/queue worker - return await api.runBenchmark(type, true) + return await api.runBenchmark(type) }, onSuccess: (data) => { - if (data?.success) { - setProgress({ - status: 'completed', - progress: 100, - message: 'Benchmark completed!', - current_stage: 'Complete', - benchmark_id: data.benchmark_id, - timestamp: new Date().toISOString(), - }) - refetchLatest() - } else { - setProgress({ - status: 'error', - progress: 0, - message: 'Benchmark failed', - current_stage: 'Error', - benchmark_id: '', - timestamp: new Date().toISOString(), - }) + // Dispatch only confirms the job started; the 'completed'/'error' SSE event + // drives the rest (see useBenchmarkRun's onFinished). + if (!data?.success) { + setIsRunning(false) + setErrorMsg('Failed to start benchmark') } - setIsRunning(false) }, onError: (error) => { - setProgress({ - status: 'error', - progress: 0, - message: error.message || 'Benchmark failed', - current_stage: 'Error', - benchmark_id: '', - timestamp: new Date().toISOString(), - }) setIsRunning(false) + setErrorMsg(error.message || 'Failed to start benchmark') }, }) @@ -204,120 +182,6 @@ export default function BenchmarkPage(props: { runBenchmark.mutate('full') } - // Simulate progress during sync benchmark (since we don't get SSE updates) - useEffect(() => { - if (!isRunning || progress?.status === 'completed' || progress?.status === 'error') return - - const stages: { - status: BenchmarkStatus - progress: number - message: string - label: string - duration: number - }[] = [ - { - status: 'detecting_hardware', - progress: 10, - message: 'Detecting system hardware...', - label: 'Detecting Hardware', - duration: 2000, - }, - { - status: 'running_cpu', - progress: 25, - message: 'Running CPU benchmark (30s)...', - label: 'CPU Benchmark', - duration: 32000, - }, - { - status: 'running_memory', - progress: 40, - message: 'Running memory benchmark...', - label: 'Memory Benchmark', - duration: 8000, - }, - { - status: 'running_disk_read', - progress: 55, - message: 'Running disk read benchmark (30s)...', - label: 'Disk Read Test', - duration: 35000, - }, - { - status: 'running_disk_write', - progress: 70, - message: 'Running disk write benchmark (30s)...', - label: 'Disk Write Test', - duration: 35000, - }, - { - status: 'downloading_ai_model', - progress: 80, - message: 'Downloading AI benchmark model (first run only)...', - label: 'Downloading AI Model', - duration: 5000, - }, - { - status: 'running_ai', - progress: 85, - message: 'Running AI inference benchmark...', - label: 'AI Inference Test', - duration: 15000, - }, - { - status: 'calculating_score', - progress: 95, - message: 'Calculating NOMAD score...', - label: 'Calculating Score', - duration: 2000, - }, - ] - - let currentStage = 0 - const advanceStage = () => { - if (currentStage < stages.length && isRunning) { - const stage = stages[currentStage] - setProgress({ - status: stage.status, - progress: stage.progress, - message: stage.message, - current_stage: stage.label, - benchmark_id: '', - timestamp: new Date().toISOString(), - }) - currentStage++ - } - } - - // Start the first stage after a short delay - const timers: NodeJS.Timeout[] = [] - let elapsed = 1000 - stages.forEach((stage) => { - timers.push(setTimeout(() => advanceStage(), elapsed)) - elapsed += stage.duration - }) - - return () => { - timers.forEach((t) => clearTimeout(t)) - } - }, [isRunning]) - - // Listen for benchmark progress via SSE (backup for async mode) - useEffect(() => { - const unsubscribe = subscribe(BROADCAST_CHANNELS.BENCHMARK_PROGRESS, (data: BenchmarkProgressWithID) => { - setProgress(data) - if (data.status === 'completed' || data.status === 'error') { - setIsRunning(false) - refetchLatestRef.current?.() - } - }) - - return () => { - unsubscribe() - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [subscribe]) - const formatBytes = (bytes: number) => { const gb = bytes / (1024 * 1024 * 1024) return `${gb.toFixed(1)} GB` @@ -329,25 +193,6 @@ export default function BenchmarkPage(props: { return 'text-red-600' } - const getProgressPercent = () => { - if (!progress) return 0 - const stages: Record = { - idle: 0, - starting: 5, - detecting_hardware: 10, - running_cpu: 25, - running_memory: 40, - running_disk_read: 55, - running_disk_write: 70, - downloading_ai_model: 80, - running_ai: 85, - calculating_score: 95, - completed: 100, - error: 0, - } - return stages[progress.status] || 0 - } - // Calculate AI score from tokens per second (normalized to 0-100) // Reference: 30 tok/s = 50 score, 60 tok/s = 100 score const getAIScore = (tokensPerSecond: number | null): number => { @@ -375,33 +220,19 @@ export default function BenchmarkPage(props: { Run Benchmark -
- {isRunning ? ( -
-
-
- - {progress?.current_stage || 'Running benchmark...'} - -
-
-
-
-

{progress?.message}

-
- ) : ( + {isRunning ? ( + + ) : ( +
- {progress?.status === 'error' && ( + {errorMsg && ( setProgress(null)} + onDismiss={() => setErrorMsg(null)} /> )} {showAIRequiredAlert && ( @@ -469,8 +300,8 @@ export default function BenchmarkPage(props: {

)}
- )} -
+
+ )} {/* Results Section */} diff --git a/admin/types/benchmark.ts b/admin/types/benchmark.ts index 9eda94b..5270ff7 100644 --- a/admin/types/benchmark.ts +++ b/admin/types/benchmark.ts @@ -64,6 +64,20 @@ export type BenchmarkSettings = { last_benchmark_run: string | null } +// A single stage in the ordered run plan (drives the frontend stage rail) +export type BenchmarkStageDescriptor = { + status: BenchmarkStatus + label: string +} + +// The raw metric produced when a stage finishes, surfaced to the live UI +export type BenchmarkPartialResult = { + status: BenchmarkStatus + label: string + value: number + unit: string +} + // Progress update for real-time feedback export type BenchmarkProgress = { status: BenchmarkStatus @@ -71,6 +85,35 @@ export type BenchmarkProgress = { message: string current_stage: string timestamp: string + // The ordered stage plan for this run + where we are in it. Optional so old + // clients / payloads without these fields still render. + stages?: BenchmarkStageDescriptor[] + stage_index?: number + stage_count?: number + // Raw result of the stage that just completed (fills the "results so far" strip) + partial_result?: BenchmarkPartialResult +} + +// High-rate live telemetry sample broadcast during a run (1-2 Hz) +export type BenchmarkTelemetry = { + benchmark_id: string | null + status: BenchmarkStatus + t: number // ms since run start + cpu: { + overall: number // 0-100 + per_core: number[] // 0-100 per host thread + } + temp_c: number | null // null when host sensors are unavailable + disk: { + read_mb_s: number + write_mb_s: number + } + // In-test metric injected by the active stage (e.g. live AI tokens/sec) + stage_metric?: { + kind: 'tokens_per_sec' | 'events_per_sec' | 'mib_s' + value: number + ttft_ms?: number + } } // API request types