From 92b161d19dd6c1a5eccdbf760a50392b6f7378bf Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Sat, 11 Jul 2026 21:53:03 -0700 Subject: [PATCH] feat(benchmark): end-of-run score reveal + NVIDIA GPU-util overlay (#1082) Phase 3 of the live benchmark run experience. - ScoreReveal: replaces the abrupt run-view unmount with a deliberate end-of-run "REPORT" card -- animated NOMAD score gauge + odometer count-up number, sub-score gauges cascading in, Continue button + 5s auto-dismiss. Takes score scale as a prop so it survives Score v2. - GPU-util overlay (NVIDIA): during the AI stage, a ~1Hz nvidia-smi poll inside the Ollama container feeds live GPU utilization + VRAM into the telemetry frames; shown in the AI hero, hidden when absent (AMD/none). Poller is side-effect-only and cleared in a finally -- scored numbers unchanged. - Disk polish: reset in-test buffers on stage transition so the write stage no longer briefly shows the carried disk-read value. Browser-validated on NOMAD3 (RTX 5060): GPU overlay live (1% util, 0.2/8.0 GB VRAM during model load); reveal cascade + count-up; AI-only score 39.8 (scored path unchanged). Part of #1082 (tracker stays open). Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/app/services/benchmark_service.ts | 92 +++++++++++ admin/app/services/benchmark_telemetry.ts | 10 +- .../components/benchmark/BenchmarkRunView.tsx | 19 +++ .../components/benchmark/ScoreReveal.tsx | 153 ++++++++++++++++++ admin/inertia/hooks/useBenchmarkRun.ts | 27 ++++ admin/inertia/pages/settings/benchmark.tsx | 20 +++ admin/types/benchmark.ts | 2 + 7 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 admin/inertia/components/benchmark/ScoreReveal.tsx diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index c9d1153..87042c7 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -11,6 +11,7 @@ import type { BenchmarkType, BenchmarkStatus, BenchmarkProgress, + BenchmarkTelemetry, HardwareInfo, DiskType, SystemScores, @@ -525,10 +526,74 @@ export class BenchmarkService { } } + /** + * Probe live NVIDIA GPU stats by exec-ing nvidia-smi inside the Ollama + * container (same approach as SystemService.getNvidiaSmiInfo). Telemetry-only: + * returns null on any failure (no Ollama container, no NVIDIA GPU / AMD, + * unparseable output) and must never affect the scored benchmark numbers. + */ + private async _probeGpuStats(): Promise | null> { + try { + const containers = await this.dockerService.docker.listContainers({ all: false }) + const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`)) + if (!ollamaContainer) return null + + const container = this.dockerService.docker.getContainer(ollamaContainer.Id) + const exec = await container.exec({ + Cmd: [ + 'nvidia-smi', + '--query-gpu=utilization.gpu,memory.used,memory.total', + '--format=csv,noheader,nounits', + ], + AttachStdout: true, + AttachStderr: true, + Tty: true, + }) + + // Read the output stream with a timeout to prevent hanging if nvidia-smi fails + const stream = await exec.start({ Tty: true }) + const output = await new Promise((resolve) => { + let data = '' + const timeout = setTimeout(() => resolve(data), 3000) + stream.on('data', (chunk: Buffer) => { + data += chunk.toString() + }) + stream.on('end', () => { + clearTimeout(timeout) + resolve(data) + }) + }) + + // Remove any non-printable characters and trim the output + const cleaned = Array.from(output) + .filter((character) => character.charCodeAt(0) > 8) + .join('') + .trim() + if (!cleaned || cleaned.toLowerCase().includes('error') || cleaned.toLowerCase().includes('not found')) { + return null + } + + // First GPU only: ", , " + const parts = cleaned.split('\n')[0].split(',').map((s) => s.trim()) + if (parts.length < 3) return null + const util = Number.parseInt(parts[0], 10) + const used = Number.parseInt(parts[1], 10) + const total = Number.parseInt(parts[2], 10) + if (!Number.isFinite(util) || !Number.isFinite(used) || !Number.isFinite(total)) return null + + return { util, vram_used_mb: used, vram_total_mb: total } + } catch { + return null + } + } + /** * Run AI benchmark using Ollama */ private async _runAIBenchmark(): Promise { + // Live GPU-util overlay poller. Side-effect-only: it feeds telemetry frames + // and is cleared in the finally below so it never outlives the AI stage. + let gpuPollTimer: NodeJS.Timeout | null = null try { this._updateStatus('running_ai', 'Running AI benchmark...') @@ -546,6 +611,29 @@ export class BenchmarkService { throw new Error(`Ollama is not running or not accessible (${errorCode}). Ensure AI Assistant is installed and running.`) } + // GPU-util overlay (NVIDIA only): probe once; if a GPU answers, poll for the + // duration of the AI stage. If the probe returns null (no GPU / AMD), the + // overlay simply never appears. Best-effort: failures never affect the run. + try { + const initialGpu = await this._probeGpuStats() + if (initialGpu) { + this.telemetry?.setGpuStats(initialGpu) + let gpuProbeInFlight = false + gpuPollTimer = setInterval(() => { + if (gpuProbeInFlight) return + gpuProbeInFlight = true + this._probeGpuStats() + .then((stats) => this.telemetry?.setGpuStats(stats)) + .catch(() => this.telemetry?.setGpuStats(null)) + .finally(() => { + gpuProbeInFlight = false + }) + }, 1000) + } + } catch { + // GPU overlay is cosmetic; ignore probe failures entirely. + } + // Check if the benchmark model is available, pull if not const ollamaService = new (await import('./ollama_service.js')).OllamaService() const modelResponse = await ollamaService.downloadModel(AI_BENCHMARK_MODEL) @@ -651,6 +739,10 @@ export class BenchmarkService { } } catch (error) { throw new Error(`AI benchmark failed: ${error.message}`) + } finally { + // Stop the GPU-util poller and clear the overlay from telemetry frames. + if (gpuPollTimer) clearInterval(gpuPollTimer) + this.telemetry?.setGpuStats(null) } } diff --git a/admin/app/services/benchmark_telemetry.ts b/admin/app/services/benchmark_telemetry.ts index 36346d7..76e203e 100644 --- a/admin/app/services/benchmark_telemetry.ts +++ b/admin/app/services/benchmark_telemetry.ts @@ -23,6 +23,7 @@ export class BenchmarkTelemetrySampler { private status: BenchmarkStatus = 'starting' private startedAt = Date.now() private stageMetric: BenchmarkTelemetry['stage_metric'] | undefined + private gpu: BenchmarkTelemetry['gpu'] | undefined private sampling = false constructor(benchmarkId: string | null) { @@ -38,10 +39,11 @@ export class BenchmarkTelemetrySampler { this.timer = setInterval(() => void this._sample(), SAMPLE_INTERVAL_MS) } - /** Tag subsequent frames with the current stage; clears any prior in-test metric. */ + /** Tag subsequent frames with the current stage; clears any prior in-test state. */ setStage(status: BenchmarkStatus) { this.status = status this.stageMetric = undefined + this.gpu = undefined } /** Inject an in-test metric (e.g. live AI tokens/sec) into subsequent frames. */ @@ -49,6 +51,11 @@ export class BenchmarkTelemetrySampler { this.stageMetric = { kind, value, ...(ttftMs !== undefined ? { ttft_ms: ttftMs } : {}) } } + /** Inject NVIDIA GPU stats into subsequent frames (null clears them). */ + setGpuStats(gpu: NonNullable | null) { + this.gpu = gpu ?? undefined + } + stop() { if (this.timer) { clearInterval(this.timer) @@ -82,6 +89,7 @@ export class BenchmarkTelemetrySampler { temp_c: tempC, disk: { read_mb_s: readMb, write_mb_s: writeMb }, ...(this.stageMetric ? { stage_metric: this.stageMetric } : {}), + ...(this.gpu ? { gpu: this.gpu } : {}), } transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_TELEMETRY, payload) diff --git a/admin/inertia/components/benchmark/BenchmarkRunView.tsx b/admin/inertia/components/benchmark/BenchmarkRunView.tsx index 5e7161f..7cad67d 100644 --- a/admin/inertia/components/benchmark/BenchmarkRunView.tsx +++ b/admin/inertia/components/benchmark/BenchmarkRunView.tsx @@ -33,6 +33,25 @@ function StageHero({ run }: { run: BenchmarkRunHook }) {
+ {run.gpuUtil !== null && ( +
+
+ GPU +
+
+ + +
+
+ )} ) diff --git a/admin/inertia/components/benchmark/ScoreReveal.tsx b/admin/inertia/components/benchmark/ScoreReveal.tsx new file mode 100644 index 0000000..7c3b5e4 --- /dev/null +++ b/admin/inertia/components/benchmark/ScoreReveal.tsx @@ -0,0 +1,153 @@ +import { useEffect, useRef, useState } from 'react' +import { IconChartBar, IconCpu, IconDatabase, IconRobot, IconServer } from '@tabler/icons-react' +import CircularGauge from '~/components/systeminfo/CircularGauge' +import StyledButton from '~/components/StyledButton' +import BenchmarkResult from '#models/benchmark_result' + +interface ScoreRevealProps { + result: BenchmarkResult + scoreScale?: { max: number; caption: string } + onDone: () => void +} + +// Same thresholds as the NOMAD Score section on the benchmark page. +const getScoreColor = (score: number) => { + if (score >= 70) return 'text-green-600' + if (score >= 40) return 'text-yellow-600' + return 'text-red-600' +} + +// AI tokens/sec normalized to 0-100 (30 tok/s = 50, 60 tok/s = 100), +// mirroring the benchmark page's getAIScore. +const getAIScore = (tokensPerSecond: number): number => + Math.min(100, Math.max(0, (tokensPerSecond / 60) * 100)) + +/** rAF count-up from 0 to `target` with ease-out, for the big score number. */ +function useCountUp(target: number, durationMs = 1200): number { + const [value, setValue] = useState(0) + useEffect(() => { + let raf = 0 + const start = performance.now() + const tick = (now: number) => { + const t = Math.min(1, (now - start) / durationMs) + const eased = 1 - Math.pow(1 - t, 3) + setValue(target * eased) + if (t < 1) raf = requestAnimationFrame(tick) + } + raf = requestAnimationFrame(tick) + return () => cancelAnimationFrame(raf) + }, [target, durationMs]) + return value +} + +/** + * End-of-run score reveal. Replaces the abrupt unmount of the live run view + * with a deliberate report: big NOMAD score gauge + count-up number, then the + * sub-score gauges cascading in. Dismisses via the Continue button or an + * auto-dismiss timer. + */ +export default function ScoreReveal({ + result, + scoreScale = { max: 100, caption: 'out of 100' }, + onDone, +}: ScoreRevealProps) { + const displayScore = useCountUp(result.nomad_score) + + // Sub-score gauges, mirroring the System Performance / AI Performance grids. + const gauges: { + label: string + value: number + variant: 'cpu' | 'memory' | 'disk' + icon: React.ReactNode + }[] = [ + { label: 'CPU', value: result.cpu_score * 100, variant: 'cpu', icon: }, + { label: 'Memory', value: result.memory_score * 100, variant: 'memory', icon: }, + { label: 'Disk Read', value: result.disk_read_score * 100, variant: 'disk', icon: }, + { label: 'Disk Write', value: result.disk_write_score * 100, variant: 'disk', icon: }, + ] + if (result.ai_tokens_per_second) { + gauges.push({ + label: 'AI Score', + value: getAIScore(result.ai_tokens_per_second), + variant: 'cpu', + icon: , + }) + } + + // Cascade: mount one gauge every 150ms; each CircularGauge animates 0→value + // on mount, so staggered mounts produce the cascade. + const [visibleCount, setVisibleCount] = useState(0) + useEffect(() => { + if (visibleCount >= gauges.length) return + const id = setTimeout(() => setVisibleCount((n) => n + 1), 150) + return () => clearTimeout(id) + }, [visibleCount, gauges.length]) + + // Dismiss exactly once, whether via the button or the auto-dismiss timer. + const doneRef = useRef(false) + const onDoneRef = useRef(onDone) + onDoneRef.current = onDone + const fireDone = () => { + if (doneRef.current) return + doneRef.current = true + onDoneRef.current() + } + useEffect(() => { + const id = setTimeout(fireDone, 5000) + return () => clearTimeout(id) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + return ( +
+
+
+
+ Report +
+ +
+
+
+ } + /> +
+
+
+ {displayScore.toFixed(1)} +
+

+ Your NOMAD Score is a weighted composite of all benchmark results. +

+
+
+ +
+ {gauges.slice(0, visibleCount).map((g) => ( +
+ +
+ ))} +
+ +
+ + Continue + +
+
+
+
+ ) +} diff --git a/admin/inertia/hooks/useBenchmarkRun.ts b/admin/inertia/hooks/useBenchmarkRun.ts index bbb7501..385184b 100644 --- a/admin/inertia/hooks/useBenchmarkRun.ts +++ b/admin/inertia/hooks/useBenchmarkRun.ts @@ -31,6 +31,10 @@ type LiveState = { /** Authoritative in-test disk throughput (sysbench interim lines), MiB/s. */ diskMibs: number | null diskMibsHistory: number[] + /** NVIDIA GPU stats during the AI stage (null when no GPU / not sampling). */ + gpuUtil: number | null + gpuVramUsedMb: number | null + gpuVramTotalMb: number | null } const EMPTY_LIVE: LiveState = { @@ -49,6 +53,9 @@ const EMPTY_LIVE: LiveState = { cpuEventsHistory: [], diskMibs: null, diskMibsHistory: [], + gpuUtil: null, + gpuVramUsedMb: null, + gpuVramTotalMb: null, } const pushRing = (arr: number[], v: number) => { @@ -70,11 +77,14 @@ export function useBenchmarkRun(opts?: { onFinished?: (status: 'completed' | 'er const [progress, setProgress] = useState(null) const [live, setLive] = useState(EMPTY_LIVE) const [partials, setPartials] = useState([]) + // Last-seen progress status, so stage transitions can reset in-test buffers. + const lastStatusRef = useRef(null) const reset = useCallback(() => { setProgress(null) setLive(EMPTY_LIVE) setPartials([]) + lastStatusRef.current = null }, []) useEffect(() => { @@ -82,6 +92,20 @@ export function useBenchmarkRun(opts?: { onFinished?: (status: 'completed' | 'er BROADCAST_CHANNELS.BENCHMARK_PROGRESS, (data: BenchmarkProgressWithID) => { setProgress(data) + // On stage transition, clear the in-test buffers so each stage starts + // clean (e.g. the disk-read throughput doesn't linger into the write + // stage). Always-on host telemetry (perCore, cpuOverall, disk proxy, + // temp) is intentionally left untouched. + if (lastStatusRef.current !== data.status) { + lastStatusRef.current = data.status + setLive((prev) => ({ + ...prev, + cpuEventsPerSec: null, + cpuEventsHistory: [], + diskMibs: null, + diskMibsHistory: [], + })) + } if (data.partial_result) { const incoming = data.partial_result setPartials((prev) => { @@ -127,6 +151,9 @@ export function useBenchmarkRun(opts?: { onFinished?: (status: 'completed' | 'er diskMibsHistory: isMib ? pushRing(prev.diskMibsHistory, data.stage_metric!.value) : prev.diskMibsHistory, + gpuUtil: data.gpu ? data.gpu.util : prev.gpuUtil, + gpuVramUsedMb: data.gpu ? data.gpu.vram_used_mb : prev.gpuVramUsedMb, + gpuVramTotalMb: data.gpu ? data.gpu.vram_total_mb : prev.gpuVramTotalMb, } }) } diff --git a/admin/inertia/pages/settings/benchmark.tsx b/admin/inertia/pages/settings/benchmark.tsx index 85b8881..12e4ede 100644 --- a/admin/inertia/pages/settings/benchmark.tsx +++ b/admin/inertia/pages/settings/benchmark.tsx @@ -24,6 +24,7 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import { SERVICE_NAMES } from '../../../constants/service_names' import { useBenchmarkRun } from '~/hooks/useBenchmarkRun' import BenchmarkRunView from '~/components/benchmark/BenchmarkRunView' +import ScoreReveal from '~/components/benchmark/ScoreReveal' export default function BenchmarkPage(props: { benchmark: { @@ -36,6 +37,7 @@ export default function BenchmarkPage(props: { const queryClient = useQueryClient() const aiInstalled = useServiceInstalledStatus(SERVICE_NAMES.OLLAMA) const [isRunning, setIsRunning] = useState(props.benchmark.status !== 'idle') + const [revealing, setRevealing] = useState(false) const [errorMsg, setErrorMsg] = useState(null) const [showDetails, setShowDetails] = useState(false) const [showHistory, setShowHistory] = useState(false) @@ -64,6 +66,7 @@ export default function BenchmarkPage(props: { setIsRunning(false) if (status === 'completed') { refetchLatest() + setRevealing(true) } else { setErrorMsg(message || 'Benchmark failed') } @@ -222,6 +225,23 @@ export default function BenchmarkPage(props: { {isRunning ? ( + ) : revealing ? ( + // Reveal slot: wait for the refetched latest result to be the one + // from this run, then hand it to the score reveal. + (() => { + const ready = + latestResult && latestResult.benchmark_id === run.progress?.benchmark_id + return ready ? ( + setRevealing(false)} /> + ) : ( +
+
+
+ Compiling report... +
+
+ ) + })() ) : (
diff --git a/admin/types/benchmark.ts b/admin/types/benchmark.ts index 5270ff7..b68a8a7 100644 --- a/admin/types/benchmark.ts +++ b/admin/types/benchmark.ts @@ -114,6 +114,8 @@ export type BenchmarkTelemetry = { value: number ttft_ms?: number } + // NVIDIA GPU stats sampled during the AI stage (absent when no NVIDIA GPU) + gpu?: { util: number; vram_used_mb: number; vram_total_mb: number } } // API request types