feat(benchmark): Score v2 app client — raws, uncapped score, v2 payload + UI
Phase 4 of the NOMAD Score v2 effort (stacked on #1089). Captures the raw channel values the leaderboard scores from, computes the uncapped v2 score in-app (byte-matching the leaderboard's server-side recompute), submits the v2 payload, and surfaces it in the benchmark UI. - Capture raws: per-channel sysbench values + O_DIRECT disk (W4), single/multi thread CPU + memory, W6 consistency companions, and run-environment metadata (run_environment, storage_path_type, gpu_compute_detected, #1016). - AI hardening: reference model llama3.1:8b, num_predict=256, VRAM eviction before the run + unload own model after, pre-flight disk check (~6.5GB, only when the model is uncached), median-of-3 run (W7). - Score: _calculateNomadScoreV2 (uncapped, reference->1000) byte-matches the leaderboard recompute. nomad_score_v2 stays null unless a full run with AI. - Payload: submitToRepository sends the v2 raws + score alongside legacy v1. - UI: v2 headline score, legacy v1 as secondary, raws/env in details. - Migration: 13 nullable columns; double (not knex float(8,2)) so millions-scale raws keep full precision and don't break the leaderboard byte-match. Reference constant REFERENCE_SCORES_V2.ai_tokens_per_second is still the stale 40.5 placeholder (TODO in source); locking it to the measured 13.2 is a follow-up gated on the AMD iGPU fix (#1074). Live prod v2 submit is separately blocked by the leaderboard TTFT floor (tracked in the leaderboard follow-up). Verified on NOMAD3 (RTX 5060) dev env: migration applies, raws recorded, v2 score computed, backend + benchmark.tsx typecheck clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood
Jake Turner
parent
c6bc40e767
commit
398ca018cd
|
|
@ -63,6 +63,60 @@ export default class BenchmarkResult extends BaseModel {
|
|||
@column()
|
||||
declare ollama_version: string | null
|
||||
|
||||
// NOMAD Score v2 raw channels (nullable — added in Score v2 Phase 4). Populated
|
||||
// on full benchmarks under benchmark_version >= 2.0.0; the leaderboard recomputes
|
||||
// the score from these on submit. cpu_events_multi is measured at
|
||||
// cpu_benchmark_threads; memory_ops_per_sec at memory_threads. Disk figures are
|
||||
// O_DIRECT MB/s. cpu_total_events/cpu_total_time are the W6 consistency companions.
|
||||
@column()
|
||||
declare cpu_events_single: number | null
|
||||
|
||||
@column()
|
||||
declare cpu_events_multi: number | null
|
||||
|
||||
@column()
|
||||
declare cpu_benchmark_threads: number | null
|
||||
|
||||
@column()
|
||||
declare cpu_total_events: number | null
|
||||
|
||||
@column()
|
||||
declare cpu_total_time: number | null
|
||||
|
||||
@column()
|
||||
declare memory_ops_per_sec: number | null
|
||||
|
||||
@column()
|
||||
declare memory_threads: number | null
|
||||
|
||||
@column()
|
||||
declare disk_read_mb_per_sec: number | null
|
||||
|
||||
@column()
|
||||
declare disk_write_mb_per_sec: number | null
|
||||
|
||||
// Uncapped NOMAD Score v2 (null for pre-v2 rows and system-only runs). The
|
||||
// legacy nomad_score below is retained in parallel for display continuity.
|
||||
// columnName + serializeAs pinned: the snake_case strategy would otherwise map
|
||||
// this property to `nomad_score_v_2` (splitting the digit) for both the DB column
|
||||
// (which the migration doesn't create) and the JSON key (which the frontend reads
|
||||
// as nomad_score_v2). Pin both so DB, API, and UI all agree.
|
||||
@column({ columnName: 'nomad_score_v2', serializeAs: 'nomad_score_v2' })
|
||||
declare nomad_score_v2: number | null
|
||||
|
||||
// Best-effort run environment metadata (issue #1016)
|
||||
@column()
|
||||
declare run_environment: string | null
|
||||
|
||||
@column()
|
||||
declare storage_path_type: string | null
|
||||
|
||||
@column({
|
||||
// Nullable tri-state; coerce the SQLite 0/1 int to a real boolean when present.
|
||||
consume: (value: number | null) => (value === null || value === undefined ? null : Boolean(value)),
|
||||
})
|
||||
declare gpu_compute_detected: boolean | null
|
||||
|
||||
// Composite NOMAD score (0-100)
|
||||
@column()
|
||||
declare nomad_score: number
|
||||
|
|
|
|||
|
|
@ -19,12 +19,18 @@ import type {
|
|||
SysbenchCpuResult,
|
||||
SysbenchMemoryResult,
|
||||
SysbenchDiskResult,
|
||||
RepositorySubmission,
|
||||
RepositorySubmissionV2,
|
||||
RepositorySubmitResponse,
|
||||
RepositoryStats,
|
||||
BenchmarkStageDescriptor,
|
||||
BenchmarkPartialResult,
|
||||
SystemBenchmarkOutput,
|
||||
SystemBenchmarkRawsV2,
|
||||
RunEnvironmentInfo,
|
||||
} from '../../types/benchmark.js'
|
||||
import KVStore from '#models/kv_store'
|
||||
import { getFreeBytes } from '../utils/image_disk_preflight.js'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { randomUUID, createHmac } from 'node:crypto'
|
||||
import { DockerService } from './docker_service.js'
|
||||
import { SERVICE_NAMES } from '../../constants/service_names.js'
|
||||
|
|
@ -48,15 +54,87 @@ const SCORE_WEIGHTS = {
|
|||
disk_write: 0.10,
|
||||
}
|
||||
|
||||
// The benchmark_version this client emits. Server dispatches on the major:
|
||||
// >= 2 → raw-channel v2 ingest (score recomputed server-side). Bumped 1.0.0 -> 2.0.0
|
||||
// for the NOMAD Score v2 rollout (uncapped index vs a frozen Reference Build).
|
||||
const BENCHMARK_VERSION = '2.0.0'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NOMAD Score v2 — the uncapped index.
|
||||
//
|
||||
// Score = 1000 × Π( (raw_i / ref_i) ^ w_i ) (weighted geometric mean, log domain).
|
||||
//
|
||||
// These constants + the computation below MUST stay byte-identical to the
|
||||
// leaderboard's score_service.ts (computeNomadScoreV2): the client shows this
|
||||
// score immediately after a run, and the server recomputes the same value from
|
||||
// the raw channels on submit. Any drift makes the displayed score disagree with
|
||||
// the leaderboard. FROZEN under benchmark_version 2.0.0 — Reference Build =
|
||||
// NOMAD6 (Ryzen 9 PRO 8945HS / 780M / 64GB, kernel 6.17), measured median-of-N
|
||||
// with direct-I/O disk 2026-07-12. There are NO clamps here; outlier control is
|
||||
// the server's rejection gates + quarantine, not the score math.
|
||||
const REFERENCE_SCORES_V2 = {
|
||||
// TODO(score-v2): STALE — 40.5 was measured on llama3.2:1b. The reference model
|
||||
// is now llama3.1:8b; re-measure this on the Reference Build (NOMAD6) with the
|
||||
// v2 harness (unload + num_predict + median-of-3) and update here AND in the
|
||||
// leaderboard's score_service.ts (must byte-match) before v2 ships.
|
||||
ai_tokens_per_second: 40.5,
|
||||
cpu_events_multi: 19231, // all-threads on the reference box (16T)
|
||||
cpu_events_single: 2351, // 1T
|
||||
memory_ops_per_sec: 26862960, // 4T, 1K block (bandwidth peak)
|
||||
disk_read_mb_per_sec: 1072, // O_DIRECT
|
||||
disk_write_mb_per_sec: 316, // O_DIRECT
|
||||
} as const
|
||||
|
||||
const SCORE_WEIGHTS_V2 = {
|
||||
ai_tokens_per_second: 0.3,
|
||||
cpu_events_multi: 0.25,
|
||||
cpu_events_single: 0.1,
|
||||
memory_ops_per_sec: 0.1,
|
||||
disk_read_mb_per_sec: 0.125,
|
||||
disk_write_mb_per_sec: 0.125,
|
||||
} as const
|
||||
|
||||
type ScoreChannelV2 = keyof typeof REFERENCE_SCORES_V2
|
||||
type ScoreRawsV2 = Record<ScoreChannelV2, number>
|
||||
const SCORE_CHANNELS_V2 = Object.keys(REFERENCE_SCORES_V2) as ScoreChannelV2[]
|
||||
|
||||
// Weights sum to 1.000; guard against a future typo silently rescaling scores.
|
||||
const WEIGHT_SUM_V2 = SCORE_CHANNELS_V2.reduce((s, k) => s + SCORE_WEIGHTS_V2[k], 0)
|
||||
if (Math.abs(WEIGHT_SUM_V2 - 1) > 1e-9) {
|
||||
throw new Error(`SCORE_WEIGHTS_V2 must sum to 1.0 (got ${WEIGHT_SUM_V2})`)
|
||||
}
|
||||
|
||||
// Benchmark configuration constants
|
||||
// Pinned by digest (was severalnines/sysbench:latest) so a latest-tag format change can't silently break the parsers fleet-wide. Digest validated on the NOMAD6 reference build 2026-07-12.
|
||||
const SYSBENCH_IMAGE = 'severalnines/sysbench@sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
|
||||
const SYSBENCH_DIGEST = 'sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
|
||||
const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench'
|
||||
|
||||
// Reference model for AI benchmark - small but meaningful
|
||||
const AI_BENCHMARK_MODEL = 'llama3.2:1b'
|
||||
// Reference model for AI benchmark. v2 uses an 8B (was llama3.2:1b): a 1B is so
|
||||
// light it runs overhead-bound on any GPU (~240 tok/s), saturating the score and
|
||||
// failing to differentiate hardware. An 8B is compute/bandwidth-bound and scales
|
||||
// with real GPU capability. Non-thinking (deterministic workload). Changing this
|
||||
// re-baselines REFERENCE_SCORES_V2.ai_tokens_per_second — re-measure on the
|
||||
// Reference Build (NOMAD6) before shipping.
|
||||
const AI_BENCHMARK_MODEL = 'llama3.1:8b'
|
||||
const AI_BENCHMARK_PROMPT = 'Explain recursion in programming in exactly 100 words.'
|
||||
// Cap generation so the workload is bounded and comparable across machines rather
|
||||
// than relying on the model honoring "100 words". ~100-150 tokens is plenty for a
|
||||
// stable tok/s rate; the cap only trims pathological long outputs.
|
||||
const AI_BENCHMARK_NUM_PREDICT = 256
|
||||
|
||||
// Memory test is frozen at 4 threads for the v2 Reference Build (the bandwidth
|
||||
// peak on the reference box). Do not vary by core count — the reference number
|
||||
// (REFERENCE_SCORES_V2.memory_ops_per_sec) was measured at exactly this value.
|
||||
const MEMORY_BENCHMARK_THREADS = 4
|
||||
|
||||
// AI is measured as the median of N runs (W7) to damp per-run inference jitter.
|
||||
const AI_BENCHMARK_RUNS = 3
|
||||
|
||||
// Minimum free disk required before pulling the AI model on first run. llama3.1:8b
|
||||
// (Q4) is ~4.9 GB; this covers the model plus headroom for the transient sysbench
|
||||
// disk-test file and normal slack. Only enforced when the model isn't already cached.
|
||||
const AI_BENCHMARK_MODEL_MIN_FREE_BYTES = 6.5 * 1024 * 1024 * 1024
|
||||
|
||||
// Reference scores for normalization (calibrated to 0-100 scale)
|
||||
// These represent "expected" scores for a mid-range system (score ~50)
|
||||
|
|
@ -162,24 +240,9 @@ export class BenchmarkService {
|
|||
throw new Error('Benchmark result has already been submitted')
|
||||
}
|
||||
|
||||
const submission: RepositorySubmission = {
|
||||
cpu_model: result.cpu_model,
|
||||
cpu_cores: result.cpu_cores,
|
||||
cpu_threads: result.cpu_threads,
|
||||
ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)),
|
||||
disk_type: result.disk_type,
|
||||
gpu_model: result.gpu_model,
|
||||
cpu_score: result.cpu_score,
|
||||
memory_score: result.memory_score,
|
||||
disk_read_score: result.disk_read_score,
|
||||
disk_write_score: result.disk_write_score,
|
||||
ai_tokens_per_second: result.ai_tokens_per_second,
|
||||
ai_time_to_first_token: result.ai_time_to_first_token,
|
||||
nomad_score: result.nomad_score,
|
||||
nomad_version: SystemService.getAppVersion(),
|
||||
benchmark_version: '1.0.0',
|
||||
builder_tag: anonymous ? null : result.builder_tag,
|
||||
}
|
||||
// Build the v2 payload (raw channels; server recomputes the score). Throws a
|
||||
// user-facing error if the row predates v2 and lacks the raw channels.
|
||||
const submission = this._buildV2Submission(result, anonymous ?? false)
|
||||
|
||||
try {
|
||||
// Generate HMAC signature for submission verification
|
||||
|
|
@ -223,6 +286,85 @@ export class BenchmarkService {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the NOMAD Score v2 submission payload from a stored result, validating
|
||||
* that every required raw channel + companion + provenance field is present.
|
||||
*
|
||||
* A row produced before this benchmark_version (or an interrupted run) won't
|
||||
* carry the v2 raws — rather than send a payload the leaderboard will reject
|
||||
* with a terse validator error, we throw a clear "re-run" message here.
|
||||
*
|
||||
* ai_time_to_first_token is stored in milliseconds but the leaderboard's v2 API
|
||||
* treats it as SECONDS, so it is divided by 1000 here.
|
||||
*/
|
||||
private _buildV2Submission(result: BenchmarkResult, anonymous: boolean): RepositorySubmissionV2 {
|
||||
const reRun = 'Re-run a Full Benchmark to share your results.'
|
||||
|
||||
// Required raw channels — all must be present and > 0.
|
||||
const requiredRaws: Array<[string, number | null]> = [
|
||||
['CPU single-thread', result.cpu_events_single],
|
||||
['CPU multi-thread', result.cpu_events_multi],
|
||||
['memory', result.memory_ops_per_sec],
|
||||
['disk read', result.disk_read_mb_per_sec],
|
||||
['disk write', result.disk_write_mb_per_sec],
|
||||
]
|
||||
for (const [label, value] of requiredRaws) {
|
||||
if (value == null || !(value > 0)) {
|
||||
throw new Error(`This benchmark is missing the ${label} raw measurement needed for the v2 leaderboard. ${reRun}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (result.ai_time_to_first_token == null || !(result.ai_time_to_first_token > 0)) {
|
||||
throw new Error(`This benchmark is missing AI time-to-first-token data. ${reRun}`)
|
||||
}
|
||||
if (
|
||||
result.cpu_benchmark_threads == null ||
|
||||
result.memory_threads == null ||
|
||||
result.cpu_total_events == null ||
|
||||
result.cpu_total_time == null
|
||||
) {
|
||||
throw new Error(`This benchmark is missing CPU test parameters needed for the v2 leaderboard. ${reRun}`)
|
||||
}
|
||||
if (!result.sysbench_digest || !result.ollama_version) {
|
||||
throw new Error(`This benchmark is missing harness provenance (sysbench/Ollama version). ${reRun}`)
|
||||
}
|
||||
|
||||
const submission: RepositorySubmissionV2 = {
|
||||
cpu_model: result.cpu_model,
|
||||
cpu_cores: result.cpu_cores,
|
||||
cpu_threads: result.cpu_threads,
|
||||
ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)),
|
||||
disk_type: result.disk_type,
|
||||
gpu_model: result.gpu_model,
|
||||
ai_tokens_per_second: result.ai_tokens_per_second!,
|
||||
cpu_events_single: result.cpu_events_single!,
|
||||
cpu_events_multi: result.cpu_events_multi!,
|
||||
memory_ops_per_sec: result.memory_ops_per_sec!,
|
||||
disk_read_mb_per_sec: result.disk_read_mb_per_sec!,
|
||||
disk_write_mb_per_sec: result.disk_write_mb_per_sec!,
|
||||
cpu_benchmark_threads: result.cpu_benchmark_threads,
|
||||
memory_threads: result.memory_threads,
|
||||
// Stored in ms; the v2 API expects seconds.
|
||||
ai_time_to_first_token: result.ai_time_to_first_token / 1000,
|
||||
cpu_total_events: result.cpu_total_events,
|
||||
cpu_total_time: result.cpu_total_time,
|
||||
ollama_version: result.ollama_version,
|
||||
sysbench_digest: result.sysbench_digest,
|
||||
nomad_version: SystemService.getAppVersion(),
|
||||
benchmark_version: BENCHMARK_VERSION,
|
||||
}
|
||||
|
||||
// Optional environment metadata — only send fields we actually have.
|
||||
if (result.run_environment) submission.run_environment = result.run_environment
|
||||
if (result.storage_path_type) submission.storage_path_type = result.storage_path_type
|
||||
if (result.gpu_compute_detected != null) submission.gpu_compute_detected = result.gpu_compute_detected
|
||||
|
||||
// Builder tag: omit entirely when anonymous or unset (validator rejects null).
|
||||
if (!anonymous && result.builder_tag) submission.builder_tag = result.builder_tag
|
||||
|
||||
return submission
|
||||
}
|
||||
|
||||
/**
|
||||
* Get comparison stats from central repository
|
||||
*/
|
||||
|
|
@ -393,6 +535,12 @@ export class BenchmarkService {
|
|||
this._updateStatus('starting', 'Starting benchmark...')
|
||||
|
||||
try {
|
||||
// Fail fast: if this run will download the (large) AI model, make sure there's
|
||||
// room BEFORE spending minutes on the system benchmarks.
|
||||
if (includeAI && (type === 'full' || type === 'ai')) {
|
||||
await this._preflightModelDiskSpace()
|
||||
}
|
||||
|
||||
// Detect hardware
|
||||
const hardware = await this.getHardwareInfo()
|
||||
|
||||
|
|
@ -403,9 +551,12 @@ export class BenchmarkService {
|
|||
disk_read_score: 0,
|
||||
disk_write_score: 0,
|
||||
}
|
||||
let systemRaws: SystemBenchmarkRawsV2 | null = null
|
||||
|
||||
if (type === 'full' || type === 'system') {
|
||||
systemScores = await this._runSystemBenchmarks()
|
||||
const systemOutput = await this._runSystemBenchmarks(hardware.cpu_threads)
|
||||
systemScores = systemOutput.scores
|
||||
systemRaws = systemOutput.raws
|
||||
}
|
||||
|
||||
// Run AI benchmark if requested and Ollama is available
|
||||
|
|
@ -431,16 +582,32 @@ export class BenchmarkService {
|
|||
}
|
||||
}
|
||||
|
||||
// Calculate NOMAD score. Only pass systemScores when the system benchmarks
|
||||
// actually ran, so an AI-only run renormalizes to just the AI weights
|
||||
// (an AI-only pass with default-zero system scores would otherwise be
|
||||
// scaled against the full NOMAD 100).
|
||||
// Calculate NOMAD scores (v1 legacy + v2 uncapped)
|
||||
this._updateStatus('calculating_score', 'Calculating NOMAD score...')
|
||||
const systemMeasured = type === 'full' || type === 'system'
|
||||
const nomadScore = this._calculateNomadScore(
|
||||
systemMeasured ? systemScores : null,
|
||||
aiScores
|
||||
)
|
||||
const nomadScore = this._calculateNomadScore(systemScores, aiScores)
|
||||
|
||||
// v2 requires a complete run of every channel (system raws + AI). Only a
|
||||
// full benchmark with AI qualifies; otherwise nomad_score_v2 stays null.
|
||||
let nomadScoreV2: number | null = null
|
||||
if (systemRaws && aiScores.ai_tokens_per_second && aiScores.ai_tokens_per_second > 0) {
|
||||
try {
|
||||
nomadScoreV2 = this._calculateNomadScoreV2({
|
||||
ai_tokens_per_second: aiScores.ai_tokens_per_second,
|
||||
cpu_events_single: systemRaws.cpu_events_single,
|
||||
cpu_events_multi: systemRaws.cpu_events_multi,
|
||||
memory_ops_per_sec: systemRaws.memory_ops_per_sec,
|
||||
disk_read_mb_per_sec: systemRaws.disk_read_mb_per_sec,
|
||||
disk_write_mb_per_sec: systemRaws.disk_write_mb_per_sec,
|
||||
})
|
||||
} catch (error) {
|
||||
// A non-positive channel shouldn't reach here (each sysbench step throws
|
||||
// on a bad parse), but never let a v2 math error sink the whole run.
|
||||
logger.warn(`NOMAD Score v2 not computed: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Best-effort environment metadata (never fails the run).
|
||||
const env = await this._detectRunEnvironment()
|
||||
|
||||
// Save result
|
||||
const result = await BenchmarkResult.create({
|
||||
|
|
@ -463,6 +630,20 @@ export class BenchmarkService {
|
|||
submitted_to_repository: false,
|
||||
sysbench_digest: SYSBENCH_DIGEST,
|
||||
ollama_version: aiScores.ai_ollama_version ?? null,
|
||||
// v2 raw channels + score (null on system-only / AI-less runs)
|
||||
cpu_events_single: systemRaws?.cpu_events_single ?? null,
|
||||
cpu_events_multi: systemRaws?.cpu_events_multi ?? null,
|
||||
cpu_benchmark_threads: systemRaws?.cpu_benchmark_threads ?? null,
|
||||
cpu_total_events: systemRaws?.cpu_total_events ?? null,
|
||||
cpu_total_time: systemRaws?.cpu_total_time ?? null,
|
||||
memory_ops_per_sec: systemRaws?.memory_ops_per_sec ?? null,
|
||||
memory_threads: systemRaws?.memory_threads ?? null,
|
||||
disk_read_mb_per_sec: systemRaws?.disk_read_mb_per_sec ?? null,
|
||||
disk_write_mb_per_sec: systemRaws?.disk_write_mb_per_sec ?? null,
|
||||
nomad_score_v2: nomadScoreV2,
|
||||
run_environment: env.run_environment,
|
||||
storage_path_type: env.storage_path_type,
|
||||
gpu_compute_detected: env.gpu_compute_detected,
|
||||
})
|
||||
|
||||
this._updateStatus('completed', 'Benchmark completed successfully')
|
||||
|
|
@ -483,21 +664,25 @@ export class BenchmarkService {
|
|||
}
|
||||
|
||||
/**
|
||||
* Run system benchmarks using sysbench in Docker
|
||||
* Run system benchmarks using sysbench in Docker.
|
||||
*
|
||||
* Produces both the legacy v1 0-1 sub-scores AND the v2 raw channels from the
|
||||
* same runs. CPU runs twice: 1 thread (single-core index) and all logical
|
||||
* threads (multi-core index); the v1 cpu_score keys off the multi pass to stay
|
||||
* comparable with historical results.
|
||||
*
|
||||
* @param multiThreads number of threads for the multi-core CPU pass (all logical CPUs)
|
||||
*/
|
||||
private async _runSystemBenchmarks(): Promise<SystemScores> {
|
||||
private async _runSystemBenchmarks(multiThreads: number): Promise<SystemBenchmarkOutput> {
|
||||
// Ensure sysbench image is available
|
||||
await this._ensureSysbenchImage()
|
||||
|
||||
// Run CPU benchmark
|
||||
this._updateStatus('running_cpu', 'Running CPU benchmark...')
|
||||
const cpuResult = await this._runSysbenchCpu()
|
||||
this._emitPartialResult({
|
||||
status: 'running_cpu',
|
||||
label: 'CPU',
|
||||
value: Math.round(cpuResult.events_per_second),
|
||||
unit: 'ev/s',
|
||||
})
|
||||
// Run CPU benchmarks: single-thread then all-thread.
|
||||
this._updateStatus('running_cpu', 'Running CPU benchmark (single-thread)...')
|
||||
const cpuSingle = await this._runSysbenchCpu(1)
|
||||
|
||||
this._updateStatus('running_cpu', `Running CPU benchmark (${multiThreads}-thread)...`)
|
||||
const cpuMulti = await this._runSysbenchCpu(multiThreads)
|
||||
|
||||
// Run memory benchmark
|
||||
this._updateStatus('running_memory', 'Running memory benchmark...')
|
||||
|
|
@ -528,13 +713,90 @@ export class BenchmarkService {
|
|||
unit: 'MB/s',
|
||||
})
|
||||
|
||||
// Normalize scores to 0-100 scale
|
||||
return {
|
||||
cpu_score: this._normalizeScore(cpuResult.events_per_second, REFERENCE_SCORES.cpu_events_per_second),
|
||||
// Legacy v1 sub-scores (0-1), normalized against the v1 references. cpu_score
|
||||
// uses the multi-thread pass to match historical (4-thread) behaviour closely.
|
||||
const scores: SystemScores = {
|
||||
cpu_score: this._normalizeScore(cpuMulti.events_per_second, REFERENCE_SCORES.cpu_events_per_second),
|
||||
memory_score: this._normalizeScore(memoryResult.operations_per_second, REFERENCE_SCORES.memory_ops_per_second),
|
||||
disk_read_score: this._normalizeScore(diskReadResult.read_mb_per_sec, REFERENCE_SCORES.disk_read_mb_per_sec),
|
||||
disk_write_score: this._normalizeScore(diskWriteResult.write_mb_per_sec, REFERENCE_SCORES.disk_write_mb_per_sec),
|
||||
}
|
||||
|
||||
// v2 raw channels — the payload of record for the leaderboard.
|
||||
const raws: SystemBenchmarkRawsV2 = {
|
||||
cpu_events_single: cpuSingle.events_per_second,
|
||||
cpu_events_multi: cpuMulti.events_per_second,
|
||||
cpu_benchmark_threads: multiThreads,
|
||||
cpu_total_events: cpuMulti.total_events,
|
||||
cpu_total_time: cpuMulti.total_time,
|
||||
memory_ops_per_sec: memoryResult.operations_per_second,
|
||||
memory_threads: MEMORY_BENCHMARK_THREADS,
|
||||
disk_read_mb_per_sec: diskReadResult.read_mb_per_sec,
|
||||
disk_write_mb_per_sec: diskWriteResult.write_mb_per_sec,
|
||||
}
|
||||
|
||||
return { scores, raws }
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe live NVIDIA GPU stats by exec-ing nvidia-smi inside the Ollama
|
||||
* container (same approach as SystemService.getNvidiaSmiInfo). Telemetry-only:
|
||||
* returns null on any failure (no Ollama container, no NVIDIA GPU / AMD,
|
||||
* unparseable output) and must never affect the scored benchmark numbers.
|
||||
*/
|
||||
private async _probeGpuStats(): Promise<NonNullable<BenchmarkTelemetry['gpu']> | null> {
|
||||
try {
|
||||
const containers = await this.dockerService.docker.listContainers({ all: false })
|
||||
const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
|
||||
if (!ollamaContainer) return null
|
||||
|
||||
const container = this.dockerService.docker.getContainer(ollamaContainer.Id)
|
||||
const exec = await container.exec({
|
||||
Cmd: [
|
||||
'nvidia-smi',
|
||||
'--query-gpu=utilization.gpu,memory.used,memory.total',
|
||||
'--format=csv,noheader,nounits',
|
||||
],
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
Tty: true,
|
||||
})
|
||||
|
||||
// Read the output stream with a timeout to prevent hanging if nvidia-smi fails
|
||||
const stream = await exec.start({ Tty: true })
|
||||
const output = await new Promise<string>((resolve) => {
|
||||
let data = ''
|
||||
const timeout = setTimeout(() => resolve(data), 3000)
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
data += chunk.toString()
|
||||
})
|
||||
stream.on('end', () => {
|
||||
clearTimeout(timeout)
|
||||
resolve(data)
|
||||
})
|
||||
})
|
||||
|
||||
// Remove any non-printable characters and trim the output
|
||||
const cleaned = Array.from(output)
|
||||
.filter((character) => character.charCodeAt(0) > 8)
|
||||
.join('')
|
||||
.trim()
|
||||
if (!cleaned || cleaned.toLowerCase().includes('error') || cleaned.toLowerCase().includes('not found')) {
|
||||
return null
|
||||
}
|
||||
|
||||
// First GPU only: "<util>, <used>, <total>"
|
||||
const parts = cleaned.split('\n')[0].split(',').map((s) => s.trim())
|
||||
if (parts.length < 3) return null
|
||||
const util = Number.parseInt(parts[0], 10)
|
||||
const used = Number.parseInt(parts[1], 10)
|
||||
const total = Number.parseInt(parts[2], 10)
|
||||
if (!Number.isFinite(util) || !Number.isFinite(used) || !Number.isFinite(total)) return null
|
||||
|
||||
return { util, vram_used_mb: used, vram_total_mb: total }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -656,10 +918,51 @@ export class BenchmarkService {
|
|||
throw new Error(`Model does not exist and failed to download: ${modelResponse.message}`)
|
||||
}
|
||||
|
||||
// Run inference benchmark. We stream the response so the live UI can show
|
||||
// tokens/sec climbing and stamp the true time-to-first-token, but the SCORED
|
||||
// numbers still come from Ollama's authoritative final eval_count /
|
||||
// eval_duration / prompt_eval_duration, exactly as the non-streaming path did.
|
||||
// Evict any other models resident in VRAM/RAM before benchmarking. With an 8B
|
||||
// reference model, a chat model left loaded can starve it of VRAM → Ollama
|
||||
// offloads layers to CPU → a low, non-comparable score. Give the benchmark
|
||||
// model the whole GPU. Best-effort: never fail the run on an eviction hiccup.
|
||||
await this._unloadResidentModels(ollamaAPIURL)
|
||||
|
||||
// Run inference AI_BENCHMARK_RUNS times and take the median run by tok/s (W7).
|
||||
// A single inference is noisy (scheduler, thermal, cache); the median damps
|
||||
// outliers without a warmup-run bias. TTFT is reported from the same run that
|
||||
// is the tok/s median, so the two figures always describe one real inference.
|
||||
const runs: { tps: number; ttft: number }[] = []
|
||||
for (let i = 0; i < AI_BENCHMARK_RUNS; i++) {
|
||||
this._updateStatus('running_ai', `Running AI benchmark (${i + 1}/${AI_BENCHMARK_RUNS})...`)
|
||||
runs.push(await this._runSingleAIInference(ollamaAPIURL))
|
||||
}
|
||||
|
||||
// Median by tok/s (odd count → middle element).
|
||||
runs.sort((a, b) => a.tps - b.tps)
|
||||
const median = runs[Math.floor(runs.length / 2)]
|
||||
|
||||
// Leave the GPU as clean as we found it: unload our benchmark model so the
|
||||
// user's next chat loads their model into a free GPU (Ollama reloads on demand;
|
||||
// we deliberately don't try to restore whichever model they had before).
|
||||
await this._unloadResidentModels(ollamaAPIURL)
|
||||
|
||||
return {
|
||||
ai_tokens_per_second: Math.round(median.tps * 100) / 100,
|
||||
ai_model_used: AI_BENCHMARK_MODEL,
|
||||
ai_time_to_first_token: Math.round(median.ttft * 100) / 100,
|
||||
ai_ollama_version: ollamaVersion,
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`AI benchmark failed: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single Ollama inference and return its tokens/sec and time-to-first-token
|
||||
* (TTFT in milliseconds). Extracted so the AI benchmark can run it N times for a
|
||||
* median. Prefers Ollama's eval_count/eval_duration; falls back to wall-clock
|
||||
* word-count estimation if those fields are absent.
|
||||
*/
|
||||
private async _runSingleAIInference(
|
||||
ollamaAPIURL: string
|
||||
): Promise<{ tps: number; ttft: number }> {
|
||||
const startTime = Date.now()
|
||||
let firstTokenAt: number | null = null
|
||||
let streamedTokens = 0
|
||||
|
|
@ -673,93 +976,106 @@ export class BenchmarkService {
|
|||
{
|
||||
model: AI_BENCHMARK_MODEL,
|
||||
prompt: AI_BENCHMARK_PROMPT,
|
||||
stream: true,
|
||||
stream: false,
|
||||
options: { num_predict: AI_BENCHMARK_NUM_PREDICT },
|
||||
},
|
||||
{ timeout: 120000, responseType: 'stream' }
|
||||
{ timeout: 120000 }
|
||||
)
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let buffer = ''
|
||||
let lastEmit = 0
|
||||
response.data.on('data', (chunk: Buffer) => {
|
||||
buffer += chunk.toString('utf8')
|
||||
// Ollama streams newline-delimited JSON; process each complete line.
|
||||
let newlineIdx: number
|
||||
while ((newlineIdx = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, newlineIdx).trim()
|
||||
buffer = buffer.slice(newlineIdx + 1)
|
||||
if (!line) continue
|
||||
try {
|
||||
const obj = JSON.parse(line)
|
||||
if (typeof obj.response === 'string' && obj.response.length > 0) {
|
||||
if (firstTokenAt === null) {
|
||||
firstTokenAt = Date.now()
|
||||
this.telemetry?.setStageMetric('tokens_per_sec', 0, firstTokenAt - startTime)
|
||||
}
|
||||
responseText += obj.response
|
||||
streamedTokens++
|
||||
const now = Date.now()
|
||||
if (firstTokenAt && now - lastEmit >= 400) {
|
||||
const elapsed = (now - firstTokenAt) / 1000
|
||||
const rate = elapsed > 0 ? streamedTokens / elapsed : 0
|
||||
this.telemetry?.setStageMetric(
|
||||
'tokens_per_sec',
|
||||
Math.round(rate * 10) / 10,
|
||||
firstTokenAt - startTime
|
||||
)
|
||||
lastEmit = now
|
||||
}
|
||||
}
|
||||
if (obj.done) {
|
||||
finalEvalCount = obj.eval_count || 0
|
||||
finalEvalDuration = obj.eval_duration || 0
|
||||
finalPromptEvalDuration = obj.prompt_eval_duration || 0
|
||||
}
|
||||
} catch {
|
||||
// Ignore a partial or non-JSON line; the next chunk completes it.
|
||||
}
|
||||
}
|
||||
})
|
||||
response.data.on('end', () => resolve())
|
||||
response.data.on('error', (err: Error) => reject(err))
|
||||
})
|
||||
const endTime = Date.now()
|
||||
const totalTime = (endTime - startTime) / 1000 // seconds
|
||||
|
||||
const totalTime = (Date.now() - startTime) / 1000 // seconds
|
||||
// Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds)
|
||||
if (response.data.eval_count && response.data.eval_duration) {
|
||||
const tokenCount = response.data.eval_count
|
||||
const evalDurationSeconds = response.data.eval_duration / 1e9
|
||||
const tokensPerSecond = tokenCount / evalDurationSeconds
|
||||
|
||||
// Authoritative scored numbers (identical to the previous non-streaming path).
|
||||
if (finalEvalCount && finalEvalDuration) {
|
||||
const evalDurationSeconds = finalEvalDuration / 1e9
|
||||
const tokensPerSecond = finalEvalCount / evalDurationSeconds
|
||||
const ttft = finalPromptEvalDuration
|
||||
? finalPromptEvalDuration / 1e6 // Convert ns to ms
|
||||
// Time to first token from prompt_eval_duration
|
||||
const ttft = response.data.prompt_eval_duration
|
||||
? response.data.prompt_eval_duration / 1e6 // Convert to ms
|
||||
: (totalTime * 1000) / 2 // Estimate if not available
|
||||
|
||||
return {
|
||||
ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
|
||||
ai_model_used: AI_BENCHMARK_MODEL,
|
||||
ai_time_to_first_token: Math.round(ttft * 100) / 100,
|
||||
ai_ollama_version: ollamaVersion,
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback if Ollama didn't return eval metrics: use the streamed counts.
|
||||
const estimatedTokens = streamedTokens || responseText.split(' ').length * 1.3 || 100
|
||||
const tokensPerSecond = totalTime > 0 ? estimatedTokens / totalTime : 0
|
||||
|
||||
return {
|
||||
ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
|
||||
ai_model_used: AI_BENCHMARK_MODEL,
|
||||
ai_time_to_first_token: firstTokenAt
|
||||
? Math.round(firstTokenAt - startTime)
|
||||
: Math.round((totalTime * 1000) / 2),
|
||||
ai_ollama_version: ollamaVersion,
|
||||
return { tps: tokensPerSecond, ttft }
|
||||
}
|
||||
|
||||
// Fallback calculation
|
||||
const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100
|
||||
const tokensPerSecond = estimatedTokens / totalTime
|
||||
return { tps: tokensPerSecond, ttft: (totalTime * 1000) / 2 }
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail fast if the AI model will need downloading and there isn't room for it.
|
||||
*
|
||||
* Only relevant on first run: if the model is already pulled, no download happens
|
||||
* and we skip the check. Best-effort throughout — if Ollama isn't reachable or free
|
||||
* space can't be determined, we do NOT block (the AI stage will surface its own
|
||||
* error), because a false "insufficient disk" is worse than letting the pull try.
|
||||
*/
|
||||
private async _preflightModelDiskSpace(): Promise<void> {
|
||||
let ollamaAPIURL: string | null = null
|
||||
try {
|
||||
ollamaAPIURL = await this.dockerService.getServiceURL(SERVICE_NAMES.OLLAMA)
|
||||
} catch {
|
||||
return // AI stage will report Ollama problems
|
||||
}
|
||||
if (!ollamaAPIURL) return
|
||||
|
||||
// If the model is already present, no download → no disk need.
|
||||
try {
|
||||
const tags = await axios.get(`${ollamaAPIURL}/api/tags`, { timeout: 5000 })
|
||||
const models: string[] = (tags.data?.models ?? []).map((m: { name?: string }) => m.name)
|
||||
if (models.includes(AI_BENCHMARK_MODEL)) return
|
||||
} catch {
|
||||
return // can't reach Ollama; let the AI stage handle it
|
||||
}
|
||||
|
||||
// Model not cached → a pull is coming. Require headroom on the root FS.
|
||||
let freeBytes: number | null = null
|
||||
try {
|
||||
const systemService = new (await import('./system_service.js')).SystemService(this.dockerService)
|
||||
freeBytes = await getFreeBytes(systemService)
|
||||
} catch {
|
||||
return // couldn't measure free space — don't block on uncertainty
|
||||
}
|
||||
if (freeBytes === null) return
|
||||
|
||||
if (freeBytes < AI_BENCHMARK_MODEL_MIN_FREE_BYTES) {
|
||||
const gib = (b: number) => `${(b / 1024 / 1024 / 1024).toFixed(1)} GB`
|
||||
throw new Error(
|
||||
`Not enough free disk space to download the AI benchmark model (${AI_BENCHMARK_MODEL}). ` +
|
||||
`About ${gib(AI_BENCHMARK_MODEL_MIN_FREE_BYTES)} is required but only ${gib(freeBytes)} is free. ` +
|
||||
`Free up some space and try again.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict every model currently resident in Ollama (VRAM/RAM) so the benchmark
|
||||
* model gets the whole GPU. Queries /api/ps for loaded models and unloads each
|
||||
* via a keep_alive:0 request. Entirely best-effort: any failure (older Ollama
|
||||
* without /api/ps, a model mid-stream) is logged and swallowed — a clean-VRAM
|
||||
* optimization must never block the benchmark itself.
|
||||
*/
|
||||
private async _unloadResidentModels(ollamaAPIURL: string): Promise<void> {
|
||||
try {
|
||||
const ps = await axios.get(`${ollamaAPIURL}/api/ps`, { timeout: 5000 })
|
||||
const loaded: string[] = (ps.data?.models ?? [])
|
||||
.map((m: { name?: string; model?: string }) => m.name || m.model)
|
||||
.filter((n: string | undefined): n is string => !!n)
|
||||
|
||||
if (loaded.length === 0) return
|
||||
|
||||
logger.info(`[BenchmarkService] Evicting ${loaded.length} resident model(s) before AI benchmark: ${loaded.join(', ')}`)
|
||||
for (const model of loaded) {
|
||||
// keep_alive:0 tells Ollama to unload immediately (no prompt needed).
|
||||
await axios
|
||||
.post(`${ollamaAPIURL}/api/generate`, { model, keep_alive: 0 }, { timeout: 15000 })
|
||||
.catch((e) => logger.warn(`[BenchmarkService] Failed to unload ${model}: ${e.message}`))
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`AI benchmark failed: ${error.message}`)
|
||||
} finally {
|
||||
// Stop the GPU-util poller and clear the overlay from telemetry frames.
|
||||
if (gpuPollTimer) clearInterval(gpuPollTimer)
|
||||
this.telemetry?.setGpuStats(null)
|
||||
logger.warn(`[BenchmarkService] Could not enumerate/unload resident models (continuing): ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -821,6 +1137,71 @@ export class BenchmarkService {
|
|||
return Math.round(Math.min(100, Math.max(0, nomadScore)) * 100) / 100
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the NOMAD Score v2 from raw channel values.
|
||||
*
|
||||
* Byte-identical to the leaderboard's computeNomadScoreV2 (score_service.ts):
|
||||
* the weighted geometric mean vs the frozen Reference Build, computed in the
|
||||
* log domain, rounded to one decimal. NO clamps. Throws if any channel is
|
||||
* <= 0 — a full v2 run measures every channel, and log2 of a non-positive
|
||||
* ratio is undefined, so a non-positive value here is a programming error, not
|
||||
* a user input to clamp. Every scored channel (incl. AI) is required; callers
|
||||
* only invoke this for a complete full-benchmark run.
|
||||
*/
|
||||
private _calculateNomadScoreV2(raws: ScoreRawsV2): number {
|
||||
let logSum = 0
|
||||
for (const channel of SCORE_CHANNELS_V2) {
|
||||
const raw = raws[channel]
|
||||
if (!(raw > 0)) {
|
||||
throw new Error(`_calculateNomadScoreV2: channel "${channel}" must be > 0 (got ${raw})`)
|
||||
}
|
||||
const ratio = raw / REFERENCE_SCORES_V2[channel]
|
||||
logSum += SCORE_WEIGHTS_V2[channel] * Math.log2(ratio)
|
||||
}
|
||||
const score = 1000 * Math.pow(2, logSum)
|
||||
// One decimal, matching the leaderboard's display convention.
|
||||
return Math.round(score * 10) / 10
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect best-effort run-environment metadata for the leaderboard (issue #1016).
|
||||
* Every probe is independently guarded — this never throws and never fails a run;
|
||||
* any field it can't determine stays null.
|
||||
*/
|
||||
private async _detectRunEnvironment(): Promise<RunEnvironmentInfo> {
|
||||
const info: RunEnvironmentInfo = {
|
||||
run_environment: null,
|
||||
storage_path_type: null,
|
||||
gpu_compute_detected: null,
|
||||
}
|
||||
|
||||
// run_environment: WSL2 vs native Linux, from the host kernel string.
|
||||
try {
|
||||
const procVersion = await readFile('/proc/version', 'utf8')
|
||||
info.run_environment = /microsoft|wsl/i.test(procVersion) ? 'wsl2' : 'native-linux'
|
||||
} catch {
|
||||
// /proc/version unreadable — leave null
|
||||
}
|
||||
|
||||
// storage_path_type: the Docker storage driver backing the benchmarked FS.
|
||||
try {
|
||||
const dockerInfo = await this.dockerService.docker.info()
|
||||
if (dockerInfo?.Driver) info.storage_path_type = String(dockerInfo.Driver)
|
||||
} catch {
|
||||
// docker info failed — leave null
|
||||
}
|
||||
|
||||
// gpu_compute_detected: whether a GPU compute type was detected + persisted.
|
||||
try {
|
||||
const gpuType = await KVStore.getValue('gpu.type')
|
||||
info.gpu_compute_detected = gpuType === 'nvidia' || gpuType === 'amd'
|
||||
} catch {
|
||||
// KV read failed — leave null
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a raw score to 0-100 scale using log scaling
|
||||
* This provides diminishing returns for very high scores
|
||||
|
|
@ -858,34 +1239,27 @@ export class BenchmarkService {
|
|||
}
|
||||
|
||||
/**
|
||||
* Run sysbench CPU benchmark
|
||||
* Run sysbench CPU benchmark at the given thread count.
|
||||
*
|
||||
* v2 runs this twice: 1 thread (single-core index) and all threads (multi-core
|
||||
* index). The multi pass also supplies the W6 consistency companions
|
||||
* (total_events / total_time) the leaderboard cross-checks against events/sec.
|
||||
*/
|
||||
private async _runSysbenchCpu(): Promise<SysbenchCpuResult> {
|
||||
// --report-interval=1 emits interim lines like:
|
||||
// [ 1s ] thds: 4 eps: 7589.48 lat (ms,95%): 0.60
|
||||
// They drive live telemetry only; the SCORED numbers still come from the
|
||||
// final report parsed below, exactly as before.
|
||||
const output = await this._runSysbenchCommandStreaming(
|
||||
[
|
||||
'sysbench',
|
||||
'cpu',
|
||||
'--cpu-max-prime=20000',
|
||||
'--threads=4',
|
||||
'--time=30',
|
||||
'--report-interval=1',
|
||||
'run',
|
||||
],
|
||||
(line) => {
|
||||
const m = line.match(/eps:\s*([\d.]+)/)
|
||||
if (m) this.telemetry?.setStageMetric('events_per_sec', parseFloat(m[1]))
|
||||
}
|
||||
)
|
||||
private async _runSysbenchCpu(threads: number): Promise<SysbenchCpuResult> {
|
||||
const output = await this._runSysbenchCommand([
|
||||
'sysbench',
|
||||
'cpu',
|
||||
'--cpu-max-prime=20000',
|
||||
`--threads=${threads}`,
|
||||
'--time=30',
|
||||
'run',
|
||||
])
|
||||
|
||||
// Parse output for events per second
|
||||
const eventsMatch = output.match(/events per second:\s*([\d.]+)/i)
|
||||
const totalTimeMatch = output.match(/total time:\s*([\d.]+)s/i)
|
||||
const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i)
|
||||
logger.debug(`[BenchmarkService] CPU output parsing - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`)
|
||||
logger.debug(`[BenchmarkService] CPU output parsing (${threads}T) - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`)
|
||||
|
||||
// Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
|
||||
const eventsPerSecond = eventsMatch ? parseFloat(eventsMatch[1]) : Number.NaN
|
||||
|
|
@ -893,10 +1267,21 @@ export class BenchmarkService {
|
|||
throw new Error('sysbench CPU benchmark produced no parseable events/sec — aborting (output may have changed or the test failed)')
|
||||
}
|
||||
|
||||
const totalTime = totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30
|
||||
// total_events feeds the leaderboard's W6 cross-check (events/sec must equal
|
||||
// total_events/total_time within 2%). If the count line ever fails to parse,
|
||||
// derive it from the rate so we submit a self-consistent payload rather than
|
||||
// a zero that would be rejected as "internally inconsistent".
|
||||
const parsedTotalEvents = totalEventsMatch ? parseInt(totalEventsMatch[1]) : Number.NaN
|
||||
const totalEvents =
|
||||
Number.isFinite(parsedTotalEvents) && parsedTotalEvents > 0
|
||||
? parsedTotalEvents
|
||||
: Math.round(eventsPerSecond * totalTime)
|
||||
|
||||
return {
|
||||
events_per_second: eventsPerSecond,
|
||||
total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30,
|
||||
total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0,
|
||||
total_time: totalTime,
|
||||
total_events: totalEvents,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -909,7 +1294,7 @@ export class BenchmarkService {
|
|||
'memory',
|
||||
'--memory-block-size=1K',
|
||||
'--memory-total-size=10G',
|
||||
'--threads=4',
|
||||
`--threads=${MEMORY_BENCHMARK_THREADS}`,
|
||||
'run',
|
||||
])
|
||||
|
||||
|
|
@ -937,23 +1322,16 @@ export class BenchmarkService {
|
|||
private async _runSysbenchDiskRead(): Promise<SysbenchDiskResult> {
|
||||
// Run prepare, test, and cleanup in a single container
|
||||
// This is necessary because each container has its own filesystem
|
||||
// --report-interval=1 (run step only) emits interim lines like:
|
||||
// [ 1s ] reads: 216.75 MiB/s writes: 0.00 MiB/s fsyncs: 551.89/s ...
|
||||
// They drive live telemetry only; the SCORED numbers still come from the
|
||||
// final report parsed below, exactly as before.
|
||||
const output = await this._runSysbenchCommandStreaming(
|
||||
[
|
||||
'sh',
|
||||
'-c',
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --time=30 --report-interval=1 run && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||||
],
|
||||
(line) => {
|
||||
const m = line.match(/reads:\s*([\d.]+)\s*MiB\/s/)
|
||||
if (m) this.telemetry?.setStageMetric('mib_s', parseFloat(m[1]))
|
||||
}
|
||||
)
|
||||
// --file-extra-flags=direct (O_DIRECT) on the run bypasses the page cache so
|
||||
// we measure real device throughput, not RAM (W4). Only the run needs it;
|
||||
// prepare/cleanup are plain file ops. v2 reference disk numbers are O_DIRECT.
|
||||
const output = await this._runSysbenchCommand([
|
||||
'sh',
|
||||
'-c',
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --file-extra-flags=direct --time=30 run && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||||
])
|
||||
|
||||
// Parse output - look for the Throughput section
|
||||
const readMatch = output.match(/read,\s*MiB\/s:\s*([\d.]+)/i)
|
||||
|
|
@ -982,21 +1360,14 @@ export class BenchmarkService {
|
|||
private async _runSysbenchDiskWrite(): Promise<SysbenchDiskResult> {
|
||||
// Run prepare, test, and cleanup in a single container
|
||||
// This is necessary because each container has its own filesystem
|
||||
// --report-interval=1 (run step only) drives live telemetry; the SCORED
|
||||
// numbers still come from the final report parsed below, exactly as before.
|
||||
const output = await this._runSysbenchCommandStreaming(
|
||||
[
|
||||
'sh',
|
||||
'-c',
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --time=30 --report-interval=1 run && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||||
],
|
||||
(line) => {
|
||||
const m = line.match(/writes:\s*([\d.]+)\s*MiB\/s/)
|
||||
if (m) this.telemetry?.setStageMetric('mib_s', parseFloat(m[1]))
|
||||
}
|
||||
)
|
||||
// --file-extra-flags=direct (O_DIRECT) on the run bypasses the page cache (W4).
|
||||
const output = await this._runSysbenchCommand([
|
||||
'sh',
|
||||
'-c',
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --file-extra-flags=direct --time=30 run && ' +
|
||||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||||
])
|
||||
|
||||
// Parse output - look for the Throughput section
|
||||
const writeMatch = output.match(/written,\s*MiB\/s:\s*([\d.]+)/i)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'benchmark_results'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
// NOMAD Score v2 (Phase 4): raw channel values the leaderboard scores from,
|
||||
// the frozen test parameters, the W6 consistency companions, the uncapped
|
||||
// score, and best-effort run-environment metadata (#1016). All nullable —
|
||||
// pre-v2 rows and system-only runs simply leave them empty.
|
||||
// double (not float): Knex's MySQL float is float(8,2) — max 999999.99 and
|
||||
// rounded to 2 decimals. memory_ops_per_sec / cpu_total_events run into the
|
||||
// millions, and rounding raws would break byte-match with the leaderboard's
|
||||
// server-side score recompute. double holds full-precision large values.
|
||||
table.double('cpu_events_single').nullable()
|
||||
table.double('cpu_events_multi').nullable()
|
||||
table.integer('cpu_benchmark_threads').nullable()
|
||||
table.double('cpu_total_events').nullable()
|
||||
table.double('cpu_total_time').nullable()
|
||||
table.double('memory_ops_per_sec').nullable()
|
||||
table.integer('memory_threads').nullable()
|
||||
table.double('disk_read_mb_per_sec').nullable()
|
||||
table.double('disk_write_mb_per_sec').nullable()
|
||||
table.double('nomad_score_v2').nullable()
|
||||
table.string('run_environment').nullable()
|
||||
table.string('storage_path_type').nullable()
|
||||
table.boolean('gpu_compute_detected').nullable()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.dropColumn('cpu_events_single')
|
||||
table.dropColumn('cpu_events_multi')
|
||||
table.dropColumn('cpu_benchmark_threads')
|
||||
table.dropColumn('cpu_total_events')
|
||||
table.dropColumn('cpu_total_time')
|
||||
table.dropColumn('memory_ops_per_sec')
|
||||
table.dropColumn('memory_threads')
|
||||
table.dropColumn('disk_read_mb_per_sec')
|
||||
table.dropColumn('disk_write_mb_per_sec')
|
||||
table.dropColumn('nomad_score_v2')
|
||||
table.dropColumn('run_environment')
|
||||
table.dropColumn('storage_path_type')
|
||||
table.dropColumn('gpu_compute_detected')
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -93,7 +93,17 @@ export default function BenchmarkPage(props: {
|
|||
setErrorMsg(null)
|
||||
run.reset()
|
||||
setIsRunning(true)
|
||||
return await api.runBenchmark(type)
|
||||
setProgress({
|
||||
status: 'starting',
|
||||
progress: 5,
|
||||
message: 'Starting benchmark... This takes 3-6 minutes.',
|
||||
current_stage: 'Starting',
|
||||
benchmark_id: '',
|
||||
timestamp: new Date().toISOString(),
|
||||
})
|
||||
|
||||
// Use sync mode - runs inline without needing Redis/queue worker
|
||||
return await api.runBenchmark(type, true)
|
||||
},
|
||||
onSuccess: (data) => {
|
||||
// Dispatch only confirms the job started; the 'completed'/'error' SSE event
|
||||
|
|
@ -279,7 +289,7 @@ export default function BenchmarkPage(props: {
|
|||
)}
|
||||
<p className="text-desert-stone-dark">
|
||||
Run a benchmark to measure your system's CPU, memory, disk, and AI inference
|
||||
performance. The benchmark takes approximately 2-5 minutes to complete.
|
||||
performance. The benchmark takes approximately 3-6 minutes to complete.
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-4">
|
||||
<StyledButton
|
||||
|
|
@ -348,7 +358,7 @@ export default function BenchmarkPage(props: {
|
|||
<div className="shrink-0">
|
||||
<CircularGauge
|
||||
value={latestResult.nomad_score}
|
||||
label={scoreInfo?.label ?? 'NOMAD Score'}
|
||||
label={latestResult.nomad_score_v2 != null ? 'Legacy Score' : 'NOMAD Score'}
|
||||
size="lg"
|
||||
variant="cpu"
|
||||
subtext="out of 100"
|
||||
|
|
@ -357,27 +367,49 @@ export default function BenchmarkPage(props: {
|
|||
/>
|
||||
</div>
|
||||
<div className="flex-1 space-y-4">
|
||||
<div className="flex items-center gap-3">
|
||||
{latestResult.nomad_score_v2 != null ? (
|
||||
<>
|
||||
<div className="flex items-baseline gap-3">
|
||||
<div className="text-5xl font-bold text-desert-green">
|
||||
{latestResult.nomad_score_v2.toFixed(1)}
|
||||
</div>
|
||||
<div className="text-sm text-desert-stone-dark flex items-center gap-1">
|
||||
NOMAD Score
|
||||
<InfoTooltip text="NOMAD Score v2 is an uncapped index versus the NOMAD Reference Build, which scores exactly 1000. Higher is better, and there is no ceiling." />
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-sm text-desert-stone-dark">
|
||||
Reference Build = 1000.{' '}
|
||||
<span className="text-desert-stone">
|
||||
Legacy scale: {latestResult.nomad_score.toFixed(1)} / 100
|
||||
</span>
|
||||
</p>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<div className="flex items-center gap-3">
|
||||
<div
|
||||
className={`text-5xl font-bold ${
|
||||
className={`text-5xl font-bold ${
|
||||
scoreInfo?.isPartial
|
||||
? 'text-desert-stone-dark'
|
||||
: getScoreColor(latestResult.nomad_score)
|
||||
}`}
|
||||
>
|
||||
{latestResult.nomad_score.toFixed(1)}
|
||||
</div>
|
||||
>
|
||||
{latestResult.nomad_score.toFixed(1)}
|
||||
</div>
|
||||
{scoreInfo?.isPartial && (
|
||||
<span className="px-2 py-1 rounded-md bg-desert-stone-light text-desert-stone-dark text-xs font-semibold uppercase tracking-wide">
|
||||
Partial
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-desert-stone-dark">
|
||||
{scoreInfo?.isPartial
|
||||
<p className="text-desert-stone-dark">
|
||||
{scoreInfo?.isPartial
|
||||
? scoreInfo.cta
|
||||
: 'Your NOMAD Score is a weighted composite of all benchmark results.'}
|
||||
</p>
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Share with Community - Only for full benchmarks with AI data */}
|
||||
{canShareBenchmark && (
|
||||
|
|
@ -644,7 +676,7 @@ export default function BenchmarkPage(props: {
|
|||
<div>
|
||||
<div className="text-desert-stone-dark">NOMAD Score</div>
|
||||
<div className="font-bold text-desert-green">
|
||||
{latestResult.nomad_score.toFixed(1)}
|
||||
{(latestResult.nomad_score_v2 ?? latestResult.nomad_score).toFixed(1)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -755,6 +787,101 @@ export default function BenchmarkPage(props: {
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* v2 raw measurements + run environment */}
|
||||
{latestResult.cpu_events_multi != null && (
|
||||
<div className="mt-6 pt-6 border-t border-desert-stone-light grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
<div>
|
||||
<h4 className="font-semibold text-desert-green mb-3">
|
||||
Measured Performance (v2)
|
||||
</h4>
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">CPU single-thread</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.cpu_events_single?.toFixed(1)} events/s
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">
|
||||
CPU multi-thread ({latestResult.cpu_benchmark_threads}T)
|
||||
</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.cpu_events_multi?.toFixed(1)} events/s
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">
|
||||
Memory ({latestResult.memory_threads}T)
|
||||
</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.memory_ops_per_sec?.toLocaleString()} ops/s
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">
|
||||
Disk read (O_DIRECT)
|
||||
</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.disk_read_mb_per_sec?.toFixed(1)} MB/s
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">
|
||||
Disk write (O_DIRECT)
|
||||
</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.disk_write_mb_per_sec?.toFixed(1)} MB/s
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="font-semibold text-desert-green mb-3">Environment</h4>
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Run environment</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.run_environment || 'Unknown'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Storage backend</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.storage_path_type || 'Unknown'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">GPU compute</span>
|
||||
<span className="font-mono">
|
||||
{latestResult.gpu_compute_detected == null
|
||||
? 'Unknown'
|
||||
: latestResult.gpu_compute_detected
|
||||
? 'Detected'
|
||||
: 'Not detected'}
|
||||
</span>
|
||||
</div>
|
||||
{latestResult.sysbench_digest && (
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">sysbench</span>
|
||||
<span className="font-mono text-xs">
|
||||
{latestResult.sysbench_digest.replace('sha256:', '').slice(0, 12)}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{latestResult.ollama_version && (
|
||||
<div className="flex justify-between">
|
||||
<span className="text-desert-stone-dark">Ollama</span>
|
||||
<span className="font-mono text-xs">
|
||||
{latestResult.ollama_version}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -160,6 +160,37 @@ export type UpdateBuilderTagResponse = {
|
|||
error: string
|
||||
}
|
||||
|
||||
// NOMAD Score v2 raw channels captured from a full benchmark run (all present
|
||||
// and > 0). cpu_events_multi is measured at cpu_benchmark_threads; memory at
|
||||
// memory_threads; disk figures are O_DIRECT MB/s. total_events/total_time are the
|
||||
// W6 consistency companions from the multi-thread CPU pass.
|
||||
export type SystemBenchmarkRawsV2 = {
|
||||
cpu_events_single: number
|
||||
cpu_events_multi: number
|
||||
cpu_benchmark_threads: number
|
||||
cpu_total_events: number
|
||||
cpu_total_time: number
|
||||
memory_ops_per_sec: number
|
||||
memory_threads: number
|
||||
disk_read_mb_per_sec: number
|
||||
disk_write_mb_per_sec: number
|
||||
}
|
||||
|
||||
// Result of a system benchmark pass: the legacy 0-1 sub-scores (v1) plus the v2
|
||||
// raw channels, both derived from the same sysbench runs.
|
||||
export type SystemBenchmarkOutput = {
|
||||
scores: SystemScores
|
||||
raws: SystemBenchmarkRawsV2
|
||||
}
|
||||
|
||||
// Best-effort run-environment metadata (issue #1016). Any field may be null when
|
||||
// detection fails; none of them gate a submission.
|
||||
export type RunEnvironmentInfo = {
|
||||
run_environment: string | null
|
||||
storage_path_type: string | null
|
||||
gpu_compute_detected: boolean | null
|
||||
}
|
||||
|
||||
// Central repository submission payload (privacy-first)
|
||||
export type RepositorySubmission = Pick<
|
||||
BenchmarkResult,
|
||||
|
|
@ -182,6 +213,46 @@ export type RepositorySubmission = Pick<
|
|||
builder_tag: string | null // null = anonymous submission
|
||||
}
|
||||
|
||||
// NOMAD Score v2 submission payload. Mirrors the leaderboard's submitValidatorV2
|
||||
// exactly: raw channels in (server recomputes the score), required test params +
|
||||
// W6 companions + provenance, optional environment metadata. ai_time_to_first_token
|
||||
// is in SECONDS here (the server treats it as seconds); the client stores TTFT in
|
||||
// ms, so submitToRepository divides by 1000.
|
||||
export type RepositorySubmissionV2 = {
|
||||
// Hardware
|
||||
cpu_model: string
|
||||
cpu_cores: number
|
||||
cpu_threads: number
|
||||
ram_gb: number
|
||||
disk_type: DiskType
|
||||
gpu_model: string | null
|
||||
// Scored raw channels (all required, > 0)
|
||||
ai_tokens_per_second: number
|
||||
cpu_events_single: number
|
||||
cpu_events_multi: number
|
||||
memory_ops_per_sec: number
|
||||
disk_read_mb_per_sec: number
|
||||
disk_write_mb_per_sec: number
|
||||
// Test parameters
|
||||
cpu_benchmark_threads: number
|
||||
memory_threads: number
|
||||
// Metadata channel (weight 0) + W6 consistency companions
|
||||
ai_time_to_first_token: number // seconds
|
||||
cpu_total_events: number
|
||||
cpu_total_time: number
|
||||
// Provenance (required)
|
||||
ollama_version: string
|
||||
sysbench_digest: string
|
||||
// Environment metadata (best-effort)
|
||||
run_environment?: string
|
||||
storage_path_type?: string
|
||||
gpu_compute_detected?: boolean
|
||||
// Benchmark metadata (shared with v1)
|
||||
nomad_version: string
|
||||
benchmark_version: string
|
||||
builder_tag?: string
|
||||
}
|
||||
|
||||
// Central repository response types
|
||||
export type RepositorySubmitResponse = {
|
||||
success: boolean
|
||||
|
|
|
|||
Loading…
Reference in New Issue