diff --git a/admin/app/models/benchmark_result.ts b/admin/app/models/benchmark_result.ts index c1b1481..656a9d9 100644 --- a/admin/app/models/benchmark_result.ts +++ b/admin/app/models/benchmark_result.ts @@ -63,6 +63,60 @@ export default class BenchmarkResult extends BaseModel { @column() declare ollama_version: string | null + // NOMAD Score v2 raw channels (nullable — added in Score v2 Phase 4). Populated + // on full benchmarks under benchmark_version >= 2.0.0; the leaderboard recomputes + // the score from these on submit. cpu_events_multi is measured at + // cpu_benchmark_threads; memory_ops_per_sec at memory_threads. Disk figures are + // O_DIRECT MB/s. cpu_total_events/cpu_total_time are the W6 consistency companions. + @column() + declare cpu_events_single: number | null + + @column() + declare cpu_events_multi: number | null + + @column() + declare cpu_benchmark_threads: number | null + + @column() + declare cpu_total_events: number | null + + @column() + declare cpu_total_time: number | null + + @column() + declare memory_ops_per_sec: number | null + + @column() + declare memory_threads: number | null + + @column() + declare disk_read_mb_per_sec: number | null + + @column() + declare disk_write_mb_per_sec: number | null + + // Uncapped NOMAD Score v2 (null for pre-v2 rows and system-only runs). The + // legacy nomad_score below is retained in parallel for display continuity. + // columnName + serializeAs pinned: the snake_case strategy would otherwise map + // this property to `nomad_score_v_2` (splitting the digit) for both the DB column + // (which the migration doesn't create) and the JSON key (which the frontend reads + // as nomad_score_v2). Pin both so DB, API, and UI all agree. + @column({ columnName: 'nomad_score_v2', serializeAs: 'nomad_score_v2' }) + declare nomad_score_v2: number | null + + // Best-effort run environment metadata (issue #1016) + @column() + declare run_environment: string | null + + @column() + declare storage_path_type: string | null + + @column({ + // Nullable tri-state; coerce the SQLite 0/1 int to a real boolean when present. + consume: (value: number | null) => (value === null || value === undefined ? null : Boolean(value)), + }) + declare gpu_compute_detected: boolean | null + // Composite NOMAD score (0-100) @column() declare nomad_score: number diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index 2c7eddf..770c902 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -18,10 +18,16 @@ import type { SysbenchCpuResult, SysbenchMemoryResult, SysbenchDiskResult, - RepositorySubmission, + RepositorySubmissionV2, RepositorySubmitResponse, RepositoryStats, + SystemBenchmarkOutput, + SystemBenchmarkRawsV2, + RunEnvironmentInfo, } from '../../types/benchmark.js' +import KVStore from '#models/kv_store' +import { getFreeBytes } from '../utils/image_disk_preflight.js' +import { readFile } from 'node:fs/promises' import { randomUUID, createHmac } from 'node:crypto' import { DockerService } from './docker_service.js' import { SERVICE_NAMES } from '../../constants/service_names.js' @@ -44,15 +50,87 @@ const SCORE_WEIGHTS = { disk_write: 0.10, } +// The benchmark_version this client emits. Server dispatches on the major: +// >= 2 → raw-channel v2 ingest (score recomputed server-side). Bumped 1.0.0 -> 2.0.0 +// for the NOMAD Score v2 rollout (uncapped index vs a frozen Reference Build). +const BENCHMARK_VERSION = '2.0.0' + +// --------------------------------------------------------------------------- +// NOMAD Score v2 — the uncapped index. +// +// Score = 1000 × Π( (raw_i / ref_i) ^ w_i ) (weighted geometric mean, log domain). +// +// These constants + the computation below MUST stay byte-identical to the +// leaderboard's score_service.ts (computeNomadScoreV2): the client shows this +// score immediately after a run, and the server recomputes the same value from +// the raw channels on submit. Any drift makes the displayed score disagree with +// the leaderboard. FROZEN under benchmark_version 2.0.0 — Reference Build = +// NOMAD6 (Ryzen 9 PRO 8945HS / 780M / 64GB, kernel 6.17), measured median-of-N +// with direct-I/O disk 2026-07-12. There are NO clamps here; outlier control is +// the server's rejection gates + quarantine, not the score math. +const REFERENCE_SCORES_V2 = { + // TODO(score-v2): STALE — 40.5 was measured on llama3.2:1b. The reference model + // is now llama3.1:8b; re-measure this on the Reference Build (NOMAD6) with the + // v2 harness (unload + num_predict + median-of-3) and update here AND in the + // leaderboard's score_service.ts (must byte-match) before v2 ships. + ai_tokens_per_second: 40.5, + cpu_events_multi: 19231, // all-threads on the reference box (16T) + cpu_events_single: 2351, // 1T + memory_ops_per_sec: 26862960, // 4T, 1K block (bandwidth peak) + disk_read_mb_per_sec: 1072, // O_DIRECT + disk_write_mb_per_sec: 316, // O_DIRECT +} as const + +const SCORE_WEIGHTS_V2 = { + ai_tokens_per_second: 0.3, + cpu_events_multi: 0.25, + cpu_events_single: 0.1, + memory_ops_per_sec: 0.1, + disk_read_mb_per_sec: 0.125, + disk_write_mb_per_sec: 0.125, +} as const + +type ScoreChannelV2 = keyof typeof REFERENCE_SCORES_V2 +type ScoreRawsV2 = Record +const SCORE_CHANNELS_V2 = Object.keys(REFERENCE_SCORES_V2) as ScoreChannelV2[] + +// Weights sum to 1.000; guard against a future typo silently rescaling scores. +const WEIGHT_SUM_V2 = SCORE_CHANNELS_V2.reduce((s, k) => s + SCORE_WEIGHTS_V2[k], 0) +if (Math.abs(WEIGHT_SUM_V2 - 1) > 1e-9) { + throw new Error(`SCORE_WEIGHTS_V2 must sum to 1.0 (got ${WEIGHT_SUM_V2})`) +} + // Benchmark configuration constants // Pinned by digest (was severalnines/sysbench:latest) so a latest-tag format change can't silently break the parsers fleet-wide. Digest validated on the NOMAD6 reference build 2026-07-12. const SYSBENCH_IMAGE = 'severalnines/sysbench@sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd' const SYSBENCH_DIGEST = 'sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd' const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench' -// Reference model for AI benchmark - small but meaningful -const AI_BENCHMARK_MODEL = 'llama3.2:1b' +// Reference model for AI benchmark. v2 uses an 8B (was llama3.2:1b): a 1B is so +// light it runs overhead-bound on any GPU (~240 tok/s), saturating the score and +// failing to differentiate hardware. An 8B is compute/bandwidth-bound and scales +// with real GPU capability. Non-thinking (deterministic workload). Changing this +// re-baselines REFERENCE_SCORES_V2.ai_tokens_per_second — re-measure on the +// Reference Build (NOMAD6) before shipping. +const AI_BENCHMARK_MODEL = 'llama3.1:8b' const AI_BENCHMARK_PROMPT = 'Explain recursion in programming in exactly 100 words.' +// Cap generation so the workload is bounded and comparable across machines rather +// than relying on the model honoring "100 words". ~100-150 tokens is plenty for a +// stable tok/s rate; the cap only trims pathological long outputs. +const AI_BENCHMARK_NUM_PREDICT = 256 + +// Memory test is frozen at 4 threads for the v2 Reference Build (the bandwidth +// peak on the reference box). Do not vary by core count — the reference number +// (REFERENCE_SCORES_V2.memory_ops_per_sec) was measured at exactly this value. +const MEMORY_BENCHMARK_THREADS = 4 + +// AI is measured as the median of N runs (W7) to damp per-run inference jitter. +const AI_BENCHMARK_RUNS = 3 + +// Minimum free disk required before pulling the AI model on first run. llama3.1:8b +// (Q4) is ~4.9 GB; this covers the model plus headroom for the transient sysbench +// disk-test file and normal slack. Only enforced when the model isn't already cached. +const AI_BENCHMARK_MODEL_MIN_FREE_BYTES = 6.5 * 1024 * 1024 * 1024 // Reference scores for normalization (calibrated to 0-100 scale) // These represent "expected" scores for a mid-range system (score ~50) @@ -139,24 +217,9 @@ export class BenchmarkService { throw new Error('Benchmark result has already been submitted') } - const submission: RepositorySubmission = { - cpu_model: result.cpu_model, - cpu_cores: result.cpu_cores, - cpu_threads: result.cpu_threads, - ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)), - disk_type: result.disk_type, - gpu_model: result.gpu_model, - cpu_score: result.cpu_score, - memory_score: result.memory_score, - disk_read_score: result.disk_read_score, - disk_write_score: result.disk_write_score, - ai_tokens_per_second: result.ai_tokens_per_second, - ai_time_to_first_token: result.ai_time_to_first_token, - nomad_score: result.nomad_score, - nomad_version: SystemService.getAppVersion(), - benchmark_version: '1.0.0', - builder_tag: anonymous ? null : result.builder_tag, - } + // Build the v2 payload (raw channels; server recomputes the score). Throws a + // user-facing error if the row predates v2 and lacks the raw channels. + const submission = this._buildV2Submission(result, anonymous ?? false) try { // Generate HMAC signature for submission verification @@ -200,6 +263,85 @@ export class BenchmarkService { } } + /** + * Build the NOMAD Score v2 submission payload from a stored result, validating + * that every required raw channel + companion + provenance field is present. + * + * A row produced before this benchmark_version (or an interrupted run) won't + * carry the v2 raws — rather than send a payload the leaderboard will reject + * with a terse validator error, we throw a clear "re-run" message here. + * + * ai_time_to_first_token is stored in milliseconds but the leaderboard's v2 API + * treats it as SECONDS, so it is divided by 1000 here. + */ + private _buildV2Submission(result: BenchmarkResult, anonymous: boolean): RepositorySubmissionV2 { + const reRun = 'Re-run a Full Benchmark to share your results.' + + // Required raw channels — all must be present and > 0. + const requiredRaws: Array<[string, number | null]> = [ + ['CPU single-thread', result.cpu_events_single], + ['CPU multi-thread', result.cpu_events_multi], + ['memory', result.memory_ops_per_sec], + ['disk read', result.disk_read_mb_per_sec], + ['disk write', result.disk_write_mb_per_sec], + ] + for (const [label, value] of requiredRaws) { + if (value == null || !(value > 0)) { + throw new Error(`This benchmark is missing the ${label} raw measurement needed for the v2 leaderboard. ${reRun}`) + } + } + + if (result.ai_time_to_first_token == null || !(result.ai_time_to_first_token > 0)) { + throw new Error(`This benchmark is missing AI time-to-first-token data. ${reRun}`) + } + if ( + result.cpu_benchmark_threads == null || + result.memory_threads == null || + result.cpu_total_events == null || + result.cpu_total_time == null + ) { + throw new Error(`This benchmark is missing CPU test parameters needed for the v2 leaderboard. ${reRun}`) + } + if (!result.sysbench_digest || !result.ollama_version) { + throw new Error(`This benchmark is missing harness provenance (sysbench/Ollama version). ${reRun}`) + } + + const submission: RepositorySubmissionV2 = { + cpu_model: result.cpu_model, + cpu_cores: result.cpu_cores, + cpu_threads: result.cpu_threads, + ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)), + disk_type: result.disk_type, + gpu_model: result.gpu_model, + ai_tokens_per_second: result.ai_tokens_per_second!, + cpu_events_single: result.cpu_events_single!, + cpu_events_multi: result.cpu_events_multi!, + memory_ops_per_sec: result.memory_ops_per_sec!, + disk_read_mb_per_sec: result.disk_read_mb_per_sec!, + disk_write_mb_per_sec: result.disk_write_mb_per_sec!, + cpu_benchmark_threads: result.cpu_benchmark_threads, + memory_threads: result.memory_threads, + // Stored in ms; the v2 API expects seconds. + ai_time_to_first_token: result.ai_time_to_first_token / 1000, + cpu_total_events: result.cpu_total_events, + cpu_total_time: result.cpu_total_time, + ollama_version: result.ollama_version, + sysbench_digest: result.sysbench_digest, + nomad_version: SystemService.getAppVersion(), + benchmark_version: BENCHMARK_VERSION, + } + + // Optional environment metadata — only send fields we actually have. + if (result.run_environment) submission.run_environment = result.run_environment + if (result.storage_path_type) submission.storage_path_type = result.storage_path_type + if (result.gpu_compute_detected != null) submission.gpu_compute_detected = result.gpu_compute_detected + + // Builder tag: omit entirely when anonymous or unset (validator rejects null). + if (!anonymous && result.builder_tag) submission.builder_tag = result.builder_tag + + return submission + } + /** * Get comparison stats from central repository */ @@ -362,6 +504,12 @@ export class BenchmarkService { this._updateStatus('starting', 'Starting benchmark...') try { + // Fail fast: if this run will download the (large) AI model, make sure there's + // room BEFORE spending minutes on the system benchmarks. + if (includeAI && (type === 'full' || type === 'ai')) { + await this._preflightModelDiskSpace() + } + // Detect hardware const hardware = await this.getHardwareInfo() @@ -372,9 +520,12 @@ export class BenchmarkService { disk_read_score: 0, disk_write_score: 0, } + let systemRaws: SystemBenchmarkRawsV2 | null = null if (type === 'full' || type === 'system') { - systemScores = await this._runSystemBenchmarks() + const systemOutput = await this._runSystemBenchmarks(hardware.cpu_threads) + systemScores = systemOutput.scores + systemRaws = systemOutput.raws } // Run AI benchmark if requested and Ollama is available @@ -392,10 +543,33 @@ export class BenchmarkService { } } - // Calculate NOMAD score + // Calculate NOMAD scores (v1 legacy + v2 uncapped) this._updateStatus('calculating_score', 'Calculating NOMAD score...') const nomadScore = this._calculateNomadScore(systemScores, aiScores) + // v2 requires a complete run of every channel (system raws + AI). Only a + // full benchmark with AI qualifies; otherwise nomad_score_v2 stays null. + let nomadScoreV2: number | null = null + if (systemRaws && aiScores.ai_tokens_per_second && aiScores.ai_tokens_per_second > 0) { + try { + nomadScoreV2 = this._calculateNomadScoreV2({ + ai_tokens_per_second: aiScores.ai_tokens_per_second, + cpu_events_single: systemRaws.cpu_events_single, + cpu_events_multi: systemRaws.cpu_events_multi, + memory_ops_per_sec: systemRaws.memory_ops_per_sec, + disk_read_mb_per_sec: systemRaws.disk_read_mb_per_sec, + disk_write_mb_per_sec: systemRaws.disk_write_mb_per_sec, + }) + } catch (error) { + // A non-positive channel shouldn't reach here (each sysbench step throws + // on a bad parse), but never let a v2 math error sink the whole run. + logger.warn(`NOMAD Score v2 not computed: ${error.message}`) + } + } + + // Best-effort environment metadata (never fails the run). + const env = await this._detectRunEnvironment() + // Save result const result = await BenchmarkResult.create({ benchmark_id: this.currentBenchmarkId, @@ -417,6 +591,20 @@ export class BenchmarkService { submitted_to_repository: false, sysbench_digest: SYSBENCH_DIGEST, ollama_version: aiScores.ai_ollama_version ?? null, + // v2 raw channels + score (null on system-only / AI-less runs) + cpu_events_single: systemRaws?.cpu_events_single ?? null, + cpu_events_multi: systemRaws?.cpu_events_multi ?? null, + cpu_benchmark_threads: systemRaws?.cpu_benchmark_threads ?? null, + cpu_total_events: systemRaws?.cpu_total_events ?? null, + cpu_total_time: systemRaws?.cpu_total_time ?? null, + memory_ops_per_sec: systemRaws?.memory_ops_per_sec ?? null, + memory_threads: systemRaws?.memory_threads ?? null, + disk_read_mb_per_sec: systemRaws?.disk_read_mb_per_sec ?? null, + disk_write_mb_per_sec: systemRaws?.disk_write_mb_per_sec ?? null, + nomad_score_v2: nomadScoreV2, + run_environment: env.run_environment, + storage_path_type: env.storage_path_type, + gpu_compute_detected: env.gpu_compute_detected, }) this._updateStatus('completed', 'Benchmark completed successfully') @@ -433,15 +621,25 @@ export class BenchmarkService { } /** - * Run system benchmarks using sysbench in Docker + * Run system benchmarks using sysbench in Docker. + * + * Produces both the legacy v1 0-1 sub-scores AND the v2 raw channels from the + * same runs. CPU runs twice: 1 thread (single-core index) and all logical + * threads (multi-core index); the v1 cpu_score keys off the multi pass to stay + * comparable with historical results. + * + * @param multiThreads number of threads for the multi-core CPU pass (all logical CPUs) */ - private async _runSystemBenchmarks(): Promise { + private async _runSystemBenchmarks(multiThreads: number): Promise { // Ensure sysbench image is available await this._ensureSysbenchImage() - // Run CPU benchmark - this._updateStatus('running_cpu', 'Running CPU benchmark...') - const cpuResult = await this._runSysbenchCpu() + // Run CPU benchmarks: single-thread then all-thread. + this._updateStatus('running_cpu', 'Running CPU benchmark (single-thread)...') + const cpuSingle = await this._runSysbenchCpu(1) + + this._updateStatus('running_cpu', `Running CPU benchmark (${multiThreads}-thread)...`) + const cpuMulti = await this._runSysbenchCpu(multiThreads) // Run memory benchmark this._updateStatus('running_memory', 'Running memory benchmark...') @@ -454,13 +652,29 @@ export class BenchmarkService { this._updateStatus('running_disk_write', 'Running disk write benchmark...') const diskWriteResult = await this._runSysbenchDiskWrite() - // Normalize scores to 0-100 scale - return { - cpu_score: this._normalizeScore(cpuResult.events_per_second, REFERENCE_SCORES.cpu_events_per_second), + // Legacy v1 sub-scores (0-1), normalized against the v1 references. cpu_score + // uses the multi-thread pass to match historical (4-thread) behaviour closely. + const scores: SystemScores = { + cpu_score: this._normalizeScore(cpuMulti.events_per_second, REFERENCE_SCORES.cpu_events_per_second), memory_score: this._normalizeScore(memoryResult.operations_per_second, REFERENCE_SCORES.memory_ops_per_second), disk_read_score: this._normalizeScore(diskReadResult.read_mb_per_sec, REFERENCE_SCORES.disk_read_mb_per_sec), disk_write_score: this._normalizeScore(diskWriteResult.write_mb_per_sec, REFERENCE_SCORES.disk_write_mb_per_sec), } + + // v2 raw channels — the payload of record for the leaderboard. + const raws: SystemBenchmarkRawsV2 = { + cpu_events_single: cpuSingle.events_per_second, + cpu_events_multi: cpuMulti.events_per_second, + cpu_benchmark_threads: multiThreads, + cpu_total_events: cpuMulti.total_events, + cpu_total_time: cpuMulti.total_time, + memory_ops_per_sec: memoryResult.operations_per_second, + memory_threads: MEMORY_BENCHMARK_THREADS, + disk_read_mb_per_sec: diskReadResult.read_mb_per_sec, + disk_write_mb_per_sec: diskWriteResult.write_mb_per_sec, + } + + return { scores, raws } } /** @@ -495,56 +709,161 @@ export class BenchmarkService { throw new Error(`Model does not exist and failed to download: ${modelResponse.message}`) } - // Run inference benchmark - const startTime = Date.now() + // Evict any other models resident in VRAM/RAM before benchmarking. With an 8B + // reference model, a chat model left loaded can starve it of VRAM → Ollama + // offloads layers to CPU → a low, non-comparable score. Give the benchmark + // model the whole GPU. Best-effort: never fail the run on an eviction hiccup. + await this._unloadResidentModels(ollamaAPIURL) - const response = await axios.post( - `${ollamaAPIURL}/api/generate`, - { - model: AI_BENCHMARK_MODEL, - prompt: AI_BENCHMARK_PROMPT, - stream: false, - }, - { timeout: 120000 } - ) + // Run inference AI_BENCHMARK_RUNS times and take the median run by tok/s (W7). + // A single inference is noisy (scheduler, thermal, cache); the median damps + // outliers without a warmup-run bias. TTFT is reported from the same run that + // is the tok/s median, so the two figures always describe one real inference. + const runs: { tps: number; ttft: number }[] = [] + for (let i = 0; i < AI_BENCHMARK_RUNS; i++) { + this._updateStatus('running_ai', `Running AI benchmark (${i + 1}/${AI_BENCHMARK_RUNS})...`) + runs.push(await this._runSingleAIInference(ollamaAPIURL)) + } - const endTime = Date.now() - const totalTime = (endTime - startTime) / 1000 // seconds + // Median by tok/s (odd count → middle element). + runs.sort((a, b) => a.tps - b.tps) + const median = runs[Math.floor(runs.length / 2)] - // Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds) - if (response.data.eval_count && response.data.eval_duration) { - const tokenCount = response.data.eval_count - const evalDurationSeconds = response.data.eval_duration / 1e9 - const tokensPerSecond = tokenCount / evalDurationSeconds + // Leave the GPU as clean as we found it: unload our benchmark model so the + // user's next chat loads their model into a free GPU (Ollama reloads on demand; + // we deliberately don't try to restore whichever model they had before). + await this._unloadResidentModels(ollamaAPIURL) - // Time to first token from prompt_eval_duration - const ttft = response.data.prompt_eval_duration - ? response.data.prompt_eval_duration / 1e6 // Convert to ms - : (totalTime * 1000) / 2 // Estimate if not available - - return { - ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, - ai_model_used: AI_BENCHMARK_MODEL, - ai_time_to_first_token: Math.round(ttft * 100) / 100, - ai_ollama_version: ollamaVersion, - } - } - - // Fallback calculation - const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100 - const tokensPerSecond = estimatedTokens / totalTime - - return { - ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, - ai_model_used: AI_BENCHMARK_MODEL, - ai_time_to_first_token: Math.round((totalTime * 1000) / 2), - ai_ollama_version: ollamaVersion, - } + return { + ai_tokens_per_second: Math.round(median.tps * 100) / 100, + ai_model_used: AI_BENCHMARK_MODEL, + ai_time_to_first_token: Math.round(median.ttft * 100) / 100, + ai_ollama_version: ollamaVersion, + } } catch (error) { throw new Error(`AI benchmark failed: ${error.message}`) } } + /** + * Run a single Ollama inference and return its tokens/sec and time-to-first-token + * (TTFT in milliseconds). Extracted so the AI benchmark can run it N times for a + * median. Prefers Ollama's eval_count/eval_duration; falls back to wall-clock + * word-count estimation if those fields are absent. + */ + private async _runSingleAIInference( + ollamaAPIURL: string + ): Promise<{ tps: number; ttft: number }> { + const startTime = Date.now() + + const response = await axios.post( + `${ollamaAPIURL}/api/generate`, + { + model: AI_BENCHMARK_MODEL, + prompt: AI_BENCHMARK_PROMPT, + stream: false, + options: { num_predict: AI_BENCHMARK_NUM_PREDICT }, + }, + { timeout: 120000 } + ) + + const endTime = Date.now() + const totalTime = (endTime - startTime) / 1000 // seconds + + // Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds) + if (response.data.eval_count && response.data.eval_duration) { + const tokenCount = response.data.eval_count + const evalDurationSeconds = response.data.eval_duration / 1e9 + const tokensPerSecond = tokenCount / evalDurationSeconds + + // Time to first token from prompt_eval_duration + const ttft = response.data.prompt_eval_duration + ? response.data.prompt_eval_duration / 1e6 // Convert to ms + : (totalTime * 1000) / 2 // Estimate if not available + + return { tps: tokensPerSecond, ttft } + } + + // Fallback calculation + const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100 + const tokensPerSecond = estimatedTokens / totalTime + return { tps: tokensPerSecond, ttft: (totalTime * 1000) / 2 } + } + + /** + * Fail fast if the AI model will need downloading and there isn't room for it. + * + * Only relevant on first run: if the model is already pulled, no download happens + * and we skip the check. Best-effort throughout — if Ollama isn't reachable or free + * space can't be determined, we do NOT block (the AI stage will surface its own + * error), because a false "insufficient disk" is worse than letting the pull try. + */ + private async _preflightModelDiskSpace(): Promise { + let ollamaAPIURL: string | null = null + try { + ollamaAPIURL = await this.dockerService.getServiceURL(SERVICE_NAMES.OLLAMA) + } catch { + return // AI stage will report Ollama problems + } + if (!ollamaAPIURL) return + + // If the model is already present, no download → no disk need. + try { + const tags = await axios.get(`${ollamaAPIURL}/api/tags`, { timeout: 5000 }) + const models: string[] = (tags.data?.models ?? []).map((m: { name?: string }) => m.name) + if (models.includes(AI_BENCHMARK_MODEL)) return + } catch { + return // can't reach Ollama; let the AI stage handle it + } + + // Model not cached → a pull is coming. Require headroom on the root FS. + let freeBytes: number | null = null + try { + const systemService = new (await import('./system_service.js')).SystemService(this.dockerService) + freeBytes = await getFreeBytes(systemService) + } catch { + return // couldn't measure free space — don't block on uncertainty + } + if (freeBytes === null) return + + if (freeBytes < AI_BENCHMARK_MODEL_MIN_FREE_BYTES) { + const gib = (b: number) => `${(b / 1024 / 1024 / 1024).toFixed(1)} GB` + throw new Error( + `Not enough free disk space to download the AI benchmark model (${AI_BENCHMARK_MODEL}). ` + + `About ${gib(AI_BENCHMARK_MODEL_MIN_FREE_BYTES)} is required but only ${gib(freeBytes)} is free. ` + + `Free up some space and try again.` + ) + } + } + + /** + * Evict every model currently resident in Ollama (VRAM/RAM) so the benchmark + * model gets the whole GPU. Queries /api/ps for loaded models and unloads each + * via a keep_alive:0 request. Entirely best-effort: any failure (older Ollama + * without /api/ps, a model mid-stream) is logged and swallowed — a clean-VRAM + * optimization must never block the benchmark itself. + */ + private async _unloadResidentModels(ollamaAPIURL: string): Promise { + try { + const ps = await axios.get(`${ollamaAPIURL}/api/ps`, { timeout: 5000 }) + const loaded: string[] = (ps.data?.models ?? []) + .map((m: { name?: string; model?: string }) => m.name || m.model) + .filter((n: string | undefined): n is string => !!n) + + if (loaded.length === 0) return + + logger.info(`[BenchmarkService] Evicting ${loaded.length} resident model(s) before AI benchmark: ${loaded.join(', ')}`) + for (const model of loaded) { + // keep_alive:0 tells Ollama to unload immediately (no prompt needed). + await axios + .post(`${ollamaAPIURL}/api/generate`, { model, keep_alive: 0 }, { timeout: 15000 }) + .catch((e) => logger.warn(`[BenchmarkService] Failed to unload ${model}: ${e.message}`)) + } + } catch (error) { + logger.warn(`[BenchmarkService] Could not enumerate/unload resident models (continuing): ${error.message}`) + } + } + /** * Calculate weighted NOMAD score */ @@ -592,6 +911,71 @@ export class BenchmarkService { return Math.round(Math.min(100, Math.max(0, nomadScore)) * 100) / 100 } + /** + * Calculate the NOMAD Score v2 from raw channel values. + * + * Byte-identical to the leaderboard's computeNomadScoreV2 (score_service.ts): + * the weighted geometric mean vs the frozen Reference Build, computed in the + * log domain, rounded to one decimal. NO clamps. Throws if any channel is + * <= 0 — a full v2 run measures every channel, and log2 of a non-positive + * ratio is undefined, so a non-positive value here is a programming error, not + * a user input to clamp. Every scored channel (incl. AI) is required; callers + * only invoke this for a complete full-benchmark run. + */ + private _calculateNomadScoreV2(raws: ScoreRawsV2): number { + let logSum = 0 + for (const channel of SCORE_CHANNELS_V2) { + const raw = raws[channel] + if (!(raw > 0)) { + throw new Error(`_calculateNomadScoreV2: channel "${channel}" must be > 0 (got ${raw})`) + } + const ratio = raw / REFERENCE_SCORES_V2[channel] + logSum += SCORE_WEIGHTS_V2[channel] * Math.log2(ratio) + } + const score = 1000 * Math.pow(2, logSum) + // One decimal, matching the leaderboard's display convention. + return Math.round(score * 10) / 10 + } + + /** + * Detect best-effort run-environment metadata for the leaderboard (issue #1016). + * Every probe is independently guarded — this never throws and never fails a run; + * any field it can't determine stays null. + */ + private async _detectRunEnvironment(): Promise { + const info: RunEnvironmentInfo = { + run_environment: null, + storage_path_type: null, + gpu_compute_detected: null, + } + + // run_environment: WSL2 vs native Linux, from the host kernel string. + try { + const procVersion = await readFile('/proc/version', 'utf8') + info.run_environment = /microsoft|wsl/i.test(procVersion) ? 'wsl2' : 'native-linux' + } catch { + // /proc/version unreadable — leave null + } + + // storage_path_type: the Docker storage driver backing the benchmarked FS. + try { + const dockerInfo = await this.dockerService.docker.info() + if (dockerInfo?.Driver) info.storage_path_type = String(dockerInfo.Driver) + } catch { + // docker info failed — leave null + } + + // gpu_compute_detected: whether a GPU compute type was detected + persisted. + try { + const gpuType = await KVStore.getValue('gpu.type') + info.gpu_compute_detected = gpuType === 'nvidia' || gpuType === 'amd' + } catch { + // KV read failed — leave null + } + + return info + } + /** * Normalize a raw score to 0-100 scale using log scaling * This provides diminishing returns for very high scores @@ -629,14 +1013,18 @@ export class BenchmarkService { } /** - * Run sysbench CPU benchmark + * Run sysbench CPU benchmark at the given thread count. + * + * v2 runs this twice: 1 thread (single-core index) and all threads (multi-core + * index). The multi pass also supplies the W6 consistency companions + * (total_events / total_time) the leaderboard cross-checks against events/sec. */ - private async _runSysbenchCpu(): Promise { + private async _runSysbenchCpu(threads: number): Promise { const output = await this._runSysbenchCommand([ 'sysbench', 'cpu', '--cpu-max-prime=20000', - '--threads=4', + `--threads=${threads}`, '--time=30', 'run', ]) @@ -645,7 +1033,7 @@ export class BenchmarkService { const eventsMatch = output.match(/events per second:\s*([\d.]+)/i) const totalTimeMatch = output.match(/total time:\s*([\d.]+)s/i) const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i) - logger.debug(`[BenchmarkService] CPU output parsing - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`) + logger.debug(`[BenchmarkService] CPU output parsing (${threads}T) - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`) // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss const eventsPerSecond = eventsMatch ? parseFloat(eventsMatch[1]) : Number.NaN @@ -653,10 +1041,21 @@ export class BenchmarkService { throw new Error('sysbench CPU benchmark produced no parseable events/sec — aborting (output may have changed or the test failed)') } + const totalTime = totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30 + // total_events feeds the leaderboard's W6 cross-check (events/sec must equal + // total_events/total_time within 2%). If the count line ever fails to parse, + // derive it from the rate so we submit a self-consistent payload rather than + // a zero that would be rejected as "internally inconsistent". + const parsedTotalEvents = totalEventsMatch ? parseInt(totalEventsMatch[1]) : Number.NaN + const totalEvents = + Number.isFinite(parsedTotalEvents) && parsedTotalEvents > 0 + ? parsedTotalEvents + : Math.round(eventsPerSecond * totalTime) + return { events_per_second: eventsPerSecond, - total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30, - total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0, + total_time: totalTime, + total_events: totalEvents, } } @@ -669,7 +1068,7 @@ export class BenchmarkService { 'memory', '--memory-block-size=1K', '--memory-total-size=10G', - '--threads=4', + `--threads=${MEMORY_BENCHMARK_THREADS}`, 'run', ]) @@ -697,11 +1096,14 @@ export class BenchmarkService { private async _runSysbenchDiskRead(): Promise { // Run prepare, test, and cleanup in a single container // This is necessary because each container has its own filesystem + // --file-extra-flags=direct (O_DIRECT) on the run bypasses the page cache so + // we measure real device throughput, not RAM (W4). Only the run needs it; + // prepare/cleanup are plain file ops. v2 reference disk numbers are O_DIRECT. const output = await this._runSysbenchCommand([ 'sh', '-c', 'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' + - 'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --time=30 run && ' + + 'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --file-extra-flags=direct --time=30 run && ' + 'sysbench fileio --file-total-size=1G --file-num=4 cleanup', ]) @@ -732,11 +1134,12 @@ export class BenchmarkService { private async _runSysbenchDiskWrite(): Promise { // Run prepare, test, and cleanup in a single container // This is necessary because each container has its own filesystem + // --file-extra-flags=direct (O_DIRECT) on the run bypasses the page cache (W4). const output = await this._runSysbenchCommand([ 'sh', '-c', 'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' + - 'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --time=30 run && ' + + 'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --file-extra-flags=direct --time=30 run && ' + 'sysbench fileio --file-total-size=1G --file-num=4 cleanup', ]) diff --git a/admin/database/migrations/1776400000002_add_benchmark_score_v2.ts b/admin/database/migrations/1776400000002_add_benchmark_score_v2.ts new file mode 100644 index 0000000..c9c5511 --- /dev/null +++ b/admin/database/migrations/1776400000002_add_benchmark_score_v2.ts @@ -0,0 +1,49 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'benchmark_results' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + // NOMAD Score v2 (Phase 4): raw channel values the leaderboard scores from, + // the frozen test parameters, the W6 consistency companions, the uncapped + // score, and best-effort run-environment metadata (#1016). All nullable — + // pre-v2 rows and system-only runs simply leave them empty. + // double (not float): Knex's MySQL float is float(8,2) — max 999999.99 and + // rounded to 2 decimals. memory_ops_per_sec / cpu_total_events run into the + // millions, and rounding raws would break byte-match with the leaderboard's + // server-side score recompute. double holds full-precision large values. + table.double('cpu_events_single').nullable() + table.double('cpu_events_multi').nullable() + table.integer('cpu_benchmark_threads').nullable() + table.double('cpu_total_events').nullable() + table.double('cpu_total_time').nullable() + table.double('memory_ops_per_sec').nullable() + table.integer('memory_threads').nullable() + table.double('disk_read_mb_per_sec').nullable() + table.double('disk_write_mb_per_sec').nullable() + table.double('nomad_score_v2').nullable() + table.string('run_environment').nullable() + table.string('storage_path_type').nullable() + table.boolean('gpu_compute_detected').nullable() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('cpu_events_single') + table.dropColumn('cpu_events_multi') + table.dropColumn('cpu_benchmark_threads') + table.dropColumn('cpu_total_events') + table.dropColumn('cpu_total_time') + table.dropColumn('memory_ops_per_sec') + table.dropColumn('memory_threads') + table.dropColumn('disk_read_mb_per_sec') + table.dropColumn('disk_write_mb_per_sec') + table.dropColumn('nomad_score_v2') + table.dropColumn('run_environment') + table.dropColumn('storage_path_type') + table.dropColumn('gpu_compute_detected') + }) + } +} diff --git a/admin/inertia/pages/settings/benchmark.tsx b/admin/inertia/pages/settings/benchmark.tsx index 8565cd0..16c6832 100644 --- a/admin/inertia/pages/settings/benchmark.tsx +++ b/admin/inertia/pages/settings/benchmark.tsx @@ -82,7 +82,7 @@ export default function BenchmarkPage(props: { setProgress({ status: 'starting', progress: 5, - message: 'Starting benchmark... This takes 2-5 minutes.', + message: 'Starting benchmark... This takes 3-6 minutes.', current_stage: 'Starting', benchmark_id: '', timestamp: new Date().toISOString(), @@ -423,7 +423,7 @@ export default function BenchmarkPage(props: { )}

Run a benchmark to measure your system's CPU, memory, disk, and AI inference - performance. The benchmark takes approximately 2-5 minutes to complete. + performance. The benchmark takes approximately 3-6 minutes to complete.

-
- {latestResult.nomad_score.toFixed(1)} -
-

- Your NOMAD Score is a weighted composite of all benchmark results. -

+ {latestResult.nomad_score_v2 != null ? ( + <> +
+
+ {latestResult.nomad_score_v2.toFixed(1)} +
+
+ NOMAD Score + +
+
+

+ Reference Build = 1000.{' '} + + Legacy scale: {latestResult.nomad_score.toFixed(1)} / 100 + +

+ + ) : ( + <> +
+ {latestResult.nomad_score.toFixed(1)} +
+

+ Your NOMAD Score is a weighted composite of all benchmark results. +

+ + )} {/* Share with Community - Only for full benchmarks with AI data */} {canShareBenchmark && ( @@ -769,7 +791,7 @@ export default function BenchmarkPage(props: {
NOMAD Score
- {latestResult.nomad_score.toFixed(1)} + {(latestResult.nomad_score_v2 ?? latestResult.nomad_score).toFixed(1)}
@@ -880,6 +902,101 @@ export default function BenchmarkPage(props: { + + {/* v2 raw measurements + run environment */} + {latestResult.cpu_events_multi != null && ( +
+
+

+ Measured Performance (v2) +

+
+
+ CPU single-thread + + {latestResult.cpu_events_single?.toFixed(1)} events/s + +
+
+ + CPU multi-thread ({latestResult.cpu_benchmark_threads}T) + + + {latestResult.cpu_events_multi?.toFixed(1)} events/s + +
+
+ + Memory ({latestResult.memory_threads}T) + + + {latestResult.memory_ops_per_sec?.toLocaleString()} ops/s + +
+
+ + Disk read (O_DIRECT) + + + {latestResult.disk_read_mb_per_sec?.toFixed(1)} MB/s + +
+
+ + Disk write (O_DIRECT) + + + {latestResult.disk_write_mb_per_sec?.toFixed(1)} MB/s + +
+
+
+ +
+

Environment

+
+
+ Run environment + + {latestResult.run_environment || 'Unknown'} + +
+
+ Storage backend + + {latestResult.storage_path_type || 'Unknown'} + +
+
+ GPU compute + + {latestResult.gpu_compute_detected == null + ? 'Unknown' + : latestResult.gpu_compute_detected + ? 'Detected' + : 'Not detected'} + +
+ {latestResult.sysbench_digest && ( +
+ sysbench + + {latestResult.sysbench_digest.replace('sha256:', '').slice(0, 12)} + +
+ )} + {latestResult.ollama_version && ( +
+ Ollama + + {latestResult.ollama_version} + +
+ )} +
+
+
+ )} )} diff --git a/admin/types/benchmark.ts b/admin/types/benchmark.ts index 2dd8261..a742e4d 100644 --- a/admin/types/benchmark.ts +++ b/admin/types/benchmark.ts @@ -115,6 +115,37 @@ export type UpdateBuilderTagResponse = { error: string } +// NOMAD Score v2 raw channels captured from a full benchmark run (all present +// and > 0). cpu_events_multi is measured at cpu_benchmark_threads; memory at +// memory_threads; disk figures are O_DIRECT MB/s. total_events/total_time are the +// W6 consistency companions from the multi-thread CPU pass. +export type SystemBenchmarkRawsV2 = { + cpu_events_single: number + cpu_events_multi: number + cpu_benchmark_threads: number + cpu_total_events: number + cpu_total_time: number + memory_ops_per_sec: number + memory_threads: number + disk_read_mb_per_sec: number + disk_write_mb_per_sec: number +} + +// Result of a system benchmark pass: the legacy 0-1 sub-scores (v1) plus the v2 +// raw channels, both derived from the same sysbench runs. +export type SystemBenchmarkOutput = { + scores: SystemScores + raws: SystemBenchmarkRawsV2 +} + +// Best-effort run-environment metadata (issue #1016). Any field may be null when +// detection fails; none of them gate a submission. +export type RunEnvironmentInfo = { + run_environment: string | null + storage_path_type: string | null + gpu_compute_detected: boolean | null +} + // Central repository submission payload (privacy-first) export type RepositorySubmission = Pick< BenchmarkResult, @@ -137,6 +168,46 @@ export type RepositorySubmission = Pick< builder_tag: string | null // null = anonymous submission } +// NOMAD Score v2 submission payload. Mirrors the leaderboard's submitValidatorV2 +// exactly: raw channels in (server recomputes the score), required test params + +// W6 companions + provenance, optional environment metadata. ai_time_to_first_token +// is in SECONDS here (the server treats it as seconds); the client stores TTFT in +// ms, so submitToRepository divides by 1000. +export type RepositorySubmissionV2 = { + // Hardware + cpu_model: string + cpu_cores: number + cpu_threads: number + ram_gb: number + disk_type: DiskType + gpu_model: string | null + // Scored raw channels (all required, > 0) + ai_tokens_per_second: number + cpu_events_single: number + cpu_events_multi: number + memory_ops_per_sec: number + disk_read_mb_per_sec: number + disk_write_mb_per_sec: number + // Test parameters + cpu_benchmark_threads: number + memory_threads: number + // Metadata channel (weight 0) + W6 consistency companions + ai_time_to_first_token: number // seconds + cpu_total_events: number + cpu_total_time: number + // Provenance (required) + ollama_version: string + sysbench_digest: string + // Environment metadata (best-effort) + run_environment?: string + storage_path_type?: string + gpu_compute_detected?: boolean + // Benchmark metadata (shared with v1) + nomad_version: string + benchmark_version: string + builder_tag?: string +} + // Central repository response types export type RepositorySubmitResponse = { success: boolean