From ac581415302a1ed289bf7d462d0cefbefbe53e86 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Sun, 12 Jul 2026 12:22:53 -0700 Subject: [PATCH] =?UTF-8?q?feat(benchmark):=20harness=20hardening=20?= =?UTF-8?q?=E2=80=94=20fail=20loudly=20+=20pin=20sysbench=20+=20record=20p?= =?UTF-8?q?rovenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Score v2 Phase 1. Safe under v1 — no scoring, weight, reference, or submission-payload changes; only the benchmark's failure behavior and forensic metadata. - Fail-on-parse-miss (W3): the four SCORED sysbench metrics (CPU events/sec, memory ops/sec, disk read/write MiB/s) now THROW when the regex misses or the value is <= 0, instead of silently returning 0. A parse failure (e.g. an upstream image output-format change) now fails the run with a clear error via the existing _runBenchmark try/catch, rather than submitting a phantom zero sub-score. Secondary/ informational fields keep their existing defaults. - Pin sysbench by digest (W3): severalnines/sysbench@sha256:64cd003b... (was :latest), so a latest-tag format change can't break the parsers fleet-wide. Digest validated on the NOMAD6 reference build. - Record provenance (W7): sysbench_digest + ollama_version (from Ollama /api/version, null-tolerant) stored on each result. New nullable columns + additive migration. Verified: pinned digest pulls/runs/parses on NOMAD3; migration applies (columns present); typecheck clean. Part of the NOMAD Score v2 effort. Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/app/models/benchmark_result.ts | 7 +++ admin/app/services/benchmark_service.ts | 44 ++++++++++++++++--- ...00000001_add_benchmark_harness_metadata.ts | 22 ++++++++++ admin/types/benchmark.ts | 5 ++- 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 admin/database/migrations/1776400000001_add_benchmark_harness_metadata.ts diff --git a/admin/app/models/benchmark_result.ts b/admin/app/models/benchmark_result.ts index fdede41..c1b1481 100644 --- a/admin/app/models/benchmark_result.ts +++ b/admin/app/models/benchmark_result.ts @@ -56,6 +56,13 @@ export default class BenchmarkResult extends BaseModel { @column() declare ai_time_to_first_token: number | null + // Harness forensic metadata (nullable — added in Score v2 Phase 1) + @column() + declare sysbench_digest: string | null + + @column() + declare ollama_version: string | null + // Composite NOMAD score (0-100) @column() declare nomad_score: number diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index 710b7ad..2c7eddf 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -45,7 +45,9 @@ const SCORE_WEIGHTS = { } // Benchmark configuration constants -const SYSBENCH_IMAGE = 'severalnines/sysbench:latest' +// Pinned by digest (was severalnines/sysbench:latest) so a latest-tag format change can't silently break the parsers fleet-wide. Digest validated on the NOMAD6 reference build 2026-07-12. +const SYSBENCH_IMAGE = 'severalnines/sysbench@sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd' +const SYSBENCH_DIGEST = 'sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd' const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench' // Reference model for AI benchmark - small but meaningful @@ -413,6 +415,8 @@ export class BenchmarkService { ai_time_to_first_token: aiScores.ai_time_to_first_token || null, nomad_score: nomadScore, submitted_to_repository: false, + sysbench_digest: SYSBENCH_DIGEST, + ollama_version: aiScores.ai_ollama_version ?? null, }) this._updateStatus('completed', 'Benchmark completed successfully') @@ -480,6 +484,10 @@ export class BenchmarkService { throw new Error(`Ollama is not running or not accessible (${errorCode}). Ensure AI Assistant is installed and running.`) } + // Record the Ollama version for forensics (null-tolerant — never fail the run on this) + const versionResp = await axios.get(`${ollamaAPIURL}/api/version`, { timeout: 5000 }).catch(() => null) + const ollamaVersion: string | null = versionResp?.data?.version ?? null + // Check if the benchmark model is available, pull if not const ollamaService = new (await import('./ollama_service.js')).OllamaService() const modelResponse = await ollamaService.downloadModel(AI_BENCHMARK_MODEL) @@ -518,6 +526,7 @@ export class BenchmarkService { ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, ai_model_used: AI_BENCHMARK_MODEL, ai_time_to_first_token: Math.round(ttft * 100) / 100, + ai_ollama_version: ollamaVersion, } } @@ -529,6 +538,7 @@ export class BenchmarkService { ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100, ai_model_used: AI_BENCHMARK_MODEL, ai_time_to_first_token: Math.round((totalTime * 1000) / 2), + ai_ollama_version: ollamaVersion, } } catch (error) { throw new Error(`AI benchmark failed: ${error.message}`) @@ -637,8 +647,14 @@ export class BenchmarkService { const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i) logger.debug(`[BenchmarkService] CPU output parsing - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`) + // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss + const eventsPerSecond = eventsMatch ? parseFloat(eventsMatch[1]) : Number.NaN + if (!eventsMatch || !Number.isFinite(eventsPerSecond) || eventsPerSecond <= 0) { + throw new Error('sysbench CPU benchmark produced no parseable events/sec — aborting (output may have changed or the test failed)') + } + return { - events_per_second: eventsMatch ? parseFloat(eventsMatch[1]) : 0, + events_per_second: eventsPerSecond, total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30, total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0, } @@ -662,8 +678,14 @@ export class BenchmarkService { const transferMatch = output.match(/([\d.]+)\s*MiB\/sec/i) const timeMatch = output.match(/total time:\s*([\d.]+)s/i) + // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss + const opsPerSecond = opsMatch ? parseFloat(opsMatch[1]) : Number.NaN + if (!opsMatch || !Number.isFinite(opsPerSecond) || opsPerSecond <= 0) { + throw new Error('sysbench memory benchmark produced no parseable ops/sec — aborting') + } + return { - operations_per_second: opsMatch ? parseFloat(opsMatch[1]) : 0, + operations_per_second: opsPerSecond, transfer_rate_mb_per_sec: transferMatch ? parseFloat(transferMatch[1]) : 0, total_time: timeMatch ? parseFloat(timeMatch[1]) : 0, } @@ -689,10 +711,16 @@ export class BenchmarkService { logger.debug(`[BenchmarkService] Disk read output parsing - read: ${readMatch?.[1]}, reads/s: ${readsPerSecMatch?.[1]}`) + // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss + const readMbPerSec = readMatch ? parseFloat(readMatch[1]) : Number.NaN + if (!readMatch || !Number.isFinite(readMbPerSec) || readMbPerSec <= 0) { + throw new Error('sysbench disk-read benchmark produced no parseable MiB/s — aborting') + } + return { reads_per_second: readsPerSecMatch ? parseFloat(readsPerSecMatch[1]) : 0, writes_per_second: 0, - read_mb_per_sec: readMatch ? parseFloat(readMatch[1]) : 0, + read_mb_per_sec: readMbPerSec, write_mb_per_sec: 0, total_time: 30, } @@ -718,11 +746,17 @@ export class BenchmarkService { logger.debug(`[BenchmarkService] Disk write output parsing - written: ${writeMatch?.[1]}, writes/s: ${writesPerSecMatch?.[1]}`) + // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss + const writeMbPerSec = writeMatch ? parseFloat(writeMatch[1]) : Number.NaN + if (!writeMatch || !Number.isFinite(writeMbPerSec) || writeMbPerSec <= 0) { + throw new Error('sysbench disk-write benchmark produced no parseable MiB/s — aborting') + } + return { reads_per_second: 0, writes_per_second: writesPerSecMatch ? parseFloat(writesPerSecMatch[1]) : 0, read_mb_per_sec: 0, - write_mb_per_sec: writeMatch ? parseFloat(writeMatch[1]) : 0, + write_mb_per_sec: writeMbPerSec, total_time: 30, } } diff --git a/admin/database/migrations/1776400000001_add_benchmark_harness_metadata.ts b/admin/database/migrations/1776400000001_add_benchmark_harness_metadata.ts new file mode 100644 index 0000000..f1fbb36 --- /dev/null +++ b/admin/database/migrations/1776400000001_add_benchmark_harness_metadata.ts @@ -0,0 +1,22 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'benchmark_results' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + // Forensic harness metadata (Score v2 Phase 1): which sysbench image digest produced the + // system scores, and which Ollama version served the AI benchmark. Both nullable — + // pre-existing rows and runs without AI simply leave them empty. + table.string('sysbench_digest').nullable() + table.string('ollama_version').nullable() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('sysbench_digest') + table.dropColumn('ollama_version') + }) + } +} diff --git a/admin/types/benchmark.ts b/admin/types/benchmark.ts index 9eda94b..2dd8261 100644 --- a/admin/types/benchmark.ts +++ b/admin/types/benchmark.ts @@ -35,7 +35,10 @@ export type SystemScores = Pick< export type AIScores = Pick< BenchmarkResult, 'ai_tokens_per_second' | 'ai_model_used' | 'ai_time_to_first_token' -> +> & { + // Forensic metadata: Ollama server version at benchmark time (null if /api/version unavailable) + ai_ollama_version?: string | null +} // Slim version for lists export type BenchmarkResultSlim = Pick<