feat(benchmark): harness hardening — fail loudly + pin sysbench + record provenance

Score v2 Phase 1. Safe under v1 — no scoring, weight, reference, or
submission-payload changes; only the benchmark's failure behavior and
forensic metadata.

- Fail-on-parse-miss (W3): the four SCORED sysbench metrics (CPU
  events/sec, memory ops/sec, disk read/write MiB/s) now THROW when the
  regex misses or the value is <= 0, instead of silently returning 0.
  A parse failure (e.g. an upstream image output-format change) now
  fails the run with a clear error via the existing _runBenchmark
  try/catch, rather than submitting a phantom zero sub-score. Secondary/
  informational fields keep their existing defaults.
- Pin sysbench by digest (W3): severalnines/sysbench@sha256:64cd003b...
  (was :latest), so a latest-tag format change can't break the parsers
  fleet-wide. Digest validated on the NOMAD6 reference build.
- Record provenance (W7): sysbench_digest + ollama_version (from Ollama
  /api/version, null-tolerant) stored on each result. New nullable
  columns + additive migration.

Verified: pinned digest pulls/runs/parses on NOMAD3; migration applies
(columns present); typecheck clean.

Part of the NOMAD Score v2 effort.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

admin/app/models/benchmark_result.ts

@@ -56,6 +56,13 @@ export default class BenchmarkResult extends BaseModel {
   @column()
   declare ai_time_to_first_token: number | null
 
+  // Harness forensic metadata (nullable — added in Score v2 Phase 1)
+  @column()
+  declare sysbench_digest: string | null
+
+  @column()
+  declare ollama_version: string | null
+
   // Composite NOMAD score (0-100)
   @column()
   declare nomad_score: number

admin/app/services/benchmark_service.ts

@@ -45,7 +45,9 @@ const SCORE_WEIGHTS = {
 }
 
 // Benchmark configuration constants
-const SYSBENCH_IMAGE = 'severalnines/sysbench:latest'
+// Pinned by digest (was severalnines/sysbench:latest) so a latest-tag format change can't silently break the parsers fleet-wide. Digest validated on the NOMAD6 reference build 2026-07-12.
+const SYSBENCH_IMAGE = 'severalnines/sysbench@sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
+const SYSBENCH_DIGEST = 'sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
 const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench'
 
 // Reference model for AI benchmark - small but meaningful
@@ -413,6 +415,8 @@ export class BenchmarkService {
         ai_time_to_first_token: aiScores.ai_time_to_first_token || null,
         nomad_score: nomadScore,
         submitted_to_repository: false,
+        sysbench_digest: SYSBENCH_DIGEST,
+        ollama_version: aiScores.ai_ollama_version ?? null,
       })
 
       this._updateStatus('completed', 'Benchmark completed successfully')
@@ -480,6 +484,10 @@ export class BenchmarkService {
       throw new Error(`Ollama is not running or not accessible (${errorCode}). Ensure AI Assistant is installed and running.`)
     }
 
+    // Record the Ollama version for forensics (null-tolerant — never fail the run on this)
+    const versionResp = await axios.get(`${ollamaAPIURL}/api/version`, { timeout: 5000 }).catch(() => null)
+    const ollamaVersion: string | null = versionResp?.data?.version ?? null
+
     // Check if the benchmark model is available, pull if not
     const ollamaService = new (await import('./ollama_service.js')).OllamaService()
     const modelResponse = await ollamaService.downloadModel(AI_BENCHMARK_MODEL)
@@ -518,6 +526,7 @@ export class BenchmarkService {
           ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
           ai_model_used: AI_BENCHMARK_MODEL,
           ai_time_to_first_token: Math.round(ttft * 100) / 100,
+          ai_ollama_version: ollamaVersion,
         }
       }
 
@@ -529,6 +538,7 @@ export class BenchmarkService {
         ai_tokens_per_second: Math.round(tokensPerSecond * 100) / 100,
         ai_model_used: AI_BENCHMARK_MODEL,
         ai_time_to_first_token: Math.round((totalTime * 1000) / 2),
+        ai_ollama_version: ollamaVersion,
       }
     } catch (error) {
       throw new Error(`AI benchmark failed: ${error.message}`)
@@ -637,8 +647,14 @@ export class BenchmarkService {
     const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i)
     logger.debug(`[BenchmarkService] CPU output parsing - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`)
 
+    // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
+    const eventsPerSecond = eventsMatch ? parseFloat(eventsMatch[1]) : Number.NaN
+    if (!eventsMatch || !Number.isFinite(eventsPerSecond) || eventsPerSecond <= 0) {
+      throw new Error('sysbench CPU benchmark produced no parseable events/sec — aborting (output may have changed or the test failed)')
+    }
+
     return {
-      events_per_second: eventsMatch ? parseFloat(eventsMatch[1]) : 0,
+      events_per_second: eventsPerSecond,
       total_time: totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30,
       total_events: totalEventsMatch ? parseInt(totalEventsMatch[1]) : 0,
     }
@@ -662,8 +678,14 @@ export class BenchmarkService {
     const transferMatch = output.match(/([\d.]+)\s*MiB\/sec/i)
     const timeMatch = output.match(/total time:\s*([\d.]+)s/i)
 
+    // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
+    const opsPerSecond = opsMatch ? parseFloat(opsMatch[1]) : Number.NaN
+    if (!opsMatch || !Number.isFinite(opsPerSecond) || opsPerSecond <= 0) {
+      throw new Error('sysbench memory benchmark produced no parseable ops/sec — aborting')
+    }
+
     return {
-      operations_per_second: opsMatch ? parseFloat(opsMatch[1]) : 0,
+      operations_per_second: opsPerSecond,
       transfer_rate_mb_per_sec: transferMatch ? parseFloat(transferMatch[1]) : 0,
       total_time: timeMatch ? parseFloat(timeMatch[1]) : 0,
     }
@@ -689,10 +711,16 @@ export class BenchmarkService {
 
     logger.debug(`[BenchmarkService] Disk read output parsing - read: ${readMatch?.[1]}, reads/s: ${readsPerSecMatch?.[1]}`)
 
+    // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
+    const readMbPerSec = readMatch ? parseFloat(readMatch[1]) : Number.NaN
+    if (!readMatch || !Number.isFinite(readMbPerSec) || readMbPerSec <= 0) {
+      throw new Error('sysbench disk-read benchmark produced no parseable MiB/s — aborting')
+    }
+
     return {
       reads_per_second: readsPerSecMatch ? parseFloat(readsPerSecMatch[1]) : 0,
       writes_per_second: 0,
-      read_mb_per_sec: readMatch ? parseFloat(readMatch[1]) : 0,
+      read_mb_per_sec: readMbPerSec,
       write_mb_per_sec: 0,
       total_time: 30,
     }
@@ -718,11 +746,17 @@ export class BenchmarkService {
 
     logger.debug(`[BenchmarkService] Disk write output parsing - written: ${writeMatch?.[1]}, writes/s: ${writesPerSecMatch?.[1]}`)
 
+    // Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
+    const writeMbPerSec = writeMatch ? parseFloat(writeMatch[1]) : Number.NaN
+    if (!writeMatch || !Number.isFinite(writeMbPerSec) || writeMbPerSec <= 0) {
+      throw new Error('sysbench disk-write benchmark produced no parseable MiB/s — aborting')
+    }
+
     return {
       reads_per_second: 0,
       writes_per_second: writesPerSecMatch ? parseFloat(writesPerSecMatch[1]) : 0,
       read_mb_per_sec: 0,
-      write_mb_per_sec: writeMatch ? parseFloat(writeMatch[1]) : 0,
+      write_mb_per_sec: writeMbPerSec,
       total_time: 30,
     }
   }

admin/database/migrations/1776400000001_add_benchmark_harness_metadata.ts

@@ -0,0 +1,22 @@
+import { BaseSchema } from '@adonisjs/lucid/schema'
+
+export default class extends BaseSchema {
+  protected tableName = 'benchmark_results'
+
+  async up() {
+    this.schema.alterTable(this.tableName, (table) => {
+      // Forensic harness metadata (Score v2 Phase 1): which sysbench image digest produced the
+      // system scores, and which Ollama version served the AI benchmark. Both nullable —
+      // pre-existing rows and runs without AI simply leave them empty.
+      table.string('sysbench_digest').nullable()
+      table.string('ollama_version').nullable()
+    })
+  }
+
+  async down() {
+    this.schema.alterTable(this.tableName, (table) => {
+      table.dropColumn('sysbench_digest')
+      table.dropColumn('ollama_version')
+    })
+  }
+}

admin/types/benchmark.ts

@@ -35,7 +35,10 @@ export type SystemScores = Pick<
 export type AIScores = Pick<
   BenchmarkResult,
   'ai_tokens_per_second' | 'ai_model_used' | 'ai_time_to_first_token'
->
+> & {
+  // Forensic metadata: Ollama server version at benchmark time (null if /api/version unavailable)
+  ai_ollama_version?: string | null
+}
 
 // Slim version for lists
 export type BenchmarkResultSlim = Pick<