diff --git a/admin/app/models/benchmark_result.ts b/admin/app/models/benchmark_result.ts index 656a9d9..ae11a1a 100644 --- a/admin/app/models/benchmark_result.ts +++ b/admin/app/models/benchmark_result.ts @@ -63,6 +63,18 @@ export default class BenchmarkResult extends BaseModel { @column() declare ollama_version: string | null + // Platform metadata (nullable). Sourced from the Docker daemon, not + // systeminformation: inside the admin container si.osInfo()/os.arch() + // describe the container, not the host. + @column() + declare cpu_architecture: string | null + + @column() + declare os_name: string | null + + @column() + declare os_version: string | null + // NOMAD Score v2 raw channels (nullable — added in Score v2 Phase 4). Populated // on full benchmarks under benchmark_version >= 2.0.0; the leaderboard recomputes // the score from these on submit. cpu_events_multi is measured at diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index 71f5865..2fff4d8 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -29,6 +29,7 @@ import type { RunEnvironmentInfo, } from '../../types/benchmark.js' import KVStore from '#models/kv_store' +import { normalizeArchitecture, deriveOsName } from '../utils/platform_metadata.js' import { getFreeBytes } from '../utils/image_disk_preflight.js' import { readFile } from 'node:fs/promises' import { randomUUID, createHmac } from 'node:crypto' @@ -106,9 +107,27 @@ if (Math.abs(WEIGHT_SUM_V2 - 1) > 1e-9) { } // Benchmark configuration constants -// Pinned by digest (was severalnines/sysbench:latest) so a latest-tag format change can't silently break the parsers fleet-wide. Digest validated on the NOMAD6 reference build 2026-07-12. -const SYSBENCH_IMAGE = 'severalnines/sysbench@sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd' -const SYSBENCH_DIGEST = 'sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd' +// Pinned by digest so a tag-format change can't silently break the parsers +// fleet-wide. +// +// Was severalnines/sysbench, which publishes amd64 ONLY — that locked ARM hosts +// out of the System Benchmark entirely (not a graceful failure: the container +// simply cannot run) and forced Apple Silicon through Rosetta emulation, which +// distorts the very measurement it is taking. +// +// This is our own multi-arch build (Debian 12 + sysbench 1.0.20+ds-5, built at +// Crosstalk-Solutions/nomad-sysbench for linux/amd64 + linux/arm64). ONE digest +// covers both architectures because a manifest list resolves per-arch on pull. +// +// Comparability: 1.0.17 -> 1.0.20 measured 1.25% apart on identical hardware +// with identical flags (7170.18 vs 7259.56 events/sec) — inside run-to-run +// noise, ~0.3% on a composite. No rescoring required. Both this and the legacy +// digest are allowlisted server-side, so the fleet can cross over gradually. +const SYSBENCH_IMAGE = + 'ghcr.io/crosstalk-solutions/nomad-sysbench@sha256:1f08e527f5d440135de9bd49006a2c13342cb1e483c59a53774e2db35e8e13f0' +const SYSBENCH_DIGEST = 'sha256:1f08e527f5d440135de9bd49006a2c13342cb1e483c59a53774e2db35e8e13f0' + + const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench' // Reference model for AI benchmark. v2 uses an 8B (was llama3.2:1b): a 1B is so @@ -400,6 +419,12 @@ export class BenchmarkService { if (result.storage_path_type) submission.storage_path_type = result.storage_path_type if (result.gpu_compute_detected != null) submission.gpu_compute_detected = result.gpu_compute_detected + // Platform metadata. Optional rather than required so results recorded + // before this shipped remain submittable. + if (result.cpu_architecture) submission.cpu_architecture = result.cpu_architecture + if (result.os_name) submission.os_name = result.os_name + if (result.os_version) submission.os_version = result.os_version + // Builder tag: omit entirely when anonymous or unset (validator rejects null). if (!anonymous && result.builder_tag) submission.builder_tag = result.builder_tag @@ -623,6 +648,11 @@ export class BenchmarkService { } } + // Record the sysbench digest that actually ran, not the compiled-in + // constant. Only meaningful when the system benchmarks ran at all — an + // AI-only benchmark never pulls the image. + const sysbenchDigest = systemRaws ? await this._resolveSysbenchDigest() : null + // Calculate NOMAD scores (v1 legacy + v2 uncapped) this._updateStatus('calculating_score', 'Calculating NOMAD score...') const nomadScore = this._calculateNomadScore(systemScores, aiScores) @@ -669,7 +699,7 @@ export class BenchmarkService { ai_time_to_first_token: aiScores.ai_time_to_first_token || null, nomad_score: nomadScore, submitted_to_repository: false, - sysbench_digest: SYSBENCH_DIGEST, + sysbench_digest: sysbenchDigest, ollama_version: aiScores.ai_ollama_version ?? null, // v2 raw channels + score (null on system-only / AI-less runs) cpu_events_single: systemRaws?.cpu_events_single ?? null, @@ -685,6 +715,9 @@ export class BenchmarkService { run_environment: env.run_environment, storage_path_type: env.storage_path_type, gpu_compute_detected: env.gpu_compute_detected, + cpu_architecture: env.cpu_architecture, + os_name: env.os_name, + os_version: env.os_version, }) this._updateStatus('completed', 'Benchmark completed successfully') @@ -1167,6 +1200,9 @@ export class BenchmarkService { run_environment: null, storage_path_type: null, gpu_compute_detected: null, + cpu_architecture: null, + os_name: null, + os_version: null, } // run_environment: WSL2 vs native Linux, from the host kernel string. @@ -1181,6 +1217,21 @@ export class BenchmarkService { try { const dockerInfo = await this.dockerService.docker.info() if (dockerInfo?.Driver) info.storage_path_type = String(dockerInfo.Driver) + + // Platform metadata from the same call. The Docker daemon reports the + // HOST, which is the whole point — os.arch() and si.osInfo() inside the + // admin container describe the container instead. + // + // Observed: Architecture 'x86_64' | 'aarch64', OSVersion '24.04', + // OperatingSystem 'Ubuntu 24.04.4 LTS'. + if (dockerInfo?.Architecture) { + info.cpu_architecture = normalizeArchitecture(String(dockerInfo.Architecture)) + } + if (dockerInfo?.OperatingSystem) { + const osVersion = dockerInfo.OSVersion ? String(dockerInfo.OSVersion) : null + info.os_version = osVersion + info.os_name = deriveOsName(String(dockerInfo.OperatingSystem), osVersion) + } } catch { // docker info failed — leave null } @@ -1232,6 +1283,38 @@ export class BenchmarkService { } } + /** + * The sysbench digest actually resolved on this machine. + * + * Previously the submission reported SYSBENCH_DIGEST — the constant the client + * was compiled with. The leaderboard validates that field against an allowlist, + * but a constant attests to how the client was BUILT, not to what it RAN, so + * any build inherits a valid value simply by carrying the same source. + * + * Reading it back from the image means a divergent build has to actively + * falsify the field rather than passively inherit it. Still forgeable — the + * client is open source and always will be — but it moves the bar from "no + * effort" to "deliberate", which is the distinction that matters when judging + * whether a submission is a mistake or a choice. + * + * Uses RepoDigests (the manifest digest we pulled by), NOT Id — Id is the + * config digest, which differs per architecture and would never match the + * allowlist. Falls back to the constant if inspection gives us nothing usable, + * so a benchmark never fails over provenance metadata. + */ + private async _resolveSysbenchDigest(): Promise { + try { + const info = await this.dockerService.docker.getImage(SYSBENCH_IMAGE).inspect() + const repoDigest = (info?.RepoDigests ?? []).find((d: string) => d.includes('@sha256:')) + const digest = repoDigest?.split('@')[1] + if (digest) return digest + logger.warn('[BenchmarkService] No RepoDigest on the sysbench image; reporting the pinned constant.') + } catch (error: any) { + logger.warn(`[BenchmarkService] Could not inspect the sysbench image (${error.message}); reporting the pinned constant.`) + } + return SYSBENCH_DIGEST + } + /** * Run sysbench CPU benchmark at the given thread count. * diff --git a/admin/app/utils/platform_metadata.ts b/admin/app/utils/platform_metadata.ts new file mode 100644 index 0000000..1617bc1 --- /dev/null +++ b/admin/app/utils/platform_metadata.ts @@ -0,0 +1,68 @@ +/** + * Pure helpers for turning the Docker daemon's platform strings into the fields + * the benchmark submission carries. + * + * These read the HOST's platform, which is the entire point: inside the admin + * container `os.arch()` and `si.osInfo()` describe the container, not the + * machine being benchmarked. `BenchmarkService` delegates to these so the string + * handling is unit-testable without a Docker daemon. + * + * Observed daemon output across the test fleet: + * + * Architecture 'x86_64' | 'aarch64' + * OSVersion '24.04' | '26.04' + * OperatingSystem 'Ubuntu 24.04.4 LTS' | 'Ubuntu 26.04 LTS' + */ + +/** + * Canonicalise the daemon's architecture string to the OCI platform names used + * everywhere else in the project (image manifests, install docs, the + * leaderboard). + * + * Docker reports `x86_64` / `aarch64`; images and the board talk in `amd64` / + * `arm64`. A fixed two-way map rather than a general normalisation table: these + * are the only architectures NOMAD targets, and anything unrecognised passes + * through verbatim rather than being guessed at, so an unexpected platform shows + * up honestly instead of mislabelled. + */ +export function normalizeArchitecture(raw: string): string { + const map: Record = { + x86_64: 'amd64', + amd64: 'amd64', + aarch64: 'arm64', + arm64: 'arm64', + } + const key = raw.trim().toLowerCase() + return map[key] ?? raw.trim() +} + +/** + * Split the distro name out of the daemon's free-form OperatingSystem string. + * + * `OperatingSystem` is a description ('Ubuntu 24.04.4 LTS') while `OSVersion` is + * structured ('24.04'). Taking the text before the version yields the name + * without hand-maintaining a list of distributions: + * + * 'Ubuntu 24.04.4 LTS' + '24.04' -> 'Ubuntu' + * 'Ubuntu 26.04 LTS' + '26.04' -> 'Ubuntu' + * 'Debian GNU/Linux 12 (bookworm)' + '12' -> 'Debian GNU/Linux' + * + * Falls back to the full description whenever the version is missing, empty, or + * doesn't appear in the string. An over-long name is harmless; a wrong one is + * not, and silently truncating an unfamiliar distro would be worse than leaving + * it verbose. + */ +export function deriveOsName(operatingSystem: string, osVersion: string | null): string { + const description = operatingSystem.trim() + if (!osVersion) return description + + const version = osVersion.trim() + if (version === '') return description + + const idx = description.indexOf(version) + // idx === 0 means the string starts with the version and has no name to take. + if (idx <= 0) return description + + const name = description.slice(0, idx).trim() + return name.length > 0 ? name : description +} diff --git a/admin/database/migrations/1776400000003_add_benchmark_platform_metadata.ts b/admin/database/migrations/1776400000003_add_benchmark_platform_metadata.ts new file mode 100644 index 0000000..d296650 --- /dev/null +++ b/admin/database/migrations/1776400000003_add_benchmark_platform_metadata.ts @@ -0,0 +1,29 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'benchmark_results' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + // Platform metadata (Score v2). The leaderboard is a single board across + // instruction sets by design, with disclosure as the fairness mechanism — + // without an architecture field an ARM result is indistinguishable from an + // x86 one, which is exactly what the disclosure is meant to prevent. + // + // All sourced from the Docker daemon rather than systeminformation, because + // si.osInfo()/os.arch() inside the admin container describe the CONTAINER, + // not the host. Nullable — pre-existing rows simply leave them empty. + table.string('cpu_architecture').nullable() + table.string('os_name').nullable() + table.string('os_version').nullable() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('cpu_architecture') + table.dropColumn('os_name') + table.dropColumn('os_version') + }) + } +} diff --git a/admin/tests/unit/platform_metadata.spec.ts b/admin/tests/unit/platform_metadata.spec.ts new file mode 100644 index 0000000..f7c3ec3 --- /dev/null +++ b/admin/tests/unit/platform_metadata.spec.ts @@ -0,0 +1,57 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { normalizeArchitecture, deriveOsName } from '../../app/utils/platform_metadata.js' + +// Values observed from `docker info` across the test fleet (NOMAD3 x86 / 26.04, +// NOMAD6 x86 / 24.04, nomad10 Raspberry Pi 5 arm64 / 26.04). + +test('normalizeArchitecture maps Docker arch strings to OCI platform names', () => { + assert.equal(normalizeArchitecture('x86_64'), 'amd64') + assert.equal(normalizeArchitecture('aarch64'), 'arm64') +}) + +test('normalizeArchitecture accepts values already in OCI form', () => { + assert.equal(normalizeArchitecture('amd64'), 'amd64') + assert.equal(normalizeArchitecture('arm64'), 'arm64') +}) + +test('normalizeArchitecture is case- and whitespace-insensitive', () => { + assert.equal(normalizeArchitecture(' X86_64 '), 'amd64') + assert.equal(normalizeArchitecture('AArch64'), 'arm64') +}) + +test('normalizeArchitecture passes unknown architectures through untouched', () => { + // Better an honest unexpected value on the board than a confidently wrong one. + assert.equal(normalizeArchitecture('riscv64'), 'riscv64') + assert.equal(normalizeArchitecture(' ppc64le '), 'ppc64le') +}) + +test('deriveOsName takes the name preceding the version', () => { + assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', '24.04'), 'Ubuntu') + assert.equal(deriveOsName('Ubuntu 26.04 LTS', '26.04'), 'Ubuntu') +}) + +test('deriveOsName handles multi-word distro names', () => { + assert.equal(deriveOsName('Debian GNU/Linux 12 (bookworm)', '12'), 'Debian GNU/Linux') + assert.equal(deriveOsName('Red Hat Enterprise Linux 9.4 (Plow)', '9.4'), 'Red Hat Enterprise Linux') +}) + +test('deriveOsName falls back to the full description without a usable version', () => { + assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', null), 'Ubuntu 24.04.4 LTS') + assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', ''), 'Ubuntu 24.04.4 LTS') + assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', ' '), 'Ubuntu 24.04.4 LTS') +}) + +test('deriveOsName falls back when the version is absent from the description', () => { + // Daemons have been known to disagree with themselves; don't truncate on a guess. + assert.equal(deriveOsName('Alpine Linux v3.20', '3.20.1'), 'Alpine Linux v3.20') +}) + +test('deriveOsName falls back when the description begins with the version', () => { + assert.equal(deriveOsName('12 Debian', '12'), '12 Debian') +}) + +test('deriveOsName trims surrounding whitespace', () => { + assert.equal(deriveOsName(' Ubuntu 24.04.4 LTS ', '24.04'), 'Ubuntu') +}) diff --git a/admin/types/benchmark.ts b/admin/types/benchmark.ts index b9e5df5..3e54427 100644 --- a/admin/types/benchmark.ts +++ b/admin/types/benchmark.ts @@ -189,6 +189,9 @@ export type RunEnvironmentInfo = { run_environment: string | null storage_path_type: string | null gpu_compute_detected: boolean | null + cpu_architecture: string | null + os_name: string | null + os_version: string | null } // Central repository submission payload (privacy-first) @@ -247,6 +250,11 @@ export type RepositorySubmissionV2 = { run_environment?: string storage_path_type?: string gpu_compute_detected?: boolean + // Platform metadata. cpu_architecture is what makes a single cross-ISA + // leaderboard honest — without it an ARM result is indistinguishable from x86. + cpu_architecture?: string + os_name?: string + os_version?: string // Benchmark metadata (shared with v1) nomad_version: string benchmark_version: string