feat(benchmark): official multi-arch sysbench, resolved digest, platform metadata (#1158)
Three changes that together let ARM hardware appear on the leaderboard honestly. Shipping them separately would leave ARM half-supported either way: without the image a Pi cannot submit at all, and without the architecture field it submits but is indistinguishable from x86. 1. PIN THE OFFICIAL MULTI-ARCH SYSBENCH IMAGE severalnines/sysbench publishes amd64 only, so ARM hosts could not run the System Benchmark at all — not a graceful failure, the container simply cannot execute. Apple Silicon could only run it under Rosetta emulation, which distorts the measurement it is taking, and that is what drove a community macOS fork to substitute a different benchmark and submit incomparable numbers. Swaps to ghcr.io/crosstalk-solutions/nomad-sysbench (Debian 12 + sysbench 1.0.20+ds-5, built for linux/amd64 + linux/arm64). One digest covers both architectures; verified that pulling the pinned manifest-list digest resolves to arm64 on a Raspberry Pi 5 and amd64 on x86, and that RepoDigests reports the same manifest-list digest on both — so a single allowlist entry serves both. No rescoring: 1.0.17 -> 1.0.20 measured 1.25% apart on identical hardware with identical flags (7170.18 vs 7259.56 events/sec), inside run-to-run noise and ~0.3% on a composite. Both digests are allowlisted server-side, so the fleet can cross over gradually. 2. REPORT THE DIGEST ACTUALLY RESOLVED The submission previously sent SYSBENCH_DIGEST, the constant the client was compiled with. The leaderboard validates that field, but a constant attests to how a client was BUILT rather than what it RAN, so any build inherits a valid value simply by carrying the same source. Now reads it back from the image. Uses RepoDigests (the manifest digest we pulled by), never Id — Id is the config digest, differs per architecture, and would never match the allowlist. Falls back to the constant if inspection yields nothing usable, so a benchmark never fails over provenance metadata. Still forgeable, and always will be with an open-source client. It moves the bar from "no effort" to "deliberate", which is the distinction that matters when judging whether a submission is a mistake or a choice. 3. RECORD CPU ARCHITECTURE AND OS The leaderboard is a single board across instruction sets by design, with disclosure as the fairness mechanism. Without an architecture field an ARM result sits unlabelled beside x86 — exactly what the disclosure exists to prevent. All three fields come from the Docker daemon, reusing the docker.info() call _detectRunEnvironment already makes. That is deliberate: inside the admin container os.arch() and si.osInfo() describe the CONTAINER, not the host being benchmarked. cpu_architecture Architecture x86_64 -> amd64, aarch64 -> arm64 os_version OSVersion '24.04' (already structured, no parsing) os_name OperatingSystem 'Ubuntu 24.04.4 LTS' minus the version run_environment is kept rather than replaced: "which distro" and "is this virtualised" are different questions, and WSL2 is a real performance factor. String handling lives in app/utils/platform_metadata.ts with unit tests, matching the amd_hsa_override convention, so it is testable without a Docker daemon. Unknown architectures pass through verbatim rather than being guessed at, and os_name falls back to the full description whenever the version is missing or absent from it — an over-long name is harmless, a wrong one is not. Columns are nullable and the submission fields optional, so results recorded before this shipped remain submittable. Closes #1156 Refs #1151
This commit is contained in:
chriscrosstalk
GitHub
parent
d1535d17b9
commit
0891d176e5
|
|
@ -63,6 +63,18 @@ export default class BenchmarkResult extends BaseModel {
|
|||
@column()
|
||||
declare ollama_version: string | null
|
||||
|
||||
// Platform metadata (nullable). Sourced from the Docker daemon, not
|
||||
// systeminformation: inside the admin container si.osInfo()/os.arch()
|
||||
// describe the container, not the host.
|
||||
@column()
|
||||
declare cpu_architecture: string | null
|
||||
|
||||
@column()
|
||||
declare os_name: string | null
|
||||
|
||||
@column()
|
||||
declare os_version: string | null
|
||||
|
||||
// NOMAD Score v2 raw channels (nullable — added in Score v2 Phase 4). Populated
|
||||
// on full benchmarks under benchmark_version >= 2.0.0; the leaderboard recomputes
|
||||
// the score from these on submit. cpu_events_multi is measured at
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import type {
|
|||
RunEnvironmentInfo,
|
||||
} from '../../types/benchmark.js'
|
||||
import KVStore from '#models/kv_store'
|
||||
import { normalizeArchitecture, deriveOsName } from '../utils/platform_metadata.js'
|
||||
import { getFreeBytes } from '../utils/image_disk_preflight.js'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { randomUUID, createHmac } from 'node:crypto'
|
||||
|
|
@ -106,9 +107,27 @@ if (Math.abs(WEIGHT_SUM_V2 - 1) > 1e-9) {
|
|||
}
|
||||
|
||||
// Benchmark configuration constants
|
||||
// Pinned by digest (was severalnines/sysbench:latest) so a latest-tag format change can't silently break the parsers fleet-wide. Digest validated on the NOMAD6 reference build 2026-07-12.
|
||||
const SYSBENCH_IMAGE = 'severalnines/sysbench@sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
|
||||
const SYSBENCH_DIGEST = 'sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
|
||||
// Pinned by digest so a tag-format change can't silently break the parsers
|
||||
// fleet-wide.
|
||||
//
|
||||
// Was severalnines/sysbench, which publishes amd64 ONLY — that locked ARM hosts
|
||||
// out of the System Benchmark entirely (not a graceful failure: the container
|
||||
// simply cannot run) and forced Apple Silicon through Rosetta emulation, which
|
||||
// distorts the very measurement it is taking.
|
||||
//
|
||||
// This is our own multi-arch build (Debian 12 + sysbench 1.0.20+ds-5, built at
|
||||
// Crosstalk-Solutions/nomad-sysbench for linux/amd64 + linux/arm64). ONE digest
|
||||
// covers both architectures because a manifest list resolves per-arch on pull.
|
||||
//
|
||||
// Comparability: 1.0.17 -> 1.0.20 measured 1.25% apart on identical hardware
|
||||
// with identical flags (7170.18 vs 7259.56 events/sec) — inside run-to-run
|
||||
// noise, ~0.3% on a composite. No rescoring required. Both this and the legacy
|
||||
// digest are allowlisted server-side, so the fleet can cross over gradually.
|
||||
const SYSBENCH_IMAGE =
|
||||
'ghcr.io/crosstalk-solutions/nomad-sysbench@sha256:1f08e527f5d440135de9bd49006a2c13342cb1e483c59a53774e2db35e8e13f0'
|
||||
const SYSBENCH_DIGEST = 'sha256:1f08e527f5d440135de9bd49006a2c13342cb1e483c59a53774e2db35e8e13f0'
|
||||
|
||||
|
||||
const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench'
|
||||
|
||||
// Reference model for AI benchmark. v2 uses an 8B (was llama3.2:1b): a 1B is so
|
||||
|
|
@ -400,6 +419,12 @@ export class BenchmarkService {
|
|||
if (result.storage_path_type) submission.storage_path_type = result.storage_path_type
|
||||
if (result.gpu_compute_detected != null) submission.gpu_compute_detected = result.gpu_compute_detected
|
||||
|
||||
// Platform metadata. Optional rather than required so results recorded
|
||||
// before this shipped remain submittable.
|
||||
if (result.cpu_architecture) submission.cpu_architecture = result.cpu_architecture
|
||||
if (result.os_name) submission.os_name = result.os_name
|
||||
if (result.os_version) submission.os_version = result.os_version
|
||||
|
||||
// Builder tag: omit entirely when anonymous or unset (validator rejects null).
|
||||
if (!anonymous && result.builder_tag) submission.builder_tag = result.builder_tag
|
||||
|
||||
|
|
@ -623,6 +648,11 @@ export class BenchmarkService {
|
|||
}
|
||||
}
|
||||
|
||||
// Record the sysbench digest that actually ran, not the compiled-in
|
||||
// constant. Only meaningful when the system benchmarks ran at all — an
|
||||
// AI-only benchmark never pulls the image.
|
||||
const sysbenchDigest = systemRaws ? await this._resolveSysbenchDigest() : null
|
||||
|
||||
// Calculate NOMAD scores (v1 legacy + v2 uncapped)
|
||||
this._updateStatus('calculating_score', 'Calculating NOMAD score...')
|
||||
const nomadScore = this._calculateNomadScore(systemScores, aiScores)
|
||||
|
|
@ -669,7 +699,7 @@ export class BenchmarkService {
|
|||
ai_time_to_first_token: aiScores.ai_time_to_first_token || null,
|
||||
nomad_score: nomadScore,
|
||||
submitted_to_repository: false,
|
||||
sysbench_digest: SYSBENCH_DIGEST,
|
||||
sysbench_digest: sysbenchDigest,
|
||||
ollama_version: aiScores.ai_ollama_version ?? null,
|
||||
// v2 raw channels + score (null on system-only / AI-less runs)
|
||||
cpu_events_single: systemRaws?.cpu_events_single ?? null,
|
||||
|
|
@ -685,6 +715,9 @@ export class BenchmarkService {
|
|||
run_environment: env.run_environment,
|
||||
storage_path_type: env.storage_path_type,
|
||||
gpu_compute_detected: env.gpu_compute_detected,
|
||||
cpu_architecture: env.cpu_architecture,
|
||||
os_name: env.os_name,
|
||||
os_version: env.os_version,
|
||||
})
|
||||
|
||||
this._updateStatus('completed', 'Benchmark completed successfully')
|
||||
|
|
@ -1167,6 +1200,9 @@ export class BenchmarkService {
|
|||
run_environment: null,
|
||||
storage_path_type: null,
|
||||
gpu_compute_detected: null,
|
||||
cpu_architecture: null,
|
||||
os_name: null,
|
||||
os_version: null,
|
||||
}
|
||||
|
||||
// run_environment: WSL2 vs native Linux, from the host kernel string.
|
||||
|
|
@ -1181,6 +1217,21 @@ export class BenchmarkService {
|
|||
try {
|
||||
const dockerInfo = await this.dockerService.docker.info()
|
||||
if (dockerInfo?.Driver) info.storage_path_type = String(dockerInfo.Driver)
|
||||
|
||||
// Platform metadata from the same call. The Docker daemon reports the
|
||||
// HOST, which is the whole point — os.arch() and si.osInfo() inside the
|
||||
// admin container describe the container instead.
|
||||
//
|
||||
// Observed: Architecture 'x86_64' | 'aarch64', OSVersion '24.04',
|
||||
// OperatingSystem 'Ubuntu 24.04.4 LTS'.
|
||||
if (dockerInfo?.Architecture) {
|
||||
info.cpu_architecture = normalizeArchitecture(String(dockerInfo.Architecture))
|
||||
}
|
||||
if (dockerInfo?.OperatingSystem) {
|
||||
const osVersion = dockerInfo.OSVersion ? String(dockerInfo.OSVersion) : null
|
||||
info.os_version = osVersion
|
||||
info.os_name = deriveOsName(String(dockerInfo.OperatingSystem), osVersion)
|
||||
}
|
||||
} catch {
|
||||
// docker info failed — leave null
|
||||
}
|
||||
|
|
@ -1232,6 +1283,38 @@ export class BenchmarkService {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The sysbench digest actually resolved on this machine.
|
||||
*
|
||||
* Previously the submission reported SYSBENCH_DIGEST — the constant the client
|
||||
* was compiled with. The leaderboard validates that field against an allowlist,
|
||||
* but a constant attests to how the client was BUILT, not to what it RAN, so
|
||||
* any build inherits a valid value simply by carrying the same source.
|
||||
*
|
||||
* Reading it back from the image means a divergent build has to actively
|
||||
* falsify the field rather than passively inherit it. Still forgeable — the
|
||||
* client is open source and always will be — but it moves the bar from "no
|
||||
* effort" to "deliberate", which is the distinction that matters when judging
|
||||
* whether a submission is a mistake or a choice.
|
||||
*
|
||||
* Uses RepoDigests (the manifest digest we pulled by), NOT Id — Id is the
|
||||
* config digest, which differs per architecture and would never match the
|
||||
* allowlist. Falls back to the constant if inspection gives us nothing usable,
|
||||
* so a benchmark never fails over provenance metadata.
|
||||
*/
|
||||
private async _resolveSysbenchDigest(): Promise<string> {
|
||||
try {
|
||||
const info = await this.dockerService.docker.getImage(SYSBENCH_IMAGE).inspect()
|
||||
const repoDigest = (info?.RepoDigests ?? []).find((d: string) => d.includes('@sha256:'))
|
||||
const digest = repoDigest?.split('@')[1]
|
||||
if (digest) return digest
|
||||
logger.warn('[BenchmarkService] No RepoDigest on the sysbench image; reporting the pinned constant.')
|
||||
} catch (error: any) {
|
||||
logger.warn(`[BenchmarkService] Could not inspect the sysbench image (${error.message}); reporting the pinned constant.`)
|
||||
}
|
||||
return SYSBENCH_DIGEST
|
||||
}
|
||||
|
||||
/**
|
||||
* Run sysbench CPU benchmark at the given thread count.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Pure helpers for turning the Docker daemon's platform strings into the fields
|
||||
* the benchmark submission carries.
|
||||
*
|
||||
* These read the HOST's platform, which is the entire point: inside the admin
|
||||
* container `os.arch()` and `si.osInfo()` describe the container, not the
|
||||
* machine being benchmarked. `BenchmarkService` delegates to these so the string
|
||||
* handling is unit-testable without a Docker daemon.
|
||||
*
|
||||
* Observed daemon output across the test fleet:
|
||||
*
|
||||
* Architecture 'x86_64' | 'aarch64'
|
||||
* OSVersion '24.04' | '26.04'
|
||||
* OperatingSystem 'Ubuntu 24.04.4 LTS' | 'Ubuntu 26.04 LTS'
|
||||
*/
|
||||
|
||||
/**
|
||||
* Canonicalise the daemon's architecture string to the OCI platform names used
|
||||
* everywhere else in the project (image manifests, install docs, the
|
||||
* leaderboard).
|
||||
*
|
||||
* Docker reports `x86_64` / `aarch64`; images and the board talk in `amd64` /
|
||||
* `arm64`. A fixed two-way map rather than a general normalisation table: these
|
||||
* are the only architectures NOMAD targets, and anything unrecognised passes
|
||||
* through verbatim rather than being guessed at, so an unexpected platform shows
|
||||
* up honestly instead of mislabelled.
|
||||
*/
|
||||
export function normalizeArchitecture(raw: string): string {
|
||||
const map: Record<string, string> = {
|
||||
x86_64: 'amd64',
|
||||
amd64: 'amd64',
|
||||
aarch64: 'arm64',
|
||||
arm64: 'arm64',
|
||||
}
|
||||
const key = raw.trim().toLowerCase()
|
||||
return map[key] ?? raw.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Split the distro name out of the daemon's free-form OperatingSystem string.
|
||||
*
|
||||
* `OperatingSystem` is a description ('Ubuntu 24.04.4 LTS') while `OSVersion` is
|
||||
* structured ('24.04'). Taking the text before the version yields the name
|
||||
* without hand-maintaining a list of distributions:
|
||||
*
|
||||
* 'Ubuntu 24.04.4 LTS' + '24.04' -> 'Ubuntu'
|
||||
* 'Ubuntu 26.04 LTS' + '26.04' -> 'Ubuntu'
|
||||
* 'Debian GNU/Linux 12 (bookworm)' + '12' -> 'Debian GNU/Linux'
|
||||
*
|
||||
* Falls back to the full description whenever the version is missing, empty, or
|
||||
* doesn't appear in the string. An over-long name is harmless; a wrong one is
|
||||
* not, and silently truncating an unfamiliar distro would be worse than leaving
|
||||
* it verbose.
|
||||
*/
|
||||
export function deriveOsName(operatingSystem: string, osVersion: string | null): string {
|
||||
const description = operatingSystem.trim()
|
||||
if (!osVersion) return description
|
||||
|
||||
const version = osVersion.trim()
|
||||
if (version === '') return description
|
||||
|
||||
const idx = description.indexOf(version)
|
||||
// idx === 0 means the string starts with the version and has no name to take.
|
||||
if (idx <= 0) return description
|
||||
|
||||
const name = description.slice(0, idx).trim()
|
||||
return name.length > 0 ? name : description
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'benchmark_results'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
// Platform metadata (Score v2). The leaderboard is a single board across
|
||||
// instruction sets by design, with disclosure as the fairness mechanism —
|
||||
// without an architecture field an ARM result is indistinguishable from an
|
||||
// x86 one, which is exactly what the disclosure is meant to prevent.
|
||||
//
|
||||
// All sourced from the Docker daemon rather than systeminformation, because
|
||||
// si.osInfo()/os.arch() inside the admin container describe the CONTAINER,
|
||||
// not the host. Nullable — pre-existing rows simply leave them empty.
|
||||
table.string('cpu_architecture').nullable()
|
||||
table.string('os_name').nullable()
|
||||
table.string('os_version').nullable()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.dropColumn('cpu_architecture')
|
||||
table.dropColumn('os_name')
|
||||
table.dropColumn('os_version')
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import * as assert from 'node:assert/strict'
|
||||
import { test } from 'node:test'
|
||||
|
||||
import { normalizeArchitecture, deriveOsName } from '../../app/utils/platform_metadata.js'
|
||||
|
||||
// Values observed from `docker info` across the test fleet (NOMAD3 x86 / 26.04,
|
||||
// NOMAD6 x86 / 24.04, nomad10 Raspberry Pi 5 arm64 / 26.04).
|
||||
|
||||
test('normalizeArchitecture maps Docker arch strings to OCI platform names', () => {
|
||||
assert.equal(normalizeArchitecture('x86_64'), 'amd64')
|
||||
assert.equal(normalizeArchitecture('aarch64'), 'arm64')
|
||||
})
|
||||
|
||||
test('normalizeArchitecture accepts values already in OCI form', () => {
|
||||
assert.equal(normalizeArchitecture('amd64'), 'amd64')
|
||||
assert.equal(normalizeArchitecture('arm64'), 'arm64')
|
||||
})
|
||||
|
||||
test('normalizeArchitecture is case- and whitespace-insensitive', () => {
|
||||
assert.equal(normalizeArchitecture(' X86_64 '), 'amd64')
|
||||
assert.equal(normalizeArchitecture('AArch64'), 'arm64')
|
||||
})
|
||||
|
||||
test('normalizeArchitecture passes unknown architectures through untouched', () => {
|
||||
// Better an honest unexpected value on the board than a confidently wrong one.
|
||||
assert.equal(normalizeArchitecture('riscv64'), 'riscv64')
|
||||
assert.equal(normalizeArchitecture(' ppc64le '), 'ppc64le')
|
||||
})
|
||||
|
||||
test('deriveOsName takes the name preceding the version', () => {
|
||||
assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', '24.04'), 'Ubuntu')
|
||||
assert.equal(deriveOsName('Ubuntu 26.04 LTS', '26.04'), 'Ubuntu')
|
||||
})
|
||||
|
||||
test('deriveOsName handles multi-word distro names', () => {
|
||||
assert.equal(deriveOsName('Debian GNU/Linux 12 (bookworm)', '12'), 'Debian GNU/Linux')
|
||||
assert.equal(deriveOsName('Red Hat Enterprise Linux 9.4 (Plow)', '9.4'), 'Red Hat Enterprise Linux')
|
||||
})
|
||||
|
||||
test('deriveOsName falls back to the full description without a usable version', () => {
|
||||
assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', null), 'Ubuntu 24.04.4 LTS')
|
||||
assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', ''), 'Ubuntu 24.04.4 LTS')
|
||||
assert.equal(deriveOsName('Ubuntu 24.04.4 LTS', ' '), 'Ubuntu 24.04.4 LTS')
|
||||
})
|
||||
|
||||
test('deriveOsName falls back when the version is absent from the description', () => {
|
||||
// Daemons have been known to disagree with themselves; don't truncate on a guess.
|
||||
assert.equal(deriveOsName('Alpine Linux v3.20', '3.20.1'), 'Alpine Linux v3.20')
|
||||
})
|
||||
|
||||
test('deriveOsName falls back when the description begins with the version', () => {
|
||||
assert.equal(deriveOsName('12 Debian', '12'), '12 Debian')
|
||||
})
|
||||
|
||||
test('deriveOsName trims surrounding whitespace', () => {
|
||||
assert.equal(deriveOsName(' Ubuntu 24.04.4 LTS ', '24.04'), 'Ubuntu')
|
||||
})
|
||||
|
|
@ -189,6 +189,9 @@ export type RunEnvironmentInfo = {
|
|||
run_environment: string | null
|
||||
storage_path_type: string | null
|
||||
gpu_compute_detected: boolean | null
|
||||
cpu_architecture: string | null
|
||||
os_name: string | null
|
||||
os_version: string | null
|
||||
}
|
||||
|
||||
// Central repository submission payload (privacy-first)
|
||||
|
|
@ -247,6 +250,11 @@ export type RepositorySubmissionV2 = {
|
|||
run_environment?: string
|
||||
storage_path_type?: string
|
||||
gpu_compute_detected?: boolean
|
||||
// Platform metadata. cpu_architecture is what makes a single cross-ISA
|
||||
// leaderboard honest — without it an ARM result is indistinguishable from x86.
|
||||
cpu_architecture?: string
|
||||
os_name?: string
|
||||
os_version?: string
|
||||
// Benchmark metadata (shared with v1)
|
||||
nomad_version: string
|
||||
benchmark_version: string
|
||||
|
|
|
|||
Loading…
Reference in New Issue