1307 lines
52 KiB
TypeScript
1307 lines
52 KiB
TypeScript
import { inject } from '@adonisjs/core'
|
||
import logger from '@adonisjs/core/services/logger'
|
||
import transmit from '@adonisjs/transmit/services/main'
|
||
import si from 'systeminformation'
|
||
import axios from 'axios'
|
||
import { DateTime } from 'luxon'
|
||
import BenchmarkResult from '#models/benchmark_result'
|
||
import BenchmarkSetting from '#models/benchmark_setting'
|
||
import { SystemService } from '#services/system_service'
|
||
import type {
|
||
BenchmarkType,
|
||
BenchmarkStatus,
|
||
BenchmarkProgress,
|
||
HardwareInfo,
|
||
DiskType,
|
||
SystemScores,
|
||
AIScores,
|
||
SysbenchCpuResult,
|
||
SysbenchMemoryResult,
|
||
SysbenchDiskResult,
|
||
RepositorySubmissionV2,
|
||
RepositorySubmitResponse,
|
||
RepositoryStats,
|
||
SystemBenchmarkOutput,
|
||
SystemBenchmarkRawsV2,
|
||
RunEnvironmentInfo,
|
||
} from '../../types/benchmark.js'
|
||
import KVStore from '#models/kv_store'
|
||
import { getFreeBytes } from '../utils/image_disk_preflight.js'
|
||
import { readFile } from 'node:fs/promises'
|
||
import { randomUUID, createHmac } from 'node:crypto'
|
||
import { DockerService } from './docker_service.js'
|
||
import { SERVICE_NAMES } from '../../constants/service_names.js'
|
||
import { BROADCAST_CHANNELS } from '../../constants/broadcast.js'
|
||
import Dockerode from 'dockerode'
|
||
|
||
// HMAC secret for signing submissions to the benchmark repository
|
||
// This provides basic protection against casual API abuse.
|
||
// Note: Since NOMAD is open source, a determined attacker could extract this.
|
||
// For stronger protection, see challenge-response authentication.
|
||
const BENCHMARK_HMAC_SECRET = '778ba65d0bc0e23119e5ffce4b3716648a7d071f0a47ec3f'
|
||
|
||
// Re-export default weights for use in service
|
||
const SCORE_WEIGHTS = {
|
||
ai_tokens_per_second: 0.30,
|
||
cpu: 0.25,
|
||
memory: 0.15,
|
||
ai_ttft: 0.10,
|
||
disk_read: 0.10,
|
||
disk_write: 0.10,
|
||
}
|
||
|
||
// The benchmark_version this client emits. Server dispatches on the major:
|
||
// >= 2 → raw-channel v2 ingest (score recomputed server-side). Bumped 1.0.0 -> 2.0.0
|
||
// for the NOMAD Score v2 rollout (uncapped index vs a frozen Reference Build).
|
||
const BENCHMARK_VERSION = '2.0.0'
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// NOMAD Score v2 — the uncapped index.
|
||
//
|
||
// Score = 1000 × Π( (raw_i / ref_i) ^ w_i ) (weighted geometric mean, log domain).
|
||
//
|
||
// These constants + the computation below MUST stay byte-identical to the
|
||
// leaderboard's score_service.ts (computeNomadScoreV2): the client shows this
|
||
// score immediately after a run, and the server recomputes the same value from
|
||
// the raw channels on submit. Any drift makes the displayed score disagree with
|
||
// the leaderboard. FROZEN under benchmark_version 2.0.0 — Reference Build =
|
||
// NOMAD6 (Ryzen 9 PRO 8945HS / 780M / 64GB, kernel 6.17), measured median-of-N
|
||
// with direct-I/O disk 2026-07-12. There are NO clamps here; outlier control is
|
||
// the server's rejection gates + quarantine, not the score math.
|
||
const REFERENCE_SCORES_V2 = {
|
||
// TODO(score-v2): STALE — 40.5 was measured on llama3.2:1b. The reference model
|
||
// is now llama3.1:8b; re-measure this on the Reference Build (NOMAD6) with the
|
||
// v2 harness (unload + num_predict + median-of-3) and update here AND in the
|
||
// leaderboard's score_service.ts (must byte-match) before v2 ships.
|
||
ai_tokens_per_second: 40.5,
|
||
cpu_events_multi: 19231, // all-threads on the reference box (16T)
|
||
cpu_events_single: 2351, // 1T
|
||
memory_ops_per_sec: 26862960, // 4T, 1K block (bandwidth peak)
|
||
disk_read_mb_per_sec: 1072, // O_DIRECT
|
||
disk_write_mb_per_sec: 316, // O_DIRECT
|
||
} as const
|
||
|
||
const SCORE_WEIGHTS_V2 = {
|
||
ai_tokens_per_second: 0.3,
|
||
cpu_events_multi: 0.25,
|
||
cpu_events_single: 0.1,
|
||
memory_ops_per_sec: 0.1,
|
||
disk_read_mb_per_sec: 0.125,
|
||
disk_write_mb_per_sec: 0.125,
|
||
} as const
|
||
|
||
type ScoreChannelV2 = keyof typeof REFERENCE_SCORES_V2
|
||
type ScoreRawsV2 = Record<ScoreChannelV2, number>
|
||
const SCORE_CHANNELS_V2 = Object.keys(REFERENCE_SCORES_V2) as ScoreChannelV2[]
|
||
|
||
// Weights sum to 1.000; guard against a future typo silently rescaling scores.
|
||
const WEIGHT_SUM_V2 = SCORE_CHANNELS_V2.reduce((s, k) => s + SCORE_WEIGHTS_V2[k], 0)
|
||
if (Math.abs(WEIGHT_SUM_V2 - 1) > 1e-9) {
|
||
throw new Error(`SCORE_WEIGHTS_V2 must sum to 1.0 (got ${WEIGHT_SUM_V2})`)
|
||
}
|
||
|
||
// Benchmark configuration constants
|
||
// Pinned by digest (was severalnines/sysbench:latest) so a latest-tag format change can't silently break the parsers fleet-wide. Digest validated on the NOMAD6 reference build 2026-07-12.
|
||
const SYSBENCH_IMAGE = 'severalnines/sysbench@sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
|
||
const SYSBENCH_DIGEST = 'sha256:64cd003bfa21eaab22f985e7b95f90d21a970229f5f628718657dd1bae669abd'
|
||
const SYSBENCH_CONTAINER_NAME = 'nomad_benchmark_sysbench'
|
||
|
||
// Reference model for AI benchmark. v2 uses an 8B (was llama3.2:1b): a 1B is so
|
||
// light it runs overhead-bound on any GPU (~240 tok/s), saturating the score and
|
||
// failing to differentiate hardware. An 8B is compute/bandwidth-bound and scales
|
||
// with real GPU capability. Non-thinking (deterministic workload). Changing this
|
||
// re-baselines REFERENCE_SCORES_V2.ai_tokens_per_second — re-measure on the
|
||
// Reference Build (NOMAD6) before shipping.
|
||
const AI_BENCHMARK_MODEL = 'llama3.1:8b'
|
||
const AI_BENCHMARK_PROMPT = 'Explain recursion in programming in exactly 100 words.'
|
||
// Cap generation so the workload is bounded and comparable across machines rather
|
||
// than relying on the model honoring "100 words". ~100-150 tokens is plenty for a
|
||
// stable tok/s rate; the cap only trims pathological long outputs.
|
||
const AI_BENCHMARK_NUM_PREDICT = 256
|
||
|
||
// Memory test is frozen at 4 threads for the v2 Reference Build (the bandwidth
|
||
// peak on the reference box). Do not vary by core count — the reference number
|
||
// (REFERENCE_SCORES_V2.memory_ops_per_sec) was measured at exactly this value.
|
||
const MEMORY_BENCHMARK_THREADS = 4
|
||
|
||
// AI is measured as the median of N runs (W7) to damp per-run inference jitter.
|
||
const AI_BENCHMARK_RUNS = 3
|
||
|
||
// Minimum free disk required before pulling the AI model on first run. llama3.1:8b
|
||
// (Q4) is ~4.9 GB; this covers the model plus headroom for the transient sysbench
|
||
// disk-test file and normal slack. Only enforced when the model isn't already cached.
|
||
const AI_BENCHMARK_MODEL_MIN_FREE_BYTES = 6.5 * 1024 * 1024 * 1024
|
||
|
||
// Reference scores for normalization (calibrated to 0-100 scale)
|
||
// These represent "expected" scores for a mid-range system (score ~50)
|
||
const REFERENCE_SCORES = {
|
||
cpu_events_per_second: 5000, // sysbench cpu events/sec for ~50 score
|
||
memory_ops_per_second: 5000000, // sysbench memory ops/sec for ~50 score
|
||
disk_read_mb_per_sec: 500, // 500 MB/s read for ~50 score
|
||
disk_write_mb_per_sec: 400, // 400 MB/s write for ~50 score
|
||
ai_tokens_per_second: 30, // 30 tok/s for ~50 score
|
||
ai_ttft_ms: 500, // 500ms time to first token for ~50 score (lower is better)
|
||
}
|
||
|
||
@inject()
|
||
export class BenchmarkService {
|
||
private currentBenchmarkId: string | null = null
|
||
private currentStatus: BenchmarkStatus = 'idle'
|
||
|
||
constructor(private dockerService: DockerService) {}
|
||
|
||
/**
|
||
* Run a full benchmark suite
|
||
*/
|
||
async runFullBenchmark(): Promise<BenchmarkResult> {
|
||
return this._runBenchmark('full', true)
|
||
}
|
||
|
||
/**
|
||
* Run system benchmarks only (CPU, memory, disk)
|
||
*/
|
||
async runSystemBenchmarks(): Promise<BenchmarkResult> {
|
||
return this._runBenchmark('system', false)
|
||
}
|
||
|
||
/**
|
||
* Run AI benchmark only
|
||
*/
|
||
async runAIBenchmark(): Promise<BenchmarkResult> {
|
||
return this._runBenchmark('ai', true)
|
||
}
|
||
|
||
/**
|
||
* Get the latest benchmark result
|
||
*/
|
||
async getLatestResult(): Promise<BenchmarkResult | null> {
|
||
return await BenchmarkResult.query().orderBy('created_at', 'desc').first()
|
||
}
|
||
|
||
/**
|
||
* Get all benchmark results
|
||
*/
|
||
async getAllResults(): Promise<BenchmarkResult[]> {
|
||
return await BenchmarkResult.query().orderBy('created_at', 'desc')
|
||
}
|
||
|
||
/**
|
||
* Get a specific benchmark result by ID
|
||
*/
|
||
async getResultById(benchmarkId: string): Promise<BenchmarkResult | null> {
|
||
return await BenchmarkResult.findBy('benchmark_id', benchmarkId)
|
||
}
|
||
|
||
/**
|
||
* Whether to show the dashboard "re-run under Score v2" banner. Shown to users
|
||
* who have a v1 leaderboard submission but no Score v2 result yet, and who
|
||
* haven't dismissed it. Self-clears two ways: dismiss sets the KV flag, and any
|
||
* result carrying a nomad_score_v2 (i.e. a v2 run happened) flips (b) false.
|
||
*/
|
||
async shouldShowRerunBanner(): Promise<boolean> {
|
||
const dismissed = await KVStore.getValue('benchmark.rerunBannerDismissed')
|
||
if (dismissed === true) return false
|
||
|
||
const hasV2Run = await BenchmarkResult.query().whereNotNull('nomad_score_v2').first()
|
||
if (hasV2Run) return false
|
||
|
||
const hasSubmittedV1 = await BenchmarkResult.query()
|
||
.where('submitted_to_repository', true)
|
||
.first()
|
||
return hasSubmittedV1 !== null
|
||
}
|
||
|
||
/**
|
||
* Submit benchmark results to central repository
|
||
*/
|
||
async submitToRepository(benchmarkId?: string, anonymous?: boolean): Promise<RepositorySubmitResponse> {
|
||
const result = benchmarkId
|
||
? await this.getResultById(benchmarkId)
|
||
: await this.getLatestResult()
|
||
|
||
if (!result) {
|
||
throw new Error('No benchmark result found to submit')
|
||
}
|
||
|
||
// Only allow full benchmarks with AI data to be submitted to repository
|
||
if (result.benchmark_type !== 'full') {
|
||
throw new Error('Only full benchmarks can be shared with the community. Run a Full Benchmark to share your results.')
|
||
}
|
||
|
||
if (!result.ai_tokens_per_second || result.ai_tokens_per_second <= 0) {
|
||
throw new Error('Benchmark must include AI performance data. Ensure AI Assistant is installed and run a Full Benchmark.')
|
||
}
|
||
|
||
if (result.submitted_to_repository) {
|
||
throw new Error('Benchmark result has already been submitted')
|
||
}
|
||
|
||
// Build the v2 payload (raw channels; server recomputes the score). Throws a
|
||
// user-facing error if the row predates v2 and lacks the raw channels.
|
||
const submission = this._buildV2Submission(result, anonymous ?? false)
|
||
|
||
try {
|
||
// Generate HMAC signature for submission verification
|
||
const timestamp = Date.now().toString()
|
||
const payload = timestamp + JSON.stringify(submission)
|
||
const signature = createHmac('sha256', BENCHMARK_HMAC_SECRET)
|
||
.update(payload)
|
||
.digest('hex')
|
||
|
||
const response = await axios.post(
|
||
'https://benchmark.projectnomad.us/api/v1/submit',
|
||
submission,
|
||
{
|
||
timeout: 30000,
|
||
headers: {
|
||
'X-NOMAD-Timestamp': timestamp,
|
||
'X-NOMAD-Signature': signature,
|
||
},
|
||
}
|
||
)
|
||
|
||
if (response.data.success) {
|
||
result.submitted_to_repository = true
|
||
result.submitted_at = DateTime.now()
|
||
result.repository_id = response.data.repository_id
|
||
await result.save()
|
||
|
||
await BenchmarkSetting.setValue('last_benchmark_run', new Date().toISOString())
|
||
}
|
||
|
||
return response.data as RepositorySubmitResponse
|
||
} catch (error) {
|
||
const detail = error.response?.data?.error || error.message || 'Unknown error'
|
||
const statusCode = error.response?.status
|
||
logger.error(`Failed to submit benchmark to repository: ${detail} (Status: ${statusCode})`)
|
||
|
||
// Create an error with the status code attached for proper handling upstream
|
||
const err: any = new Error(`Failed to submit benchmark: ${detail}`)
|
||
err.statusCode = statusCode
|
||
throw err
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Build the NOMAD Score v2 submission payload from a stored result, validating
|
||
* that every required raw channel + companion + provenance field is present.
|
||
*
|
||
* A row produced before this benchmark_version (or an interrupted run) won't
|
||
* carry the v2 raws — rather than send a payload the leaderboard will reject
|
||
* with a terse validator error, we throw a clear "re-run" message here.
|
||
*
|
||
* ai_time_to_first_token is stored in milliseconds but the leaderboard's v2 API
|
||
* treats it as SECONDS, so it is divided by 1000 here.
|
||
*/
|
||
private _buildV2Submission(result: BenchmarkResult, anonymous: boolean): RepositorySubmissionV2 {
|
||
const reRun = 'Re-run a Full Benchmark to share your results.'
|
||
|
||
// Required raw channels — all must be present and > 0.
|
||
const requiredRaws: Array<[string, number | null]> = [
|
||
['CPU single-thread', result.cpu_events_single],
|
||
['CPU multi-thread', result.cpu_events_multi],
|
||
['memory', result.memory_ops_per_sec],
|
||
['disk read', result.disk_read_mb_per_sec],
|
||
['disk write', result.disk_write_mb_per_sec],
|
||
]
|
||
for (const [label, value] of requiredRaws) {
|
||
if (value == null || !(value > 0)) {
|
||
throw new Error(`This benchmark is missing the ${label} raw measurement needed for the v2 leaderboard. ${reRun}`)
|
||
}
|
||
}
|
||
|
||
if (result.ai_time_to_first_token == null || !(result.ai_time_to_first_token > 0)) {
|
||
throw new Error(`This benchmark is missing AI time-to-first-token data. ${reRun}`)
|
||
}
|
||
if (
|
||
result.cpu_benchmark_threads == null ||
|
||
result.memory_threads == null ||
|
||
result.cpu_total_events == null ||
|
||
result.cpu_total_time == null
|
||
) {
|
||
throw new Error(`This benchmark is missing CPU test parameters needed for the v2 leaderboard. ${reRun}`)
|
||
}
|
||
if (!result.sysbench_digest || !result.ollama_version) {
|
||
throw new Error(`This benchmark is missing harness provenance (sysbench/Ollama version). ${reRun}`)
|
||
}
|
||
|
||
const submission: RepositorySubmissionV2 = {
|
||
cpu_model: result.cpu_model,
|
||
cpu_cores: result.cpu_cores,
|
||
cpu_threads: result.cpu_threads,
|
||
ram_gb: Math.round(result.ram_bytes / (1024 * 1024 * 1024)),
|
||
disk_type: result.disk_type,
|
||
gpu_model: result.gpu_model,
|
||
ai_tokens_per_second: result.ai_tokens_per_second!,
|
||
cpu_events_single: result.cpu_events_single!,
|
||
cpu_events_multi: result.cpu_events_multi!,
|
||
memory_ops_per_sec: result.memory_ops_per_sec!,
|
||
disk_read_mb_per_sec: result.disk_read_mb_per_sec!,
|
||
disk_write_mb_per_sec: result.disk_write_mb_per_sec!,
|
||
cpu_benchmark_threads: result.cpu_benchmark_threads,
|
||
memory_threads: result.memory_threads,
|
||
// Stored in ms; the v2 API expects seconds.
|
||
ai_time_to_first_token: result.ai_time_to_first_token / 1000,
|
||
cpu_total_events: result.cpu_total_events,
|
||
cpu_total_time: result.cpu_total_time,
|
||
ollama_version: result.ollama_version,
|
||
sysbench_digest: result.sysbench_digest,
|
||
nomad_version: SystemService.getAppVersion(),
|
||
benchmark_version: BENCHMARK_VERSION,
|
||
}
|
||
|
||
// Optional environment metadata — only send fields we actually have.
|
||
if (result.run_environment) submission.run_environment = result.run_environment
|
||
if (result.storage_path_type) submission.storage_path_type = result.storage_path_type
|
||
if (result.gpu_compute_detected != null) submission.gpu_compute_detected = result.gpu_compute_detected
|
||
|
||
// Builder tag: omit entirely when anonymous or unset (validator rejects null).
|
||
if (!anonymous && result.builder_tag) submission.builder_tag = result.builder_tag
|
||
|
||
return submission
|
||
}
|
||
|
||
/**
|
||
* Get comparison stats from central repository
|
||
*/
|
||
async getComparisonStats(): Promise<RepositoryStats | null> {
|
||
try {
|
||
const response = await axios.get('https://benchmark.projectnomad.us/api/v1/stats', {
|
||
timeout: 10000,
|
||
})
|
||
return response.data as RepositoryStats
|
||
} catch (error) {
|
||
logger.warn(`Failed to fetch comparison stats: ${error.message}`)
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get current benchmark status
|
||
*/
|
||
getStatus(): { status: BenchmarkStatus; benchmarkId: string | null } {
|
||
return {
|
||
status: this.currentStatus,
|
||
benchmarkId: this.currentBenchmarkId,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Detect system hardware information
|
||
*/
|
||
async getHardwareInfo(): Promise<HardwareInfo> {
|
||
this._updateStatus('detecting_hardware', 'Detecting system hardware...')
|
||
|
||
try {
|
||
const [cpu, mem, diskLayout, graphics] = await Promise.all([
|
||
si.cpu(),
|
||
si.mem(),
|
||
si.diskLayout(),
|
||
si.graphics(),
|
||
])
|
||
|
||
// Determine disk type from primary disk
|
||
let diskType: DiskType = 'unknown'
|
||
if (diskLayout.length > 0) {
|
||
const primaryDisk = diskLayout[0]
|
||
if (primaryDisk.type?.toLowerCase().includes('nvme')) {
|
||
diskType = 'nvme'
|
||
} else if (primaryDisk.type?.toLowerCase().includes('ssd')) {
|
||
diskType = 'ssd'
|
||
} else if (primaryDisk.type?.toLowerCase().includes('hdd') || primaryDisk.interfaceType === 'SATA') {
|
||
// SATA could be SSD or HDD, check if it's rotational
|
||
diskType = 'hdd'
|
||
}
|
||
}
|
||
|
||
// Get GPU model (prefer discrete GPU with dedicated VRAM)
|
||
let gpuModel: string | null = null
|
||
if (graphics.controllers && graphics.controllers.length > 0) {
|
||
// First, look for discrete GPUs (NVIDIA, AMD discrete, or any with significant VRAM)
|
||
const discreteGpu = graphics.controllers.find((g) => {
|
||
const vendor = g.vendor?.toLowerCase() || ''
|
||
const model = g.model?.toLowerCase() || ''
|
||
// NVIDIA GPUs are always discrete
|
||
if (vendor.includes('nvidia') || model.includes('geforce') || model.includes('rtx') || model.includes('quadro')) {
|
||
return true
|
||
}
|
||
// AMD discrete GPUs (Radeon, not integrated APU graphics)
|
||
if ((vendor.includes('amd') || vendor.includes('ati')) &&
|
||
(model.includes('radeon') || model.includes('rx ') || model.includes('vega')) &&
|
||
!model.includes('graphics')) {
|
||
return true
|
||
}
|
||
// Any GPU with dedicated VRAM > 512MB is likely discrete
|
||
if (g.vram && g.vram > 512) {
|
||
return true
|
||
}
|
||
return false
|
||
})
|
||
gpuModel = discreteGpu?.model || graphics.controllers[0]?.model || null
|
||
}
|
||
|
||
// Fallback: Check Docker for nvidia runtime and query GPU model via nvidia-smi
|
||
if (!gpuModel) {
|
||
try {
|
||
const dockerInfo = await this.dockerService.docker.info()
|
||
const runtimes = dockerInfo.Runtimes || {}
|
||
if ('nvidia' in runtimes) {
|
||
logger.info('[BenchmarkService] NVIDIA container runtime detected, querying GPU model via nvidia-smi')
|
||
|
||
const systemService = new (await import('./system_service.js')).SystemService(this.dockerService)
|
||
const nvidiaInfo = await systemService.getNvidiaSmiInfo()
|
||
if (Array.isArray(nvidiaInfo) && nvidiaInfo.length > 0) {
|
||
gpuModel = nvidiaInfo[0].model
|
||
} else {
|
||
logger.warn(`[BenchmarkService] NVIDIA runtime detected but failed to get GPU info: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`)
|
||
}
|
||
}
|
||
} catch (dockerError) {
|
||
logger.warn(`[BenchmarkService] Could not query Docker info for GPU detection: ${dockerError.message}`)
|
||
}
|
||
}
|
||
|
||
// Fallback: Extract integrated GPU from CPU model name
|
||
if (!gpuModel) {
|
||
const cpuFullName = `${cpu.manufacturer} ${cpu.brand}`
|
||
|
||
// AMD APUs: e.g., "AMD Ryzen AI 9 HX 370 w/ Radeon 890M" -> "Radeon 890M"
|
||
const radeonMatch = cpuFullName.match(/w\/\s*(Radeon\s+\d+\w*)/i)
|
||
if (radeonMatch) {
|
||
gpuModel = radeonMatch[1]
|
||
}
|
||
|
||
// Intel Core Ultra: These have Intel Arc Graphics integrated
|
||
// e.g., "Intel Core Ultra 9 285HX" -> "Intel Arc Graphics (Integrated)"
|
||
if (!gpuModel && cpu.manufacturer?.toLowerCase().includes('intel')) {
|
||
if (cpu.brand?.toLowerCase().includes('core ultra')) {
|
||
gpuModel = 'Intel Arc Graphics (Integrated)'
|
||
}
|
||
}
|
||
}
|
||
|
||
// Fallback: AMD discrete cards. si.graphics() returns empty inside Docker for AMD,
|
||
// the nvidia-smi path doesn't apply, and the APU regex only catches integrated parts.
|
||
// SystemService.getSystemInfo() already handles AMD via the marker file + Ollama log
|
||
// probe added in PR #804, so reuse that plumbing rather than duplicating it here.
|
||
if (!gpuModel) {
|
||
try {
|
||
const systemService = new (await import('./system_service.js')).SystemService(this.dockerService)
|
||
const sysInfo = await systemService.getSystemInfo()
|
||
const sysGpuModel = sysInfo?.graphics?.controllers?.[0]?.model
|
||
if (sysGpuModel) {
|
||
gpuModel = sysGpuModel
|
||
}
|
||
} catch (sysError: any) {
|
||
logger.warn(`[BenchmarkService] system_service AMD fallback failed: ${sysError.message}`)
|
||
}
|
||
}
|
||
|
||
return {
|
||
cpu_model: `${cpu.manufacturer} ${cpu.brand}`,
|
||
cpu_cores: cpu.physicalCores,
|
||
cpu_threads: cpu.cores,
|
||
ram_bytes: mem.total,
|
||
disk_type: diskType,
|
||
gpu_model: gpuModel,
|
||
}
|
||
} catch (error) {
|
||
logger.error(`Error detecting hardware: ${error.message}`)
|
||
throw new Error(`Failed to detect hardware: ${error.message}`)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Main benchmark execution method
|
||
*/
|
||
private async _runBenchmark(type: BenchmarkType, includeAI: boolean): Promise<BenchmarkResult> {
|
||
if (this.currentStatus !== 'idle') {
|
||
throw new Error('A benchmark is already running')
|
||
}
|
||
|
||
this.currentBenchmarkId = randomUUID()
|
||
this._updateStatus('starting', 'Starting benchmark...')
|
||
|
||
try {
|
||
// Fail fast: if this run will download the (large) AI model, make sure there's
|
||
// room BEFORE spending minutes on the system benchmarks.
|
||
if (includeAI && (type === 'full' || type === 'ai')) {
|
||
await this._preflightModelDiskSpace()
|
||
}
|
||
|
||
// Detect hardware
|
||
const hardware = await this.getHardwareInfo()
|
||
|
||
// Run system benchmarks
|
||
let systemScores: SystemScores = {
|
||
cpu_score: 0,
|
||
memory_score: 0,
|
||
disk_read_score: 0,
|
||
disk_write_score: 0,
|
||
}
|
||
let systemRaws: SystemBenchmarkRawsV2 | null = null
|
||
|
||
if (type === 'full' || type === 'system') {
|
||
const systemOutput = await this._runSystemBenchmarks(hardware.cpu_threads)
|
||
systemScores = systemOutput.scores
|
||
systemRaws = systemOutput.raws
|
||
}
|
||
|
||
// Run AI benchmark if requested and Ollama is available
|
||
let aiScores: Partial<AIScores> = {}
|
||
if (includeAI && (type === 'full' || type === 'ai')) {
|
||
try {
|
||
aiScores = await this._runAIBenchmark()
|
||
} catch (error) {
|
||
// For AI-only benchmarks, failing is fatal - don't save useless results with all zeros
|
||
if (type === 'ai') {
|
||
throw new Error(`AI benchmark failed: ${error.message}. Make sure AI Assistant is installed and running.`)
|
||
}
|
||
// For full benchmarks, AI is optional - continue without it
|
||
logger.warn(`AI benchmark skipped: ${error.message}`)
|
||
}
|
||
}
|
||
|
||
// Calculate NOMAD scores (v1 legacy + v2 uncapped)
|
||
this._updateStatus('calculating_score', 'Calculating NOMAD score...')
|
||
const nomadScore = this._calculateNomadScore(systemScores, aiScores)
|
||
|
||
// v2 requires a complete run of every channel (system raws + AI). Only a
|
||
// full benchmark with AI qualifies; otherwise nomad_score_v2 stays null.
|
||
let nomadScoreV2: number | null = null
|
||
if (systemRaws && aiScores.ai_tokens_per_second && aiScores.ai_tokens_per_second > 0) {
|
||
try {
|
||
nomadScoreV2 = this._calculateNomadScoreV2({
|
||
ai_tokens_per_second: aiScores.ai_tokens_per_second,
|
||
cpu_events_single: systemRaws.cpu_events_single,
|
||
cpu_events_multi: systemRaws.cpu_events_multi,
|
||
memory_ops_per_sec: systemRaws.memory_ops_per_sec,
|
||
disk_read_mb_per_sec: systemRaws.disk_read_mb_per_sec,
|
||
disk_write_mb_per_sec: systemRaws.disk_write_mb_per_sec,
|
||
})
|
||
} catch (error) {
|
||
// A non-positive channel shouldn't reach here (each sysbench step throws
|
||
// on a bad parse), but never let a v2 math error sink the whole run.
|
||
logger.warn(`NOMAD Score v2 not computed: ${error.message}`)
|
||
}
|
||
}
|
||
|
||
// Best-effort environment metadata (never fails the run).
|
||
const env = await this._detectRunEnvironment()
|
||
|
||
// Save result
|
||
const result = await BenchmarkResult.create({
|
||
benchmark_id: this.currentBenchmarkId,
|
||
benchmark_type: type,
|
||
cpu_model: hardware.cpu_model,
|
||
cpu_cores: hardware.cpu_cores,
|
||
cpu_threads: hardware.cpu_threads,
|
||
ram_bytes: hardware.ram_bytes,
|
||
disk_type: hardware.disk_type,
|
||
gpu_model: hardware.gpu_model,
|
||
cpu_score: systemScores.cpu_score,
|
||
memory_score: systemScores.memory_score,
|
||
disk_read_score: systemScores.disk_read_score,
|
||
disk_write_score: systemScores.disk_write_score,
|
||
ai_tokens_per_second: aiScores.ai_tokens_per_second || null,
|
||
ai_model_used: aiScores.ai_model_used || null,
|
||
ai_time_to_first_token: aiScores.ai_time_to_first_token || null,
|
||
nomad_score: nomadScore,
|
||
submitted_to_repository: false,
|
||
sysbench_digest: SYSBENCH_DIGEST,
|
||
ollama_version: aiScores.ai_ollama_version ?? null,
|
||
// v2 raw channels + score (null on system-only / AI-less runs)
|
||
cpu_events_single: systemRaws?.cpu_events_single ?? null,
|
||
cpu_events_multi: systemRaws?.cpu_events_multi ?? null,
|
||
cpu_benchmark_threads: systemRaws?.cpu_benchmark_threads ?? null,
|
||
cpu_total_events: systemRaws?.cpu_total_events ?? null,
|
||
cpu_total_time: systemRaws?.cpu_total_time ?? null,
|
||
memory_ops_per_sec: systemRaws?.memory_ops_per_sec ?? null,
|
||
memory_threads: systemRaws?.memory_threads ?? null,
|
||
disk_read_mb_per_sec: systemRaws?.disk_read_mb_per_sec ?? null,
|
||
disk_write_mb_per_sec: systemRaws?.disk_write_mb_per_sec ?? null,
|
||
nomad_score_v2: nomadScoreV2,
|
||
run_environment: env.run_environment,
|
||
storage_path_type: env.storage_path_type,
|
||
gpu_compute_detected: env.gpu_compute_detected,
|
||
})
|
||
|
||
this._updateStatus('completed', 'Benchmark completed successfully')
|
||
this.currentStatus = 'idle'
|
||
this.currentBenchmarkId = null
|
||
|
||
return result
|
||
} catch (error) {
|
||
this._updateStatus('error', `Benchmark failed: ${error.message}`)
|
||
this.currentStatus = 'idle'
|
||
this.currentBenchmarkId = null
|
||
throw error
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Run system benchmarks using sysbench in Docker.
|
||
*
|
||
* Produces both the legacy v1 0-1 sub-scores AND the v2 raw channels from the
|
||
* same runs. CPU runs twice: 1 thread (single-core index) and all logical
|
||
* threads (multi-core index); the v1 cpu_score keys off the multi pass to stay
|
||
* comparable with historical results.
|
||
*
|
||
* @param multiThreads number of threads for the multi-core CPU pass (all logical CPUs)
|
||
*/
|
||
private async _runSystemBenchmarks(multiThreads: number): Promise<SystemBenchmarkOutput> {
|
||
// Ensure sysbench image is available
|
||
await this._ensureSysbenchImage()
|
||
|
||
// Run CPU benchmarks: single-thread then all-thread.
|
||
this._updateStatus('running_cpu', 'Running CPU benchmark (single-thread)...')
|
||
const cpuSingle = await this._runSysbenchCpu(1)
|
||
|
||
this._updateStatus('running_cpu', `Running CPU benchmark (${multiThreads}-thread)...`)
|
||
const cpuMulti = await this._runSysbenchCpu(multiThreads)
|
||
|
||
// Run memory benchmark
|
||
this._updateStatus('running_memory', 'Running memory benchmark...')
|
||
const memoryResult = await this._runSysbenchMemory()
|
||
|
||
// Run disk benchmarks
|
||
this._updateStatus('running_disk_read', 'Running disk read benchmark...')
|
||
const diskReadResult = await this._runSysbenchDiskRead()
|
||
|
||
this._updateStatus('running_disk_write', 'Running disk write benchmark...')
|
||
const diskWriteResult = await this._runSysbenchDiskWrite()
|
||
|
||
// Legacy v1 sub-scores (0-1), normalized against the v1 references. cpu_score
|
||
// uses the multi-thread pass to match historical (4-thread) behaviour closely.
|
||
const scores: SystemScores = {
|
||
cpu_score: this._normalizeScore(cpuMulti.events_per_second, REFERENCE_SCORES.cpu_events_per_second),
|
||
memory_score: this._normalizeScore(memoryResult.operations_per_second, REFERENCE_SCORES.memory_ops_per_second),
|
||
disk_read_score: this._normalizeScore(diskReadResult.read_mb_per_sec, REFERENCE_SCORES.disk_read_mb_per_sec),
|
||
disk_write_score: this._normalizeScore(diskWriteResult.write_mb_per_sec, REFERENCE_SCORES.disk_write_mb_per_sec),
|
||
}
|
||
|
||
// v2 raw channels — the payload of record for the leaderboard.
|
||
const raws: SystemBenchmarkRawsV2 = {
|
||
cpu_events_single: cpuSingle.events_per_second,
|
||
cpu_events_multi: cpuMulti.events_per_second,
|
||
cpu_benchmark_threads: multiThreads,
|
||
cpu_total_events: cpuMulti.total_events,
|
||
cpu_total_time: cpuMulti.total_time,
|
||
memory_ops_per_sec: memoryResult.operations_per_second,
|
||
memory_threads: MEMORY_BENCHMARK_THREADS,
|
||
disk_read_mb_per_sec: diskReadResult.read_mb_per_sec,
|
||
disk_write_mb_per_sec: diskWriteResult.write_mb_per_sec,
|
||
}
|
||
|
||
return { scores, raws }
|
||
}
|
||
|
||
/**
|
||
* Run AI benchmark using Ollama
|
||
*/
|
||
private async _runAIBenchmark(): Promise<AIScores> {
|
||
try {
|
||
|
||
this._updateStatus('running_ai', 'Running AI benchmark...')
|
||
|
||
const ollamaAPIURL = await this.dockerService.getServiceURL(SERVICE_NAMES.OLLAMA)
|
||
if (!ollamaAPIURL) {
|
||
throw new Error('AI Assistant service location could not be determined. Ensure AI Assistant is installed and running.')
|
||
}
|
||
|
||
// Check if Ollama is available
|
||
try {
|
||
await axios.get(`${ollamaAPIURL}/api/tags`, { timeout: 5000 })
|
||
} catch (error) {
|
||
const errorCode = error.code || error.response?.status || 'unknown'
|
||
throw new Error(`Ollama is not running or not accessible (${errorCode}). Ensure AI Assistant is installed and running.`)
|
||
}
|
||
|
||
// Record the Ollama version for forensics (null-tolerant — never fail the run on this)
|
||
const versionResp = await axios.get(`${ollamaAPIURL}/api/version`, { timeout: 5000 }).catch(() => null)
|
||
const ollamaVersion: string | null = versionResp?.data?.version ?? null
|
||
|
||
// Check if the benchmark model is available, pull if not
|
||
const ollamaService = new (await import('./ollama_service.js')).OllamaService()
|
||
const modelResponse = await ollamaService.downloadModel(AI_BENCHMARK_MODEL)
|
||
if (!modelResponse.success) {
|
||
throw new Error(`Model does not exist and failed to download: ${modelResponse.message}`)
|
||
}
|
||
|
||
// Evict any other models resident in VRAM/RAM before benchmarking. With an 8B
|
||
// reference model, a chat model left loaded can starve it of VRAM → Ollama
|
||
// offloads layers to CPU → a low, non-comparable score. Give the benchmark
|
||
// model the whole GPU. Best-effort: never fail the run on an eviction hiccup.
|
||
await this._unloadResidentModels(ollamaAPIURL)
|
||
|
||
// Run inference AI_BENCHMARK_RUNS times and take the median run by tok/s (W7).
|
||
// A single inference is noisy (scheduler, thermal, cache); the median damps
|
||
// outliers without a warmup-run bias. TTFT is reported from the same run that
|
||
// is the tok/s median, so the two figures always describe one real inference.
|
||
const runs: { tps: number; ttft: number }[] = []
|
||
for (let i = 0; i < AI_BENCHMARK_RUNS; i++) {
|
||
this._updateStatus('running_ai', `Running AI benchmark (${i + 1}/${AI_BENCHMARK_RUNS})...`)
|
||
runs.push(await this._runSingleAIInference(ollamaAPIURL))
|
||
}
|
||
|
||
// Median by tok/s (odd count → middle element).
|
||
runs.sort((a, b) => a.tps - b.tps)
|
||
const median = runs[Math.floor(runs.length / 2)]
|
||
|
||
// Leave the GPU as clean as we found it: unload our benchmark model so the
|
||
// user's next chat loads their model into a free GPU (Ollama reloads on demand;
|
||
// we deliberately don't try to restore whichever model they had before).
|
||
await this._unloadResidentModels(ollamaAPIURL)
|
||
|
||
return {
|
||
ai_tokens_per_second: Math.round(median.tps * 100) / 100,
|
||
ai_model_used: AI_BENCHMARK_MODEL,
|
||
ai_time_to_first_token: Math.round(median.ttft * 100) / 100,
|
||
ai_ollama_version: ollamaVersion,
|
||
}
|
||
} catch (error) {
|
||
throw new Error(`AI benchmark failed: ${error.message}`)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Run a single Ollama inference and return its tokens/sec and time-to-first-token
|
||
* (TTFT in milliseconds). Extracted so the AI benchmark can run it N times for a
|
||
* median. Prefers Ollama's eval_count/eval_duration; falls back to wall-clock
|
||
* word-count estimation if those fields are absent.
|
||
*/
|
||
private async _runSingleAIInference(
|
||
ollamaAPIURL: string
|
||
): Promise<{ tps: number; ttft: number }> {
|
||
const startTime = Date.now()
|
||
|
||
const response = await axios.post(
|
||
`${ollamaAPIURL}/api/generate`,
|
||
{
|
||
model: AI_BENCHMARK_MODEL,
|
||
prompt: AI_BENCHMARK_PROMPT,
|
||
stream: false,
|
||
options: { num_predict: AI_BENCHMARK_NUM_PREDICT },
|
||
},
|
||
{ timeout: 120000 }
|
||
)
|
||
|
||
const endTime = Date.now()
|
||
const totalTime = (endTime - startTime) / 1000 // seconds
|
||
|
||
// Ollama returns eval_count (tokens generated) and eval_duration (nanoseconds)
|
||
if (response.data.eval_count && response.data.eval_duration) {
|
||
const tokenCount = response.data.eval_count
|
||
const evalDurationSeconds = response.data.eval_duration / 1e9
|
||
const tokensPerSecond = tokenCount / evalDurationSeconds
|
||
|
||
// Time to first token from prompt_eval_duration
|
||
const ttft = response.data.prompt_eval_duration
|
||
? response.data.prompt_eval_duration / 1e6 // Convert to ms
|
||
: (totalTime * 1000) / 2 // Estimate if not available
|
||
|
||
return { tps: tokensPerSecond, ttft }
|
||
}
|
||
|
||
// Fallback calculation
|
||
const estimatedTokens = response.data.response?.split(' ').length * 1.3 || 100
|
||
const tokensPerSecond = estimatedTokens / totalTime
|
||
return { tps: tokensPerSecond, ttft: (totalTime * 1000) / 2 }
|
||
}
|
||
|
||
/**
|
||
* Fail fast if the AI model will need downloading and there isn't room for it.
|
||
*
|
||
* Only relevant on first run: if the model is already pulled, no download happens
|
||
* and we skip the check. Best-effort throughout — if Ollama isn't reachable or free
|
||
* space can't be determined, we do NOT block (the AI stage will surface its own
|
||
* error), because a false "insufficient disk" is worse than letting the pull try.
|
||
*/
|
||
private async _preflightModelDiskSpace(): Promise<void> {
|
||
let ollamaAPIURL: string | null = null
|
||
try {
|
||
ollamaAPIURL = await this.dockerService.getServiceURL(SERVICE_NAMES.OLLAMA)
|
||
} catch {
|
||
return // AI stage will report Ollama problems
|
||
}
|
||
if (!ollamaAPIURL) return
|
||
|
||
// If the model is already present, no download → no disk need.
|
||
try {
|
||
const tags = await axios.get(`${ollamaAPIURL}/api/tags`, { timeout: 5000 })
|
||
const models: string[] = (tags.data?.models ?? []).map((m: { name?: string }) => m.name)
|
||
if (models.includes(AI_BENCHMARK_MODEL)) return
|
||
} catch {
|
||
return // can't reach Ollama; let the AI stage handle it
|
||
}
|
||
|
||
// Model not cached → a pull is coming. Require headroom on the root FS.
|
||
let freeBytes: number | null = null
|
||
try {
|
||
const systemService = new (await import('./system_service.js')).SystemService(this.dockerService)
|
||
freeBytes = await getFreeBytes(systemService)
|
||
} catch {
|
||
return // couldn't measure free space — don't block on uncertainty
|
||
}
|
||
if (freeBytes === null) return
|
||
|
||
if (freeBytes < AI_BENCHMARK_MODEL_MIN_FREE_BYTES) {
|
||
const gib = (b: number) => `${(b / 1024 / 1024 / 1024).toFixed(1)} GB`
|
||
throw new Error(
|
||
`Not enough free disk space to download the AI benchmark model (${AI_BENCHMARK_MODEL}). ` +
|
||
`About ${gib(AI_BENCHMARK_MODEL_MIN_FREE_BYTES)} is required but only ${gib(freeBytes)} is free. ` +
|
||
`Free up some space and try again.`
|
||
)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Evict every model currently resident in Ollama (VRAM/RAM) so the benchmark
|
||
* model gets the whole GPU. Queries /api/ps for loaded models and unloads each
|
||
* via a keep_alive:0 request. Entirely best-effort: any failure (older Ollama
|
||
* without /api/ps, a model mid-stream) is logged and swallowed — a clean-VRAM
|
||
* optimization must never block the benchmark itself.
|
||
*/
|
||
private async _unloadResidentModels(ollamaAPIURL: string): Promise<void> {
|
||
try {
|
||
const ps = await axios.get(`${ollamaAPIURL}/api/ps`, { timeout: 5000 })
|
||
const loaded: string[] = (ps.data?.models ?? [])
|
||
.map((m: { name?: string; model?: string }) => m.name || m.model)
|
||
.filter((n: string | undefined): n is string => !!n)
|
||
|
||
if (loaded.length === 0) return
|
||
|
||
logger.info(`[BenchmarkService] Evicting ${loaded.length} resident model(s) before AI benchmark: ${loaded.join(', ')}`)
|
||
for (const model of loaded) {
|
||
// keep_alive:0 tells Ollama to unload immediately (no prompt needed).
|
||
await axios
|
||
.post(`${ollamaAPIURL}/api/generate`, { model, keep_alive: 0 }, { timeout: 15000 })
|
||
.catch((e) => logger.warn(`[BenchmarkService] Failed to unload ${model}: ${e.message}`))
|
||
}
|
||
} catch (error) {
|
||
logger.warn(`[BenchmarkService] Could not enumerate/unload resident models (continuing): ${error.message}`)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Calculate weighted NOMAD score
|
||
*/
|
||
private _calculateNomadScore(systemScores: SystemScores, aiScores: Partial<AIScores>): number {
|
||
let totalWeight = 0
|
||
let weightedSum = 0
|
||
|
||
// CPU score
|
||
weightedSum += systemScores.cpu_score * SCORE_WEIGHTS.cpu
|
||
totalWeight += SCORE_WEIGHTS.cpu
|
||
|
||
// Memory score
|
||
weightedSum += systemScores.memory_score * SCORE_WEIGHTS.memory
|
||
totalWeight += SCORE_WEIGHTS.memory
|
||
|
||
// Disk scores
|
||
weightedSum += systemScores.disk_read_score * SCORE_WEIGHTS.disk_read
|
||
totalWeight += SCORE_WEIGHTS.disk_read
|
||
weightedSum += systemScores.disk_write_score * SCORE_WEIGHTS.disk_write
|
||
totalWeight += SCORE_WEIGHTS.disk_write
|
||
|
||
// AI scores (if available)
|
||
if (aiScores.ai_tokens_per_second !== undefined && aiScores.ai_tokens_per_second !== null) {
|
||
const aiScore = this._normalizeScore(
|
||
aiScores.ai_tokens_per_second,
|
||
REFERENCE_SCORES.ai_tokens_per_second
|
||
)
|
||
weightedSum += aiScore * SCORE_WEIGHTS.ai_tokens_per_second
|
||
totalWeight += SCORE_WEIGHTS.ai_tokens_per_second
|
||
}
|
||
|
||
if (aiScores.ai_time_to_first_token !== undefined && aiScores.ai_time_to_first_token !== null) {
|
||
// For TTFT, lower is better, so we invert the score
|
||
const ttftScore = this._normalizeScoreInverse(
|
||
aiScores.ai_time_to_first_token,
|
||
REFERENCE_SCORES.ai_ttft_ms
|
||
)
|
||
weightedSum += ttftScore * SCORE_WEIGHTS.ai_ttft
|
||
totalWeight += SCORE_WEIGHTS.ai_ttft
|
||
}
|
||
|
||
// Normalize by actual weight used (in case AI benchmarks were skipped)
|
||
const nomadScore = totalWeight > 0 ? (weightedSum / totalWeight) * 100 : 0
|
||
|
||
return Math.round(Math.min(100, Math.max(0, nomadScore)) * 100) / 100
|
||
}
|
||
|
||
/**
|
||
* Calculate the NOMAD Score v2 from raw channel values.
|
||
*
|
||
* Byte-identical to the leaderboard's computeNomadScoreV2 (score_service.ts):
|
||
* the weighted geometric mean vs the frozen Reference Build, computed in the
|
||
* log domain, rounded to one decimal. NO clamps. Throws if any channel is
|
||
* <= 0 — a full v2 run measures every channel, and log2 of a non-positive
|
||
* ratio is undefined, so a non-positive value here is a programming error, not
|
||
* a user input to clamp. Every scored channel (incl. AI) is required; callers
|
||
* only invoke this for a complete full-benchmark run.
|
||
*/
|
||
private _calculateNomadScoreV2(raws: ScoreRawsV2): number {
|
||
let logSum = 0
|
||
for (const channel of SCORE_CHANNELS_V2) {
|
||
const raw = raws[channel]
|
||
if (!(raw > 0)) {
|
||
throw new Error(`_calculateNomadScoreV2: channel "${channel}" must be > 0 (got ${raw})`)
|
||
}
|
||
const ratio = raw / REFERENCE_SCORES_V2[channel]
|
||
logSum += SCORE_WEIGHTS_V2[channel] * Math.log2(ratio)
|
||
}
|
||
const score = 1000 * Math.pow(2, logSum)
|
||
// One decimal, matching the leaderboard's display convention.
|
||
return Math.round(score * 10) / 10
|
||
}
|
||
|
||
/**
|
||
* Detect best-effort run-environment metadata for the leaderboard (issue #1016).
|
||
* Every probe is independently guarded — this never throws and never fails a run;
|
||
* any field it can't determine stays null.
|
||
*/
|
||
private async _detectRunEnvironment(): Promise<RunEnvironmentInfo> {
|
||
const info: RunEnvironmentInfo = {
|
||
run_environment: null,
|
||
storage_path_type: null,
|
||
gpu_compute_detected: null,
|
||
}
|
||
|
||
// run_environment: WSL2 vs native Linux, from the host kernel string.
|
||
try {
|
||
const procVersion = await readFile('/proc/version', 'utf8')
|
||
info.run_environment = /microsoft|wsl/i.test(procVersion) ? 'wsl2' : 'native-linux'
|
||
} catch {
|
||
// /proc/version unreadable — leave null
|
||
}
|
||
|
||
// storage_path_type: the Docker storage driver backing the benchmarked FS.
|
||
try {
|
||
const dockerInfo = await this.dockerService.docker.info()
|
||
if (dockerInfo?.Driver) info.storage_path_type = String(dockerInfo.Driver)
|
||
} catch {
|
||
// docker info failed — leave null
|
||
}
|
||
|
||
// gpu_compute_detected: whether a GPU compute type was detected + persisted.
|
||
try {
|
||
const gpuType = await KVStore.getValue('gpu.type')
|
||
info.gpu_compute_detected = gpuType === 'nvidia' || gpuType === 'amd'
|
||
} catch {
|
||
// KV read failed — leave null
|
||
}
|
||
|
||
return info
|
||
}
|
||
|
||
/**
|
||
* Normalize a raw score to 0-100 scale using log scaling
|
||
* This provides diminishing returns for very high scores
|
||
*/
|
||
private _normalizeScore(value: number, reference: number): number {
|
||
if (value <= 0) return 0
|
||
// Log scale with widened range: dividing log2 by 3 prevents scores from
|
||
// clamping to 0% for below-average hardware. Gives 50% at reference value.
|
||
const ratio = value / reference
|
||
const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)) / 3)
|
||
return Math.min(100, Math.max(0, score)) / 100
|
||
}
|
||
|
||
/**
|
||
* Normalize a score where lower is better (like latency)
|
||
*/
|
||
private _normalizeScoreInverse(value: number, reference: number): number {
|
||
if (value <= 0) return 1
|
||
// Inverse: lower values = higher scores, with widened log range
|
||
const ratio = reference / value
|
||
const score = 50 * (1 + Math.log2(Math.max(0.01, ratio)) / 3)
|
||
return Math.min(100, Math.max(0, score)) / 100
|
||
}
|
||
|
||
/**
|
||
* Ensure sysbench Docker image is available
|
||
*/
|
||
private async _ensureSysbenchImage(): Promise<void> {
|
||
try {
|
||
await this.dockerService.docker.getImage(SYSBENCH_IMAGE).inspect()
|
||
} catch {
|
||
this._updateStatus('starting', `Pulling sysbench image...`)
|
||
await this.dockerService.pullImage(SYSBENCH_IMAGE)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Run sysbench CPU benchmark at the given thread count.
|
||
*
|
||
* v2 runs this twice: 1 thread (single-core index) and all threads (multi-core
|
||
* index). The multi pass also supplies the W6 consistency companions
|
||
* (total_events / total_time) the leaderboard cross-checks against events/sec.
|
||
*/
|
||
private async _runSysbenchCpu(threads: number): Promise<SysbenchCpuResult> {
|
||
const output = await this._runSysbenchCommand([
|
||
'sysbench',
|
||
'cpu',
|
||
'--cpu-max-prime=20000',
|
||
`--threads=${threads}`,
|
||
'--time=30',
|
||
'run',
|
||
])
|
||
|
||
// Parse output for events per second
|
||
const eventsMatch = output.match(/events per second:\s*([\d.]+)/i)
|
||
const totalTimeMatch = output.match(/total time:\s*([\d.]+)s/i)
|
||
const totalEventsMatch = output.match(/total number of events:\s*(\d+)/i)
|
||
logger.debug(`[BenchmarkService] CPU output parsing (${threads}T) - events/s: ${eventsMatch?.[1]}, total_time: ${totalTimeMatch?.[1]}, total_events: ${totalEventsMatch?.[1]}`)
|
||
|
||
// Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
|
||
const eventsPerSecond = eventsMatch ? parseFloat(eventsMatch[1]) : Number.NaN
|
||
if (!eventsMatch || !Number.isFinite(eventsPerSecond) || eventsPerSecond <= 0) {
|
||
throw new Error('sysbench CPU benchmark produced no parseable events/sec — aborting (output may have changed or the test failed)')
|
||
}
|
||
|
||
const totalTime = totalTimeMatch ? parseFloat(totalTimeMatch[1]) : 30
|
||
// total_events feeds the leaderboard's W6 cross-check (events/sec must equal
|
||
// total_events/total_time within 2%). If the count line ever fails to parse,
|
||
// derive it from the rate so we submit a self-consistent payload rather than
|
||
// a zero that would be rejected as "internally inconsistent".
|
||
const parsedTotalEvents = totalEventsMatch ? parseInt(totalEventsMatch[1]) : Number.NaN
|
||
const totalEvents =
|
||
Number.isFinite(parsedTotalEvents) && parsedTotalEvents > 0
|
||
? parsedTotalEvents
|
||
: Math.round(eventsPerSecond * totalTime)
|
||
|
||
return {
|
||
events_per_second: eventsPerSecond,
|
||
total_time: totalTime,
|
||
total_events: totalEvents,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Run sysbench memory benchmark
|
||
*/
|
||
private async _runSysbenchMemory(): Promise<SysbenchMemoryResult> {
|
||
const output = await this._runSysbenchCommand([
|
||
'sysbench',
|
||
'memory',
|
||
'--memory-block-size=1K',
|
||
'--memory-total-size=10G',
|
||
`--threads=${MEMORY_BENCHMARK_THREADS}`,
|
||
'run',
|
||
])
|
||
|
||
// Parse output
|
||
const opsMatch = output.match(/Total operations:\s*\d+\s*\(([\d.]+)\s*per second\)/i)
|
||
const transferMatch = output.match(/([\d.]+)\s*MiB\/sec/i)
|
||
const timeMatch = output.match(/total time:\s*([\d.]+)s/i)
|
||
|
||
// Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
|
||
const opsPerSecond = opsMatch ? parseFloat(opsMatch[1]) : Number.NaN
|
||
if (!opsMatch || !Number.isFinite(opsPerSecond) || opsPerSecond <= 0) {
|
||
throw new Error('sysbench memory benchmark produced no parseable ops/sec — aborting')
|
||
}
|
||
|
||
return {
|
||
operations_per_second: opsPerSecond,
|
||
transfer_rate_mb_per_sec: transferMatch ? parseFloat(transferMatch[1]) : 0,
|
||
total_time: timeMatch ? parseFloat(timeMatch[1]) : 0,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Run sysbench disk read benchmark
|
||
*/
|
||
private async _runSysbenchDiskRead(): Promise<SysbenchDiskResult> {
|
||
// Run prepare, test, and cleanup in a single container
|
||
// This is necessary because each container has its own filesystem
|
||
// --file-extra-flags=direct (O_DIRECT) on the run bypasses the page cache so
|
||
// we measure real device throughput, not RAM (W4). Only the run needs it;
|
||
// prepare/cleanup are plain file ops. v2 reference disk numbers are O_DIRECT.
|
||
const output = await this._runSysbenchCommand([
|
||
'sh',
|
||
'-c',
|
||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqrd --file-extra-flags=direct --time=30 run && ' +
|
||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||
])
|
||
|
||
// Parse output - look for the Throughput section
|
||
const readMatch = output.match(/read,\s*MiB\/s:\s*([\d.]+)/i)
|
||
const readsPerSecMatch = output.match(/reads\/s:\s*([\d.]+)/i)
|
||
|
||
logger.debug(`[BenchmarkService] Disk read output parsing - read: ${readMatch?.[1]}, reads/s: ${readsPerSecMatch?.[1]}`)
|
||
|
||
// Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
|
||
const readMbPerSec = readMatch ? parseFloat(readMatch[1]) : Number.NaN
|
||
if (!readMatch || !Number.isFinite(readMbPerSec) || readMbPerSec <= 0) {
|
||
throw new Error('sysbench disk-read benchmark produced no parseable MiB/s — aborting')
|
||
}
|
||
|
||
return {
|
||
reads_per_second: readsPerSecMatch ? parseFloat(readsPerSecMatch[1]) : 0,
|
||
writes_per_second: 0,
|
||
read_mb_per_sec: readMbPerSec,
|
||
write_mb_per_sec: 0,
|
||
total_time: 30,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Run sysbench disk write benchmark
|
||
*/
|
||
private async _runSysbenchDiskWrite(): Promise<SysbenchDiskResult> {
|
||
// Run prepare, test, and cleanup in a single container
|
||
// This is necessary because each container has its own filesystem
|
||
// --file-extra-flags=direct (O_DIRECT) on the run bypasses the page cache (W4).
|
||
const output = await this._runSysbenchCommand([
|
||
'sh',
|
||
'-c',
|
||
'sysbench fileio --file-total-size=1G --file-num=4 prepare && ' +
|
||
'sysbench fileio --file-total-size=1G --file-num=4 --file-test-mode=seqwr --file-extra-flags=direct --time=30 run && ' +
|
||
'sysbench fileio --file-total-size=1G --file-num=4 cleanup',
|
||
])
|
||
|
||
// Parse output - look for the Throughput section
|
||
const writeMatch = output.match(/written,\s*MiB\/s:\s*([\d.]+)/i)
|
||
const writesPerSecMatch = output.match(/writes\/s:\s*([\d.]+)/i)
|
||
|
||
logger.debug(`[BenchmarkService] Disk write output parsing - written: ${writeMatch?.[1]}, writes/s: ${writesPerSecMatch?.[1]}`)
|
||
|
||
// Scored primary metric: fail loudly instead of silently scoring zero on a parse miss
|
||
const writeMbPerSec = writeMatch ? parseFloat(writeMatch[1]) : Number.NaN
|
||
if (!writeMatch || !Number.isFinite(writeMbPerSec) || writeMbPerSec <= 0) {
|
||
throw new Error('sysbench disk-write benchmark produced no parseable MiB/s — aborting')
|
||
}
|
||
|
||
return {
|
||
reads_per_second: 0,
|
||
writes_per_second: writesPerSecMatch ? parseFloat(writesPerSecMatch[1]) : 0,
|
||
read_mb_per_sec: 0,
|
||
write_mb_per_sec: writeMbPerSec,
|
||
total_time: 30,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Run a sysbench command in a Docker container
|
||
*/
|
||
private async _runSysbenchCommand(cmd: string[]): Promise<string> {
|
||
let container: Dockerode.Container | null = null
|
||
try {
|
||
// Create container with TTY to avoid multiplexed output
|
||
container = await this.dockerService.docker.createContainer({
|
||
Image: SYSBENCH_IMAGE,
|
||
Cmd: cmd,
|
||
name: `${SYSBENCH_CONTAINER_NAME}_${Date.now()}`,
|
||
Tty: true, // Important: prevents multiplexed stdout/stderr headers
|
||
HostConfig: {
|
||
AutoRemove: false, // Don't auto-remove to avoid race condition with fetching logs
|
||
},
|
||
})
|
||
|
||
// Start container
|
||
await container.start()
|
||
|
||
// Wait for completion
|
||
await container.wait()
|
||
|
||
// Get logs after container has finished
|
||
const logs = await container.logs({
|
||
stdout: true,
|
||
stderr: true,
|
||
})
|
||
|
||
// Parse logs (Docker logs include header bytes)
|
||
const output = logs.toString('utf8')
|
||
.replace(/[\x00-\x08]/g, '') // Remove control characters
|
||
.trim()
|
||
|
||
// Manually remove the container after getting logs
|
||
try {
|
||
await container.remove()
|
||
} catch (removeError) {
|
||
// Log but don't fail if removal fails (container might already be gone)
|
||
logger.warn(`Failed to remove sysbench container: ${removeError.message}`)
|
||
}
|
||
|
||
return output
|
||
} catch (error) {
|
||
// Clean up container on error if it exists
|
||
if (container) {
|
||
try {
|
||
await container.remove({ force: true })
|
||
} catch (removeError) {
|
||
// Ignore removal errors
|
||
}
|
||
}
|
||
logger.error(`Sysbench command failed: ${error.message}`)
|
||
throw new Error(`Sysbench command failed: ${error.message}`)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Broadcast benchmark progress update
|
||
*/
|
||
private _updateStatus(status: BenchmarkStatus, message: string) {
|
||
this.currentStatus = status
|
||
|
||
const progress: BenchmarkProgress = {
|
||
status,
|
||
progress: this._getProgressPercent(status),
|
||
message,
|
||
current_stage: this._getStageLabel(status),
|
||
timestamp: new Date().toISOString(),
|
||
}
|
||
|
||
transmit.broadcast(BROADCAST_CHANNELS.BENCHMARK_PROGRESS, {
|
||
benchmark_id: this.currentBenchmarkId,
|
||
...progress,
|
||
})
|
||
|
||
logger.info(`[BenchmarkService] ${status}: ${message}`)
|
||
}
|
||
|
||
/**
|
||
* Get progress percentage for a given status
|
||
*/
|
||
private _getProgressPercent(status: BenchmarkStatus): number {
|
||
const progressMap: Record<BenchmarkStatus, number> = {
|
||
idle: 0,
|
||
starting: 5,
|
||
detecting_hardware: 10,
|
||
running_cpu: 25,
|
||
running_memory: 40,
|
||
running_disk_read: 55,
|
||
running_disk_write: 70,
|
||
downloading_ai_model: 80,
|
||
running_ai: 85,
|
||
calculating_score: 95,
|
||
completed: 100,
|
||
error: 0,
|
||
}
|
||
return progressMap[status] || 0
|
||
}
|
||
|
||
/**
|
||
* Get human-readable stage label
|
||
*/
|
||
private _getStageLabel(status: BenchmarkStatus): string {
|
||
const labelMap: Record<BenchmarkStatus, string> = {
|
||
idle: 'Idle',
|
||
starting: 'Starting',
|
||
detecting_hardware: 'Detecting Hardware',
|
||
running_cpu: 'CPU Benchmark',
|
||
running_memory: 'Memory Benchmark',
|
||
running_disk_read: 'Disk Read Test',
|
||
running_disk_write: 'Disk Write Test',
|
||
downloading_ai_model: 'Downloading AI Model',
|
||
running_ai: 'AI Inference Test',
|
||
calculating_score: 'Calculating Score',
|
||
completed: 'Complete',
|
||
error: 'Error',
|
||
}
|
||
return labelMap[status] || status
|
||
}
|
||
}
|