From d1535d17b9e1ba392bfeedefa47a46ab0dd0d797 Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:17:14 -0700 Subject: [PATCH] fix(benchmark): block leaderboard submission when AI runs on a remote host (#1157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DockerService.getServiceURL() resolves ai.remoteOllamaUrl ahead of the local container, so when a remote AI host is configured the AI channel measures THAT machine while every other channel measures this one. The submission then reports someone else's tok/s under this hardware's CPU, RAM and disk. Under v2 this matters more than it did under v1: ai_tokens_per_second carries 0.30 of the weight and the score is uncapped, so a remote GPU's throughput is no longer limited by a clamp. Adds a guard alongside the existing submit-time checks. Uses the same truthiness predicate as getServiceURL, so the guard fires exactly when the remote routing it is guarding against would occur. KVStore.clearValue() nulls the value and getValue() returns null for that, so a cleared key correctly does not trip it. Blocks submission only. Running the benchmark locally is still useful to the operator — it just isn't a result about this box, so it shouldn't go on a board that ranks hardware. Known limitation: the check reads the KV at submit time, not at measurement time, so benchmarking with a remote host and then clearing the setting before submitting would still get through. That is a deliberate workaround rather than an accident, and closing it properly needs a column on benchmark_results to record how inference was reached. Worth doing if it ever shows up in practice; not worth a migration on the evidence available. Refs #1151 --- admin/app/services/benchmark_service.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index bbac885..71f5865 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -260,6 +260,25 @@ export class BenchmarkService { throw new Error('Benchmark result has already been submitted') } + // Remote inference cannot be attributed to this machine. + // + // DockerService.getServiceURL() resolves ai.remoteOllamaUrl ahead of the + // local container, so when a remote host is configured the AI channel + // measures THAT machine while every other channel measures this one. The + // submission would report someone else's tok/s under this hardware, and AI + // carries 0.30 of an uncapped v2 score, so the distortion is large. + // + // Blocks submission only — running the benchmark locally is still useful to + // the operator, it just isn't a result about this box. + const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl') + if (remoteOllamaUrl) { + throw new Error( + 'This NOMAD is configured to use a remote AI host, so its AI results measure that machine rather than this one. ' + + 'Leaderboard results must be measured entirely on the submitting hardware. ' + + 'To share a result, install the local AI Assistant and run a Full Benchmark.' + ) + } + // Build the v2 payload (raw channels; server recomputes the score). Throws a // user-facing error if the row predates v2 and lacks the raw channels. const submission = this._buildV2Submission(result, anonymous ?? false)