fix(benchmark): partial runs are not the NOMAD Score (relabel + renormalize)
A System-Only or AI-Only run is a partial result, not the NOMAD Score (which is the full-benchmark composite). Two problems addressed: 1. Scoring bug: AI-only runs were NOT renormalized. _calculateNomadScore always added the system weights (0.60) to the denominator even for an AI-only run (default-zero system scores), so an excellent AI-only run scored ~39.8 -- scaled against the full NOMAD 100 where AI is only 40% -- while system-only already renormalized correctly. Fix: pass systemScores only when the system benchmarks actually ran, so AI-only renormalizes to its own 0-100 (39.8 -> ~99.7). Full and System-only scores are unchanged. 2. Presentation: partial runs were shown with the full "NOMAD Score" label + big gauge, outweighing the small "Partial" notice. Now partial runs are relabelled "System Score" / "AI Score" with a PARTIAL badge, a muted (neutral) gauge + number, and a "run a Full Benchmark for your NOMAD Score" CTA -- applied to both the persistent score section and the Phase 3 ScoreReveal via a shared getScoreDisplay() helper. Adds a `muted` prop to CircularGauge. Browser-validated on NOMAD3: AI-only now shows "AI Score" + PARTIAL, muted gauge, 99.7 (was 39.8); Full still shows "NOMAD Score" green. Implements the display-layer fix from the Score v2 red-team's W2; the AI-ceiling saturation (W1) remains v2 work. Part of #1082. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood
parent
92b161d19d
commit
74c63174fd
|
|
@ -429,9 +429,16 @@ export class BenchmarkService {
|
|||
}
|
||||
}
|
||||
|
||||
// Calculate NOMAD score
|
||||
// Calculate NOMAD score. Only pass systemScores when the system benchmarks
|
||||
// actually ran, so an AI-only run renormalizes to just the AI weights
|
||||
// (an AI-only pass with default-zero system scores would otherwise be
|
||||
// scaled against the full NOMAD 100).
|
||||
this._updateStatus('calculating_score', 'Calculating NOMAD score...')
|
||||
const nomadScore = this._calculateNomadScore(systemScores, aiScores)
|
||||
const systemMeasured = type === 'full' || type === 'system'
|
||||
const nomadScore = this._calculateNomadScore(
|
||||
systemMeasured ? systemScores : null,
|
||||
aiScores
|
||||
)
|
||||
|
||||
// Save result
|
||||
const result = await BenchmarkResult.create({
|
||||
|
|
@ -749,23 +756,34 @@ export class BenchmarkService {
|
|||
/**
|
||||
* Calculate weighted NOMAD score
|
||||
*/
|
||||
private _calculateNomadScore(systemScores: SystemScores, aiScores: Partial<AIScores>): number {
|
||||
private _calculateNomadScore(
|
||||
systemScores: SystemScores | null,
|
||||
aiScores: Partial<AIScores>
|
||||
): number {
|
||||
let totalWeight = 0
|
||||
let weightedSum = 0
|
||||
|
||||
// CPU score
|
||||
weightedSum += systemScores.cpu_score * SCORE_WEIGHTS.cpu
|
||||
totalWeight += SCORE_WEIGHTS.cpu
|
||||
// System scores (only when the system benchmarks actually ran). Passing null
|
||||
// for an AI-only run keeps the system weights OUT of the denominator so the
|
||||
// score renormalizes to just the AI portion's 0-100 range, rather than being
|
||||
// scaled against the full NOMAD 100 (where an excellent AI setup would cap
|
||||
// at ~40). System-only already renormalizes correctly because aiScores is
|
||||
// empty and its weights are likewise skipped below.
|
||||
if (systemScores) {
|
||||
// CPU score
|
||||
weightedSum += systemScores.cpu_score * SCORE_WEIGHTS.cpu
|
||||
totalWeight += SCORE_WEIGHTS.cpu
|
||||
|
||||
// Memory score
|
||||
weightedSum += systemScores.memory_score * SCORE_WEIGHTS.memory
|
||||
totalWeight += SCORE_WEIGHTS.memory
|
||||
// Memory score
|
||||
weightedSum += systemScores.memory_score * SCORE_WEIGHTS.memory
|
||||
totalWeight += SCORE_WEIGHTS.memory
|
||||
|
||||
// Disk scores
|
||||
weightedSum += systemScores.disk_read_score * SCORE_WEIGHTS.disk_read
|
||||
totalWeight += SCORE_WEIGHTS.disk_read
|
||||
weightedSum += systemScores.disk_write_score * SCORE_WEIGHTS.disk_write
|
||||
totalWeight += SCORE_WEIGHTS.disk_write
|
||||
// Disk scores
|
||||
weightedSum += systemScores.disk_read_score * SCORE_WEIGHTS.disk_read
|
||||
totalWeight += SCORE_WEIGHTS.disk_read
|
||||
weightedSum += systemScores.disk_write_score * SCORE_WEIGHTS.disk_write
|
||||
totalWeight += SCORE_WEIGHTS.disk_write
|
||||
}
|
||||
|
||||
// AI scores (if available)
|
||||
if (aiScores.ai_tokens_per_second !== undefined && aiScores.ai_tokens_per_second !== null) {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from 'react'
|
|||
import { IconChartBar, IconCpu, IconDatabase, IconRobot, IconServer } from '@tabler/icons-react'
|
||||
import CircularGauge from '~/components/systeminfo/CircularGauge'
|
||||
import StyledButton from '~/components/StyledButton'
|
||||
import { getScoreDisplay } from '~/lib/benchmarkScore'
|
||||
import BenchmarkResult from '#models/benchmark_result'
|
||||
|
||||
interface ScoreRevealProps {
|
||||
|
|
@ -52,6 +53,8 @@ export default function ScoreReveal({
|
|||
onDone,
|
||||
}: ScoreRevealProps) {
|
||||
const displayScore = useCountUp(result.nomad_score)
|
||||
// A partial (System/AI Only) run is not the NOMAD Score -- relabel + flag it.
|
||||
const scoreInfo = getScoreDisplay(result.benchmark_type)
|
||||
|
||||
// Sub-score gauges, mirroring the System Performance / AI Performance grids.
|
||||
const gauges: {
|
||||
|
|
@ -104,6 +107,11 @@ export default function ScoreReveal({
|
|||
<div className="bg-desert-olive px-6 py-2 flex items-center gap-2">
|
||||
<div className="w-1 h-4 bg-desert-green" />
|
||||
<span className="text-xs font-semibold text-white uppercase tracking-wide">Report</span>
|
||||
{scoreInfo.isPartial && (
|
||||
<span className="ml-auto px-2 py-0.5 rounded-full bg-desert-white/20 text-white text-xs font-semibold uppercase tracking-wide">
|
||||
Partial
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="p-8">
|
||||
|
|
@ -111,21 +119,26 @@ export default function ScoreReveal({
|
|||
<div className="shrink-0">
|
||||
<CircularGauge
|
||||
value={Math.min(100, (result.nomad_score / scoreScale.max) * 100)}
|
||||
label="NOMAD Score"
|
||||
label={scoreInfo.label}
|
||||
size="lg"
|
||||
variant="cpu"
|
||||
subtext={scoreScale.caption}
|
||||
muted={scoreInfo.isPartial}
|
||||
icon={<IconChartBar className="w-8 h-8" />}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex-1 space-y-4">
|
||||
<div
|
||||
className={`text-5xl font-bold font-mono tabular-nums ${getScoreColor(result.nomad_score)}`}
|
||||
className={`text-5xl font-bold font-mono tabular-nums ${
|
||||
scoreInfo.isPartial ? 'text-desert-stone-dark' : getScoreColor(result.nomad_score)
|
||||
}`}
|
||||
>
|
||||
{displayScore.toFixed(1)}
|
||||
</div>
|
||||
<p className="text-desert-stone-dark">
|
||||
Your NOMAD Score is a weighted composite of all benchmark results.
|
||||
{scoreInfo.isPartial
|
||||
? scoreInfo.cta
|
||||
: 'Your NOMAD Score is a weighted composite of all benchmark results.'}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ interface CircularGaugeProps {
|
|||
variant?: 'cpu' | 'memory' | 'disk' | 'default'
|
||||
subtext?: string
|
||||
animated?: boolean
|
||||
/** Render the ring in a neutral tone (e.g. for a partial, non-NOMAD score). */
|
||||
muted?: boolean
|
||||
}
|
||||
|
||||
export default function CircularGauge({
|
||||
|
|
@ -19,6 +21,7 @@ export default function CircularGauge({
|
|||
variant = 'default',
|
||||
subtext,
|
||||
animated = true,
|
||||
muted = false,
|
||||
}: CircularGaugeProps) {
|
||||
const [animatedValue, setAnimatedValue] = useState(animated ? 0 : value)
|
||||
|
||||
|
|
@ -61,6 +64,8 @@ export default function CircularGauge({
|
|||
const offset = circumference - (displayValue / 100) * circumference
|
||||
|
||||
const getColor = () => {
|
||||
// Neutral tone signals this isn't a real NOMAD Score (partial run).
|
||||
if (muted) return 'desert-stone'
|
||||
// For benchmarks: higher scores = better = green
|
||||
if (value >= 75) return 'desert-green'
|
||||
if (value >= 50) return 'desert-olive'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
import type { BenchmarkType } from '../../types/benchmark'
|
||||
|
||||
/**
|
||||
* How a benchmark result's headline score should be presented. A partial run
|
||||
* (System Only / AI Only) is NOT the NOMAD Score -- the NOMAD Score is the full
|
||||
* benchmark composite -- so partial results are relabelled and flagged to avoid
|
||||
* users mistaking a partial number for their NOMAD Score. The stored score for
|
||||
* a partial run is already renormalized to that category's own 0-100 range
|
||||
* (see BenchmarkService._calculateNomadScore).
|
||||
*/
|
||||
export function getScoreDisplay(type: BenchmarkType): {
|
||||
label: string
|
||||
isPartial: boolean
|
||||
/** Which full benchmark to run to get the real NOMAD Score, for the CTA copy. */
|
||||
cta: string
|
||||
} {
|
||||
switch (type) {
|
||||
case 'system':
|
||||
return {
|
||||
label: 'System Score',
|
||||
isPartial: true,
|
||||
cta: 'This is a partial result, not your NOMAD Score. Run a Full Benchmark to get your NOMAD Score.',
|
||||
}
|
||||
case 'ai':
|
||||
return {
|
||||
label: 'AI Score',
|
||||
isPartial: true,
|
||||
cta: 'This is a partial result, not your NOMAD Score. Run a Full Benchmark to get your NOMAD Score.',
|
||||
}
|
||||
default:
|
||||
return { label: 'NOMAD Score', isPartial: false, cta: '' }
|
||||
}
|
||||
}
|
||||
|
|
@ -25,6 +25,7 @@ import { SERVICE_NAMES } from '../../../constants/service_names'
|
|||
import { useBenchmarkRun } from '~/hooks/useBenchmarkRun'
|
||||
import BenchmarkRunView from '~/components/benchmark/BenchmarkRunView'
|
||||
import ScoreReveal from '~/components/benchmark/ScoreReveal'
|
||||
import { getScoreDisplay } from '~/lib/benchmarkScore'
|
||||
|
||||
export default function BenchmarkPage(props: {
|
||||
benchmark: {
|
||||
|
|
@ -175,6 +176,10 @@ export default function BenchmarkPage(props: {
|
|||
latestResult.ai_tokens_per_second > 0 &&
|
||||
!latestResult.submitted_to_repository
|
||||
|
||||
// How to present the headline score: partial (System/AI Only) runs are NOT the
|
||||
// NOMAD Score and are relabelled + flagged so users don't mistake them for it.
|
||||
const scoreInfo = latestResult ? getScoreDisplay(latestResult.benchmark_type) : null
|
||||
|
||||
// Handle Full Benchmark click with pre-flight check
|
||||
const handleFullBenchmarkClick = () => {
|
||||
if (!aiInstalled) {
|
||||
|
|
@ -330,7 +335,12 @@ export default function BenchmarkPage(props: {
|
|||
<section className="mb-12">
|
||||
<h2 className="text-2xl font-bold text-desert-green mb-6 flex items-center gap-2">
|
||||
<div className="w-1 h-6 bg-desert-green" />
|
||||
NOMAD Score
|
||||
{scoreInfo?.label ?? 'NOMAD Score'}
|
||||
{scoreInfo?.isPartial && (
|
||||
<span className="ml-1 px-2 py-0.5 rounded-full bg-desert-stone-light text-desert-stone-dark text-xs font-semibold uppercase tracking-wide">
|
||||
Partial
|
||||
</span>
|
||||
)}
|
||||
</h2>
|
||||
|
||||
<div className="bg-desert-white rounded-lg p-8 border border-desert-stone-light shadow-sm">
|
||||
|
|
@ -338,21 +348,35 @@ export default function BenchmarkPage(props: {
|
|||
<div className="shrink-0">
|
||||
<CircularGauge
|
||||
value={latestResult.nomad_score}
|
||||
label="NOMAD Score"
|
||||
label={scoreInfo?.label ?? 'NOMAD Score'}
|
||||
size="lg"
|
||||
variant="cpu"
|
||||
subtext="out of 100"
|
||||
muted={scoreInfo?.isPartial}
|
||||
icon={<IconChartBar className="w-8 h-8" />}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex-1 space-y-4">
|
||||
<div
|
||||
className={`text-5xl font-bold ${getScoreColor(latestResult.nomad_score)}`}
|
||||
>
|
||||
{latestResult.nomad_score.toFixed(1)}
|
||||
<div className="flex items-center gap-3">
|
||||
<div
|
||||
className={`text-5xl font-bold ${
|
||||
scoreInfo?.isPartial
|
||||
? 'text-desert-stone-dark'
|
||||
: getScoreColor(latestResult.nomad_score)
|
||||
}`}
|
||||
>
|
||||
{latestResult.nomad_score.toFixed(1)}
|
||||
</div>
|
||||
{scoreInfo?.isPartial && (
|
||||
<span className="px-2 py-1 rounded-md bg-desert-stone-light text-desert-stone-dark text-xs font-semibold uppercase tracking-wide">
|
||||
Partial
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-desert-stone-dark">
|
||||
Your NOMAD Score is a weighted composite of all benchmark results.
|
||||
{scoreInfo?.isPartial
|
||||
? scoreInfo.cta
|
||||
: 'Your NOMAD Score is a weighted composite of all benchmark results.'}
|
||||
</p>
|
||||
|
||||
{/* Share with Community - Only for full benchmarks with AI data */}
|
||||
|
|
|
|||
Loading…
Reference in New Issue