From dbc49e92d43f9a6d6a8e0f80a66befd1f85ed30a Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Mon, 13 Jul 2026 11:20:10 -0700 Subject: [PATCH] feat(benchmark): dashboard re-run banner prompting a Score v2 re-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shows a dismissible dashboard banner to users who submitted a benchmark under v1 scoring but don't yet have a Score v2 result, nudging them to re-run so the community leaderboard gets their v2 number. - Show logic (benchmark_service.shouldShowRerunBanner): has a result with submitted_to_repository=true AND no result with nomad_score_v2 AND the dismiss KV isn't set. Self-clears two ways — dismiss sets the KV, and any v2 run gives a result a nomad_score_v2 so the condition flips off on its own. - New GET /api/benchmark/rerun-banner + useBenchmarkRerunBanner hook (mirrors useUpdateAvailable), rendered as a dismissible Alert on the dashboard with a "Re-run benchmark" CTA to /settings/benchmark. - New KV key benchmark.rerunBannerDismissed (boolean) in both KV_STORE_SCHEMA and the SETTINGS_KEYS whitelist (the PATCH /system/settings validator enum); dismiss writes it via the existing updateSetting endpoint + invalidates the query. Sibling to the Score v2 app client (#1094); stacked on it for the nomad_score_v2 column. Browser-verified on the NOMAD3 dev env: show/dismiss/ reload-persist/self-clear-on-v2 all correct. Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/app/controllers/benchmark_controller.ts | 7 ++++ admin/app/services/benchmark_service.ts | 19 +++++++++++ admin/constants/kv_store.ts | 2 +- .../inertia/hooks/useBenchmarkRerunBanner.ts | 15 ++++++++ admin/inertia/lib/api.ts | 7 ++++ admin/inertia/pages/home.tsx | 34 +++++++++++++++++++ admin/start/routes.ts | 1 + admin/types/kv_store.ts | 1 + 8 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 admin/inertia/hooks/useBenchmarkRerunBanner.ts diff --git a/admin/app/controllers/benchmark_controller.ts b/admin/app/controllers/benchmark_controller.ts index da483c0..144bb54 100644 --- a/admin/app/controllers/benchmark_controller.ts +++ b/admin/app/controllers/benchmark_controller.ts @@ -248,6 +248,13 @@ export default class BenchmarkController { return this.benchmarkService.getStatus() } + /** + * Whether to show the dashboard "re-run under Score v2" banner + */ + async rerunBanner({}: HttpContext) { + return { show: await this.benchmarkService.shouldShowRerunBanner() } + } + /** * Get benchmark settings */ diff --git a/admin/app/services/benchmark_service.ts b/admin/app/services/benchmark_service.ts index 770c902..a37c066 100644 --- a/admin/app/services/benchmark_service.ts +++ b/admin/app/services/benchmark_service.ts @@ -192,6 +192,25 @@ export class BenchmarkService { return await BenchmarkResult.findBy('benchmark_id', benchmarkId) } + /** + * Whether to show the dashboard "re-run under Score v2" banner. Shown to users + * who have a v1 leaderboard submission but no Score v2 result yet, and who + * haven't dismissed it. Self-clears two ways: dismiss sets the KV flag, and any + * result carrying a nomad_score_v2 (i.e. a v2 run happened) flips (b) false. + */ + async shouldShowRerunBanner(): Promise { + const dismissed = await KVStore.getValue('benchmark.rerunBannerDismissed') + if (dismissed === true) return false + + const hasV2Run = await BenchmarkResult.query().whereNotNull('nomad_score_v2').first() + if (hasV2Run) return false + + const hasSubmittedV1 = await BenchmarkResult.query() + .where('submitted_to_repository', true) + .first() + return hasSubmittedV1 !== null + } + /** * Submit benchmark results to central repository */ diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 35b9617..a55e2d9 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -1,3 +1,3 @@ import { KVStoreKey } from "../types/kv_store.js"; -export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow']; \ No newline at end of file +export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow', 'benchmark.rerunBannerDismissed']; \ No newline at end of file diff --git a/admin/inertia/hooks/useBenchmarkRerunBanner.ts b/admin/inertia/hooks/useBenchmarkRerunBanner.ts new file mode 100644 index 0000000..cf83772 --- /dev/null +++ b/admin/inertia/hooks/useBenchmarkRerunBanner.ts @@ -0,0 +1,15 @@ +import api from "~/lib/api" +import { useQuery } from "@tanstack/react-query" + +export const BENCHMARK_RERUN_BANNER_QUERY_KEY = ['benchmark-rerun-banner'] + +export const useBenchmarkRerunBanner = () => { + const queryData = useQuery<{ show: boolean } | undefined>({ + queryKey: BENCHMARK_RERUN_BANNER_QUERY_KEY, + queryFn: () => api.checkBenchmarkRerunBanner(), + refetchInterval: Infinity, // Disable automatic refetching + refetchOnWindowFocus: false, + }) + + return queryData.data +} diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 9782e9e..bdd5bdb 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -374,6 +374,13 @@ class API { })() } + async checkBenchmarkRerunBanner() { + return catchInternal(async () => { + const response = await this.client.get<{ show: boolean }>('/benchmark/rerun-banner') + return response.data + })() + } + async getChatSessions() { return catchInternal(async () => { const response = await this.client.get< diff --git a/admin/inertia/pages/home.tsx b/admin/inertia/pages/home.tsx index a782327..5f5d487 100644 --- a/admin/inertia/pages/home.tsx +++ b/admin/inertia/pages/home.tsx @@ -13,6 +13,12 @@ import { ServiceSlim } from '../../types/services' import DynamicIcon, { DynamicIconName } from '~/components/DynamicIcon' import { useUpdateAvailable } from '~/hooks/useUpdateAvailable' import { useSystemSetting } from '~/hooks/useSystemSetting' +import { + useBenchmarkRerunBanner, + BENCHMARK_RERUN_BANNER_QUERY_KEY, +} from '~/hooks/useBenchmarkRerunBanner' +import { useQueryClient } from '@tanstack/react-query' +import api from '~/lib/api' import Alert from '~/components/Alert' import { SERVICE_NAMES } from '../../constants/service_names' @@ -91,8 +97,15 @@ export default function Home(props: { }) { const items: DashboardItem[] = [] const updateInfo = useUpdateAvailable(); + const rerunBanner = useBenchmarkRerunBanner() + const queryClient = useQueryClient() const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props + const handleDismissRerunBanner = async () => { + await api.updateSetting('benchmark.rerunBannerDismissed', true) + queryClient.invalidateQueries({ queryKey: BENCHMARK_RERUN_BANNER_QUERY_KEY }) + } + // Check if user has visited Easy Setup const { data: easySetupVisited } = useSystemSetting({ key: 'ui.hasVisitedEasySetup' @@ -155,6 +168,27 @@ export default function Home(props: { ) } + { + rerunBanner?.show && ( +
+ router.visit('/settings/benchmark'), + }} + /> +
+ ) + }
{items.map((item) => { const isEasySetup = item.label === 'Easy Setup' diff --git a/admin/start/routes.ts b/admin/start/routes.ts index ca068cc..85dfb31 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -237,6 +237,7 @@ router router.post('/builder-tag', [BenchmarkController, 'updateBuilderTag']) router.get('/comparison', [BenchmarkController, 'comparison']) router.get('/status', [BenchmarkController, 'status']) + router.get('/rerun-banner', [BenchmarkController, 'rerunBanner']) router.get('/settings', [BenchmarkController, 'settings']) router.post('/settings', [BenchmarkController, 'updateSettings']) }) diff --git a/admin/types/kv_store.ts b/admin/types/kv_store.ts index 1156c3e..ee35ae6 100644 --- a/admin/types/kv_store.ts +++ b/admin/types/kv_store.ts @@ -42,6 +42,7 @@ export const KV_STORE_SCHEMA = { 'ai.amdHsaOverride': 'string', 'ai.autoFixGpuPassthrough': 'boolean', 'gpu.autoRemediatedAt': 'string', + 'benchmark.rerunBannerDismissed': 'boolean', } as const type KVTagToType = T extends 'boolean' ? boolean : string