277 lines
12 KiB
TypeScript
277 lines
12 KiB
TypeScript
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
|
import type { RetrievalAggregate } from '../../app/utils/eval/retrieval_metrics.js'
|
|
|
|
/**
|
|
* Score NOMAD's retrieval against the frozen golden set.
|
|
*
|
|
* No chat model is involved, so this is deterministic and hardware-independent:
|
|
* a movement in these numbers is a code change, not a slow machine or an unlucky
|
|
* sample. It is the fast inner loop for anything touching chunking, embedding,
|
|
* thresholds, or reranking.
|
|
*
|
|
* node ace eval:retrieval
|
|
* node ace eval:retrieval --ablate # is the reranker helping?
|
|
* node ace eval:retrieval --threshold=0.5 # sweep the cutoff
|
|
* node ace eval:retrieval --tag=multi-hop # one slice only
|
|
*/
|
|
export default class EvalRetrieval extends BaseCommand {
|
|
static commandName = 'eval:retrieval'
|
|
static description = 'Score RAG retrieval against the golden set (deterministic, no chat model)'
|
|
|
|
@flags.string({ description: 'Chunks to retrieve per query (default: the production value)' })
|
|
declare topK: string
|
|
|
|
@flags.string({ description: 'Minimum similarity score (default: the production value)' })
|
|
declare threshold: string
|
|
|
|
@flags.boolean({ description: 'Also score the raw dense, reranked, and diversified orderings' })
|
|
declare ablate: boolean
|
|
|
|
@flags.string({ description: 'Only run goldens carrying this tag' })
|
|
declare tag: string
|
|
|
|
@flags.boolean({ description: 'Print each failing question and what it retrieved' })
|
|
declare verbose: boolean
|
|
|
|
@flags.boolean({ description: 'Leave application debug logging on (very noisy)' })
|
|
declare debug: boolean
|
|
|
|
@flags.boolean({ description: 'Write a JSON + Markdown report to tests/eval/reports/' })
|
|
declare report: boolean
|
|
|
|
static options: CommandOptions = {
|
|
startApp: true,
|
|
}
|
|
|
|
async run() {
|
|
const { EvalCorpusService } = await import('#services/eval_corpus_service')
|
|
const { EvalRetrievalService } = await import('#services/eval_retrieval_service')
|
|
const { EvalReportService } = await import('#services/eval_report_service')
|
|
const { quietLogging } = await import('../../app/utils/eval/quiet.js')
|
|
|
|
const corpusService = await this.app.container.make(EvalCorpusService)
|
|
const retrievalService = await this.app.container.make(EvalRetrievalService)
|
|
const reportService = await this.app.container.make(EvalReportService)
|
|
const restoreLogging = quietLogging(this.debug)
|
|
|
|
try {
|
|
const chunks = await retrievalService.assertCorpusReady()
|
|
const fingerprint = await corpusService.fingerprint()
|
|
let goldens = await corpusService.loadGoldens()
|
|
|
|
if (this.tag) {
|
|
goldens = goldens.filter((g) => g.tags.includes(this.tag))
|
|
if (goldens.length === 0) {
|
|
this.logger.error(`No goldens carry the tag "${this.tag}"`)
|
|
this.exitCode = 1
|
|
return
|
|
}
|
|
}
|
|
|
|
this.logger.info(`Corpus ${fingerprint} · ${chunks} chunks · ${goldens.length} goldens`)
|
|
this.logger.info('')
|
|
|
|
const started = Date.now()
|
|
const result = await retrievalService.run(goldens, {
|
|
topK: this.topK ? Number.parseInt(this.topK, 10) : undefined,
|
|
scoreThreshold: this.threshold ? Number.parseFloat(this.threshold) : undefined,
|
|
ablate: this.ablate,
|
|
})
|
|
const elapsed = ((Date.now() - started) / 1000).toFixed(1)
|
|
|
|
this.logger.info(
|
|
`Params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} (${elapsed}s)`
|
|
)
|
|
this.logger.info('')
|
|
this.printAggregate('OVERALL', result.overall, result.params.kValues)
|
|
|
|
if (result.unresolvedChunks > 0) {
|
|
// Every retrieved chunk should belong to the eval corpus. Anything else
|
|
// means the collection filter leaked and the numbers above describe a
|
|
// corpus nobody chose.
|
|
this.logger.error(
|
|
`${result.unresolvedChunks} retrieved chunk(s) did not belong to the eval corpus — the collection filter leaked.`
|
|
)
|
|
this.exitCode = 1
|
|
}
|
|
|
|
this.printThresholdGuidance(result.overall)
|
|
|
|
if (result.ablation) {
|
|
this.logger.info('')
|
|
this.logger.info('=== Stage ablation (does each heuristic earn its place?) ===')
|
|
const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
|
|
const row = (name: string, agg: RetrievalAggregate) =>
|
|
this.logger.info(
|
|
` ${name.padEnd(14)} recall@${k}=${fmt(agg.recall[k])} ndcg@${k}=${fmt(agg.ndcg[k])} mrr=${fmt(agg.mrr)} prec@${k}=${fmt(agg.precision[k])}`
|
|
)
|
|
row('dense only', result.ablation.dense)
|
|
row('+ rerank', result.ablation.reranked)
|
|
row('+ diversity', result.ablation.diversified)
|
|
this.explainAblation(result.ablation, k)
|
|
}
|
|
|
|
this.logger.info('')
|
|
this.logger.info('=== By tag ===')
|
|
for (const [tag, agg] of Object.entries(result.byTag).sort()) {
|
|
const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
|
|
this.logger.info(
|
|
` ${tag.padEnd(20)} n=${String(agg.cases).padStart(3)} recall@${k}=${fmt(agg.recall[k])} ndcg@${k}=${fmt(agg.ndcg[k])}`
|
|
)
|
|
}
|
|
|
|
const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
|
|
const misses = result.cases.filter((c) => !c.expectRefusal && (c.recall[k] ?? 1) < 1)
|
|
this.logger.info('')
|
|
this.logger.info(`${misses.length} of ${result.overall.answerable} answerable questions missed at k=${k}`)
|
|
|
|
if (this.verbose && misses.length > 0) {
|
|
this.logger.info('')
|
|
this.logger.info('=== Misses ===')
|
|
for (const miss of misses) {
|
|
this.logger.info(` ${miss.id}`)
|
|
this.logger.info(` wanted: ${miss.relevantDocIds.join(', ') || '(none)'}`)
|
|
this.logger.info(` retrieved: ${miss.retrievedDocIds.join(', ') || '(nothing)'}`)
|
|
}
|
|
} else if (misses.length > 0) {
|
|
this.logger.info('Re-run with --verbose to see which questions and what they retrieved.')
|
|
}
|
|
|
|
if (this.report) {
|
|
const meta = await reportService.buildMeta('retrieval', fingerprint, {
|
|
...result.params,
|
|
tag: this.tag ?? null,
|
|
})
|
|
const doc = reportService.fromRetrieval(meta, result)
|
|
const slug = `retrieval-${meta.createdAt.replace(/[:.]/g, '-')}`
|
|
const path = await reportService.write(doc, slug, renderRetrievalMarkdown(doc, result, misses))
|
|
this.logger.info('')
|
|
this.logger.success(`Report written: ${path}`)
|
|
}
|
|
} catch (error) {
|
|
this.logger.error(error instanceof Error ? error.message : String(error))
|
|
this.exitCode = 1
|
|
} finally {
|
|
restoreLogging()
|
|
}
|
|
}
|
|
|
|
private printAggregate(label: string, agg: RetrievalAggregate, kValues: number[]) {
|
|
this.logger.info(`=== ${label} (${agg.answerable} answerable of ${agg.cases}) ===`)
|
|
const row = (name: string, values: Record<number, number | null>) =>
|
|
this.logger.info(
|
|
` ${name.padEnd(10)} ${kValues.map((k) => `@${k}=${fmt(values[k])}`).join(' ')}`
|
|
)
|
|
row('recall', agg.recall)
|
|
row('hit rate', agg.hitRate)
|
|
row('precision', agg.precision)
|
|
row('ndcg', agg.ndcg)
|
|
this.logger.info(` mrr ${fmt(agg.mrr)}`)
|
|
}
|
|
|
|
/**
|
|
* Turn the score distributions into an actual recommendation. The raw
|
|
* percentiles are the evidence; this is the reading of them, which is what
|
|
* the threshold constants have never had.
|
|
*/
|
|
private printThresholdGuidance(agg: RetrievalAggregate) {
|
|
const rel = agg.relevantScores
|
|
const irr = agg.irrelevantScores
|
|
this.logger.info('')
|
|
this.logger.info('=== Score separation (how to calibrate the threshold) ===')
|
|
if (rel) {
|
|
this.logger.info(
|
|
` relevant chunks n=${rel.count} min=${rel.min.toFixed(3)} p10=${rel.p10.toFixed(3)} median=${rel.median.toFixed(3)} p90=${rel.p90.toFixed(3)}`
|
|
)
|
|
}
|
|
if (irr) {
|
|
this.logger.info(
|
|
` irrelevant chunks n=${irr.count} min=${irr.min.toFixed(3)} p10=${irr.p10.toFixed(3)} median=${irr.median.toFixed(3)} p90=${irr.p90.toFixed(3)}`
|
|
)
|
|
}
|
|
if (rel && irr) {
|
|
if (rel.p10 > irr.p90) {
|
|
this.logger.success(
|
|
` Clean separation: a threshold between ${irr.p90.toFixed(3)} and ${rel.p10.toFixed(3)} splits them.`
|
|
)
|
|
} else {
|
|
this.logger.warning(
|
|
` Overlapping: relevant p10 (${rel.p10.toFixed(3)}) sits below irrelevant p90 (${irr.p90.toFixed(3)}).`
|
|
)
|
|
this.logger.warning(
|
|
' No cutoff separates these cleanly — the retriever needs work, not the threshold.'
|
|
)
|
|
}
|
|
}
|
|
if (agg.emptyRateOnAnswerable !== null && agg.emptyRateOnAnswerable > 0) {
|
|
this.logger.warning(
|
|
` ${pct(agg.emptyRateOnAnswerable)} of answerable questions retrieved nothing — threshold may be too high.`
|
|
)
|
|
}
|
|
if (agg.nonEmptyRateOnRefusal !== null && agg.nonEmptyRateOnRefusal > 0) {
|
|
this.logger.warning(
|
|
` ${pct(agg.nonEmptyRateOnRefusal)} of out-of-corpus questions retrieved something anyway — that context is what produces confident wrong answers.`
|
|
)
|
|
}
|
|
}
|
|
|
|
private explainAblation(ablation: { dense: RetrievalAggregate; reranked: RetrievalAggregate; diversified: RetrievalAggregate }, k: number) {
|
|
const verdict = (name: string, before: number | null, after: number | null) => {
|
|
if (before === null || after === null) return
|
|
const delta = after - before
|
|
if (Math.abs(delta) < 1e-6) {
|
|
this.logger.warning(` ${name} changed nothing at k=${k} — it is complexity with no measured benefit.`)
|
|
} else if (delta < 0) {
|
|
this.logger.warning(` ${name} made ndcg@${k} worse by ${Math.abs(delta).toFixed(4)}.`)
|
|
} else {
|
|
this.logger.success(` ${name} improved ndcg@${k} by ${delta.toFixed(4)}.`)
|
|
}
|
|
}
|
|
verdict('Reranking', ablation.dense.ndcg[k], ablation.reranked.ndcg[k])
|
|
verdict('Source diversity', ablation.reranked.ndcg[k], ablation.diversified.ndcg[k])
|
|
}
|
|
}
|
|
|
|
const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3))
|
|
const pct = (v: number) => `${(v * 100).toFixed(0)}%`
|
|
|
|
/**
|
|
* The human-readable half of a report. Leads with the misses, because when a
|
|
* number moves the next question is always "which questions?" and a table of
|
|
* aggregates cannot answer it.
|
|
*/
|
|
function renderRetrievalMarkdown(doc: any, result: any, misses: any[]): string {
|
|
const lines: string[] = ['# Retrieval eval', '']
|
|
lines.push(`- corpus: \`${doc.meta.corpusFingerprint}\``)
|
|
lines.push(`- commit: \`${doc.meta.gitSha ?? 'unknown'}\`${doc.meta.gitDirty ? ' (dirty tree)' : ''}`)
|
|
lines.push(`- when: ${doc.meta.createdAt}`)
|
|
lines.push(`- params: topK=${result.params.topK} threshold=${result.params.scoreThreshold}`)
|
|
lines.push('')
|
|
lines.push('## Metrics', '')
|
|
lines.push('| metric | value |', '|---|---:|')
|
|
for (const [name, value] of Object.entries(doc.metrics)) {
|
|
lines.push(`| ${name} | ${value === null ? 'n/a' : (value as number).toFixed(4)} |`)
|
|
}
|
|
lines.push('')
|
|
lines.push(`## Misses (${misses.length})`, '')
|
|
if (misses.length === 0) {
|
|
lines.push('None.')
|
|
} else {
|
|
for (const m of misses) {
|
|
lines.push(`### \`${m.id}\``)
|
|
lines.push(`- wanted: ${m.relevantDocIds.join(', ') || '_none_'}`)
|
|
lines.push(`- retrieved: ${m.retrievedDocIds.join(', ') || '_nothing_'}`)
|
|
lines.push('')
|
|
}
|
|
}
|
|
lines.push('## By tag', '')
|
|
lines.push('| tag | recall@5 | ndcg@5 |', '|---|---:|---:|')
|
|
for (const [tag, metrics] of Object.entries(doc.byTag).sort()) {
|
|
const m = metrics as Record<string, number | null>
|
|
const cell = (v: number | null | undefined) => (v === null || v === undefined ? 'n/a' : v.toFixed(4))
|
|
lines.push(`| ${tag} | ${cell(m['recall@5'])} | ${cell(m['ndcg@5'])} |`)
|
|
}
|
|
return lines.join('\n') + '\n'
|
|
}
|