RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(eval): standardized RAG quality evaluation harness (#1233)

This commit is contained in:
Jake Turner 2026-08-13 11:10:47 -07:00 committed by GitHub
parent 0bd1c6f4f9
commit aff56ad4a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
63 changed files with 10665 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CONTRIBUTING.md
View File

@ -95,7 +95,21 @@ Because NOMAD relies heavily on Docker, we actually recommend against installing
3. **Make your changes.** Follow existing code style and conventions. Test your changes locally against a running NOMAD instance before submitting.
4. **Add release notes** (see [Release Notes](#release-notes) below).
4. **If you touched the AI Assistant or RAG, measure it.** Anything affecting
chunking, embedding, retrieval, reranking, thresholds, the system prompts, or
context assembly should be backed by numbers rather than a spot check —
"it seemed better in the chat window" is how a regression ships. From `admin/`:
```bash
node ace eval:corpus --ingest # once; safe, never touches your real knowledge base
node ace eval:retrieval --ablate # seconds, deterministic, no chat model
node ace eval:generation --model=<your model> --all-modes
```
Include the before/after numbers in your pull request. See
[`admin/tests/eval/README.md`](admin/tests/eval/README.md) for what the
metrics mean, how the three generation modes separate a code bug from a model
that is simply too small, and the harness's known limitations.
5. **Commit your changes** using [Conventional Commits](#commit-messages).

196
admin/app/controllers/ollama_controller.ts
View File

@ -1,7 +1,7 @@
import { ChatService } from '#services/chat_service'
import { DockerService } from '#services/docker_service'
import { NomadMdService } from '#services/nomad_md_service'
import { OllamaService } from '#services/ollama_service'
import { RagPipelineService } from '#services/rag_pipeline_service'
import { RagService } from '#services/rag_service'
import Service from '#models/service'
import KVStore from '#models/kv_store'
@ -10,10 +10,8 @@ import { chatSchema, getAvailableModelsSchema, unloadChatModelsSchema } from '#v
import { assertNotCloudMetadataUrl } from '#validators/common'
import { inject } from '@adonisjs/core'
import type { HttpContext } from '@adonisjs/core/http'
import { RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import logger from '@adonisjs/core/services/logger'
type Message = { role: 'system' | 'user' | 'assistant'; content: string }
@inject()
export default class OllamaController {
@ -21,8 +19,8 @@ export default class OllamaController {
private chatService: ChatService,
private dockerService: DockerService,
private ollamaService: OllamaService,
private ragService: RagService,
private nomadMdService: NomadMdService
private ragPipelineService: RagPipelineService,
private ragService: RagService
) { }
async availableModels({ request }: HttpContext) {
@ -62,101 +60,15 @@ export default class OllamaController {
}
try {
// If there are no system messages in the chat inject system prompts
const hasSystemMessage = reqData.messages.some((msg) => msg.role === 'system')
if (!hasSystemMessage) {
const systemPrompt = {
role: 'system' as const,
content: SYSTEM_PROMPTS.default,
}
logger.debug('[OllamaController] Injecting system prompt')
reqData.messages.unshift(systemPrompt)
}
// Inject the user-managed NOMAD.md as its own leading system message so the
// user's persistent instructions take precedence, while the default
// formatting prompt and any RAG context below remain intact. A missing or
// blank file yields null and changes nothing.
const nomadPrompt = await this.nomadMdService.getSystemPrompt()
if (nomadPrompt) {
logger.debug('[OllamaController] Injecting NOMAD.md system prompt')
reqData.messages.unshift({ role: 'system' as const, content: nomadPrompt })
}
// Query rewriting for better RAG retrieval with manageable context
// Will return user's latest message if no rewriting is needed
const rewrittenQuery = await this.rewriteQueryWithContext(reqData.messages, reqData.model)
logger.debug(`[OllamaController] Rewritten query for RAG: "${rewrittenQuery}"`)
if (rewrittenQuery) {
const collectionFilter: string | null = request.input('collection', null)
const relevantDocs = await this.ragService.searchSimilarDocuments(
rewrittenQuery,
5, // Top 5 most relevant chunks
0.3, // Minimum similarity score of 0.3
collectionFilter ?? undefined
)
logger.debug(`[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${rewrittenQuery}"`)
// If relevant context is found, inject as a system message with adaptive limits
if (relevantDocs.length > 0) {
// Determine context budget based on model size
const { maxResults, maxTokens } = this.getContextLimitsForModel(reqData.model)
let trimmedDocs = relevantDocs.slice(0, maxResults)
// Apply token cap if set (estimate ~3.5 chars per token)
// Always include the first (most relevant) result — the cap only gates subsequent results
if (maxTokens > 0) {
const charCap = maxTokens * 3.5
let totalChars = 0
trimmedDocs = trimmedDocs.filter((doc, idx) => {
totalChars += doc.text.length
return idx === 0 || totalChars <= charCap
})
}
logger.debug(
`[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${reqData.model}, maxResults: ${maxResults}, maxTokens: ${maxTokens || 'unlimited'})`
)
// Label each context block with its source title when available (a neutral,
// honest provenance signal) but never the raw relevance score — nomic cosine
// scores for genuinely relevant passages sit ~0.4-0.6, and surfacing e.g.
// "42%" primes the model to distrust correct context. Scores stay in the logs
// above for debugging.
const contextText = trimmedDocs
.map((doc, idx) => {
const title = doc.metadata?.full_title || doc.metadata?.article_title
const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]`
return `${label}\n${doc.text}`
})
.join('\n\n')
const systemMessage = {
role: 'system' as const,
content: SYSTEM_PROMPTS.rag_context(contextText),
}
// Insert system message at the beginning (after any existing system messages)
const firstNonSystemIndex = reqData.messages.findIndex((msg) => msg.role !== 'system')
const insertIndex = firstNonSystemIndex === -1 ? 0 : firstNonSystemIndex
reqData.messages.splice(insertIndex, 0, systemMessage)
}
}
// If system messages are large (e.g. due to RAG context), request a context window big
// enough to fit them. Ollama respects num_ctx per-request; LM Studio ignores it gracefully.
const systemChars = reqData.messages
.filter((m) => m.role === 'system')
.reduce((sum, m) => sum + m.content.length, 0)
const estimatedSystemTokens = Math.ceil(systemChars / 3.5)
let numCtx: number | undefined
if (estimatedSystemTokens > 3000) {
const needed = estimatedSystemTokens + 2048 // leave room for conversation + response
numCtx = [8192, 16384, 32768, 65536].find((n) => n >= needed) ?? 65536
logger.debug(`[OllamaController] Large system prompt (~${estimatedSystemTokens} tokens), requesting num_ctx: ${numCtx}`)
}
// Everything from system-prompt assembly through query rewriting,
// retrieval, context trimming and the num_ctx decision lives in
// RagPipelineService so the eval harness exercises this exact code path.
const collectionFilter: string | null = request.input('collection', null)
const trace = await this.ragPipelineService.buildPrompt(reqData.messages, reqData.model, {
collection: collectionFilter ?? undefined,
})
reqData.messages = trace.messages
const numCtx = trace.numCtx
// Check if the model supports "thinking" capability for enhanced response generation.
// Thinking is only enabled when the model supports it AND the user wants it: the explicit
@ -420,88 +332,4 @@ export default class OllamaController {
return models.map((m, i) => ({ ...m, thinking: thinking[i] }))
}
/**
* Determines RAG context limits based on model size extracted from the model name.
* Parses size indicators like "1b", "3b", "8b", "70b" from model names/tags.
*/
private getContextLimitsForModel(modelName: string): { maxResults: number; maxTokens: number } {
// Extract parameter count from model name (e.g., "llama3.2:3b", "qwen2.5:1.5b", "gemma:7b")
const sizeMatch = modelName.match(/(\d+\.?\d*)[bB]/)
const paramBillions = sizeMatch ? parseFloat(sizeMatch[1]) : 8 // default to 8B if unknown
for (const tier of RAG_CONTEXT_LIMITS) {
if (paramBillions <= tier.maxParams) {
return { maxResults: tier.maxResults, maxTokens: tier.maxTokens }
}
}
// Fallback: no limits
return { maxResults: 5, maxTokens: 0 }
}
private async rewriteQueryWithContext(
messages: Message[],
model: string
): Promise<string | null> {
const lastUserMessage = [...messages].reverse().find(msg => msg.role === 'user')
try {
// Skip the entire RAG pipeline if there are no documents to search
const hasDocuments = await this.ragService.hasDocuments()
if (!hasDocuments) {
return null
}
// Get recent conversation history (last 6 messages for 3 turns)
const recentMessages = messages.slice(-6)
// Skip rewriting on the very first turn — with only one user message
// there is no prior context to fold in, so the rewrite would just echo
// the message back at the cost of an extra LLM round-trip. From the
// first follow-up onward we need the rewrite so the RAG query carries
// entities and topics from earlier turns ("the bars" → "Hershey's bars
// chocolate poisoning dog"); without it, embeddings match nothing and
// the assistant loses the thread.
const userMessages = recentMessages.filter(msg => msg.role === 'user')
if (userMessages.length < 2) {
return lastUserMessage?.content || null
}
const conversationContext = recentMessages
.map(msg => {
const role = msg.role === 'user' ? 'User' : 'Assistant'
// Truncate assistant messages to first 200 chars to keep context manageable
const content = msg.role === 'assistant'
? msg.content.slice(0, 200) + (msg.content.length > 200 ? '...' : '')
: msg.content
return `${role}: "${content}"`
})
.join('\n')
const response = await this.ollamaService.chat({
model,
messages: [
{
role: 'system',
content: SYSTEM_PROMPTS.query_rewrite,
},
{
role: 'user',
content: `Conversation:\n${conversationContext}\n\nRewritten Query:`,
},
],
})
const rewrittenQuery = response.message.content.trim()
logger.info(`[RAG] Query rewritten: "${rewrittenQuery}"`)
return rewrittenQuery
} catch (error) {
logger.error(
`[RAG] Query rewriting failed: ${error instanceof Error ? error.message : error}`
)
// Fallback to last user message if rewriting fails
return lastUserMessage?.content || null
}
}
}

172
admin/app/services/eval_corpus_service.ts Normal file
View File

@ -0,0 +1,172 @@
import { RagService } from '#services/rag_service'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import { createHash } from 'node:crypto'
import { readdir, readFile } from 'node:fs/promises'
import { basename, join, resolve } from 'node:path'
import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js'
import { EMBEDDING_MODEL_NAME } from '../../constants/ollama.js'
import {
assertGoldensMatchCorpus,
computeCorpusFingerprint,
parseGoldens,
type Golden,
} from '../utils/eval/golden_set.js'
import { EVAL_CORPUS_DIR } from '../utils/eval/corpus_source.js'
/** Where the golden question set lives, relative to the app root. */
export const EVAL_GOLDENS_DIR = 'tests/eval/goldens'
// EVAL_CORPUS_DIR and docIdFromSource live in app/utils/eval/corpus_source.ts:
// they are pure path logic, and keeping them out of this service is what lets
// the leak guard be unit-tested without booting AdonisJS.
export { docIdFromSource, EVAL_CORPUS_DIR } from '../utils/eval/corpus_source.js'
export type CorpusDocument = {
/** Filename without extension. This is the id goldens refer to. */
docId: string
/** Absolute path on disk; becomes the Qdrant `source` payload. */
path: string
text: string
}
export type IngestSummary = {
fingerprint: string
documents: number
chunks: number
removedBeforeIngest: number
failures: Array<{ docId: string; reason: string }>
}
/**
* Owns the frozen evaluation corpus: reading it off disk, fingerprinting it,
* and pushing it through NOMAD's real ingest path into the reserved
* `__nomad_eval__` collection tag.
*
* Ingesting through `RagService.embedAndStoreText` rather than writing vectors
* directly is the whole point — chunk size, the token-estimate ratio, the
* search_document prefix, and the embedding model are all in scope of the
* measurement, so a change to any of them shows up as a retrieval score
* movement instead of hiding.
*/
@inject()
export class EvalCorpusService {
constructor(private ragService: RagService) {}
private corpusPath(): string {
return resolve(join(process.cwd(), EVAL_CORPUS_DIR))
}
/** Read every markdown document in the corpus, sorted for determinism. */
async loadCorpus(): Promise<CorpusDocument[]> {
const dir = this.corpusPath()
const entries = (await readdir(dir)).filter((f) => f.endsWith('.md')).sort()
if (entries.length === 0) {
throw new Error(`No corpus documents found in ${dir}`)
}
return Promise.all(
entries.map(async (file) => {
const path = join(dir, file)
return { docId: basename(file, '.md'), path, text: await readFile(path, 'utf8') }
})
)
}
/** Load and validate every golden file, cross-checked against the corpus. */
async loadGoldens(): Promise<Golden[]> {
const dir = resolve(join(process.cwd(), EVAL_GOLDENS_DIR))
const files = (await readdir(dir)).filter((f) => f.endsWith('.jsonl')).sort()
if (files.length === 0) throw new Error(`No golden files found in ${dir}`)
const goldens: Golden[] = []
const seen = new Set<string>()
for (const file of files) {
const parsed = parseGoldens(await readFile(join(dir, file), 'utf8'), file)
for (const g of parsed) {
// parseGoldens dedupes within a file; this catches collisions across files.
if (seen.has(g.id)) throw new Error(`Duplicate golden id "${g.id}" in ${file}`)
seen.add(g.id)
goldens.push(g)
}
}
const corpus = await this.loadCorpus()
assertGoldensMatchCorpus(
goldens,
corpus.map((d) => d.docId)
)
return goldens
}
/**
* Hash the corpus together with the ingest parameters that shaped it.
* Reports carry this; two reports with different fingerprints are not
* comparable and `eval:compare` refuses to pretend otherwise.
*/
async fingerprint(): Promise<string> {
const corpus = await this.loadCorpus()
return computeCorpusFingerprint(
{
documents: new Map(corpus.map((d) => [d.docId, d.text])),
chunkTokens: RagService.TARGET_TOKENS_PER_CHUNK,
chunkOverlapTokens: RagService.CHUNK_OVERLAP_TOKENS,
charToTokenRatio: RagService.CHAR_TO_TOKEN_RATIO,
embeddingModel: EMBEDDING_MODEL_NAME,
embeddingDimension: RagService.EMBEDDING_DIMENSION,
},
(input) => createHash('sha256').update(input).digest('hex')
)
}
/** Remove every eval point. Never touches user content. */
async reset(): Promise<number> {
const removed = await this.ragService.deleteCollectionPoints(KB_EVAL_COLLECTION)
logger.info(`[Eval] Removed ${removed} eval corpus chunks`)
return removed
}
/** How many eval chunks are currently in the vector store. */
async count(): Promise<number> {
return this.ragService.countChunksInCollection(KB_EVAL_COLLECTION)
}
/**
* Wipe and rebuild the eval corpus.
*
* Always a full rebuild: a partial re-ingest would leave the vector store in
* a state no fingerprint describes, and a fingerprint that does not describe
* the store is worse than no fingerprint at all.
*/
async ingest(onProgress?: (docId: string, index: number, total: number) => void): Promise<IngestSummary> {
const corpus = await this.loadCorpus()
const removedBeforeIngest = await this.reset()
let chunks = 0
const failures: IngestSummary['failures'] = []
for (const [index, doc] of corpus.entries()) {
onProgress?.(doc.docId, index + 1, corpus.length)
try {
const result = await this.ragService.embedAndStoreText(doc.text, {
source: doc.path,
collection: KB_EVAL_COLLECTION,
})
if (!result) {
failures.push({ docId: doc.docId, reason: 'embedAndStoreText returned null' })
continue
}
chunks += result.chunks
} catch (error) {
failures.push({ docId: doc.docId, reason: error instanceof Error ? error.message : String(error) })
}
}
return {
fingerprint: await this.fingerprint(),
documents: corpus.length,
chunks,
removedBeforeIngest,
failures,
}
}
}

350
admin/app/services/eval_generation_service.ts Normal file
View File

@ -0,0 +1,350 @@
import { EvalCorpusService } from '#services/eval_corpus_service'
import { OllamaService } from '#services/ollama_service'
import { RagPipelineService } from '#services/rag_pipeline_service'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js'
import type { OllamaChatMessage } from '../../types/ollama.js'
import type { PipelineOptions, RetrievedChunk } from '../../types/rag.js'
import { docIdFromSource } from '../utils/eval/corpus_source.js'
import type { Golden } from '../utils/eval/golden_set.js'
import {
scoreAnswer,
summarizeNumeric,
summarizeRepeats,
type GenerationScores,
type NumericSummary,
type RepeatStats,
} from '../utils/eval/generation_metrics.js'
/**
* The three ways to run a question, and what each one isolates.
*
* This is the part of the harness that answers "code bug or weak model?".
* Running the same question all three ways turns one ambiguous score into a
* decomposition:
*
* - `oracle` gives the model perfect context by construction. A low score here
* is the model (or the prompt) failing to use good context — retrieval is
* provably not at fault.
* - `e2e` is the real product. `oracle - e2e` is the cost of imperfect retrieval.
* - `noretrieval` is the model's parametric baseline. `e2e - noretrieval` is
* what RAG is actually buying, and on the fictional questions it should be
* nearly everything, since no model can know them.
*/
export type GenerationMode = 'oracle' | 'e2e' | 'noretrieval'
/**
* Sentinel model name for the extractive reference run.
*
* `--model=mock` needs no Ollama at all: it answers by echoing whatever context
* the pipeline injected, and refuses when nothing was injected. That makes it
* two useful things at once — a way to exercise scoring and reporting with zero
* models installed (so the harness itself is CI-testable), and a genuine
* *ceiling* line: the best a perfectly extractive model could score given this
* retrieval. A real model below the mock line is the bottleneck; a mock line
* that is itself low means retrieval is.
*/
export const MOCK_MODEL = 'mock'
/** Fixed sampling for eval runs. Never used by production chat. */
export const EVAL_TEMPERATURE = 0
export const EVAL_SEED = 42
export type GenerationRunOptions = {
mode: GenerationMode
model: string
repeats?: number
topK?: number
scoreThreshold?: number
/** Skip the history-aware rewrite even on multi-turn goldens. */
skipQueryRewrite?: boolean
onProgress?: (id: string, index: number, total: number) => void
}
export type GenerationCaseResult = {
id: string
tags: string[]
expectRefusal: boolean
/** One entry per repeat. */
answers: string[]
scores: GenerationScores[]
correctness: RepeatStats
refusalCorrectness: RepeatStats
leakageFree: RepeatStats
groundedness: NumericSummary | null
retrievedDocIds: string[]
injectedChunks: number
/** Non-null only when something went wrong talking to the model. */
error?: string
}
export type GenerationAggregate = {
cases: number
/** Cases whose pass/fail flipped across repeats; excluded from gating. */
unstable: number
errors: number
correctness: number | null
refusalCorrectness: number | null
leakageRate: number | null
thinkTagLeakRate: number | null
markdownRate: number | null
groundedness: NumericSummary | null
meanAnswerLength: number | null
}
export type GenerationRunResult = {
params: {
mode: GenerationMode
model: string
repeats: number
temperature: number
seed: number
topK?: number
scoreThreshold?: number
}
overall: GenerationAggregate
byTag: Record<string, GenerationAggregate>
cases: GenerationCaseResult[]
elapsedMs: number
}
@inject()
export class EvalGenerationService {
constructor(
private ollamaService: OllamaService,
private pipeline: RagPipelineService,
private corpusService: EvalCorpusService
) {}
async run(goldens: Golden[], options: GenerationRunOptions): Promise<GenerationRunResult> {
const repeats = Math.max(1, options.repeats ?? 3)
const started = Date.now()
const isMock = options.model === MOCK_MODEL
// Oracle mode needs the corpus text on hand to synthesize perfect context.
const corpusText = options.mode === 'oracle' ? await this.loadCorpusText() : null
if (!isMock) await this.prepareModel(options.model)
const cases: GenerationCaseResult[] = []
for (const [index, golden] of goldens.entries()) {
options.onProgress?.(golden.id, index + 1, goldens.length)
cases.push(await this.runCase(golden, options, repeats, corpusText, isMock))
}
return {
params: {
mode: options.mode,
model: options.model,
repeats,
temperature: EVAL_TEMPERATURE,
seed: EVAL_SEED,
topK: options.topK,
scoreThreshold: options.scoreThreshold,
},
overall: aggregateGeneration(cases),
byTag: aggregateGenerationByTag(cases),
cases,
elapsedMs: Date.now() - started,
}
}
/**
* Evict other resident models and burn one throwaway generation.
*
* Both borrowed from BenchmarkService, and for the same reasons it added
* them: a cold first run is dramatically slower and behaves differently, and
* leftover models in VRAM change how the one under test is scheduled. Quality
* runs are less timing-sensitive than throughput runs, but a first-token
* timeout or an OOM-driven CPU fallback absolutely does change the answer.
*/
private async prepareModel(model: string): Promise<void> {
try {
await this.ollamaService.unloadAllChatModelsExcept(model)
} catch (error) {
logger.warn(`[Eval] Could not evict resident models: ${errorText(error)}`)
}
try {
await this.ollamaService.chat({
model,
messages: [{ role: 'user', content: 'Reply with the single word: ready' }],
temperature: EVAL_TEMPERATURE,
seed: EVAL_SEED,
})
} catch (error) {
// Best-effort: if warm-up fails the scored runs will surface the real
// problem with a better message than we could produce here.
logger.warn(`[Eval] Warm-up generation failed: ${errorText(error)}`)
}
}
private async loadCorpusText(): Promise<Map<string, { text: string; path: string }>> {
const docs = await this.corpusService.loadCorpus()
return new Map(docs.map((d) => [d.docId, { text: d.text, path: d.path }]))
}
private async runCase(
golden: Golden,
options: GenerationRunOptions,
repeats: number,
corpusText: Map<string, { text: string; path: string }> | null,
isMock: boolean
): Promise<GenerationCaseResult> {
const messages: OllamaChatMessage[] = [
...golden.turns.map((t) => ({ role: t.role, content: t.content })),
{ role: 'user' as const, content: golden.query },
]
const pipelineOptions: PipelineOptions = {
collection: KB_EVAL_COLLECTION,
topK: options.topK,
scoreThreshold: options.scoreThreshold,
// The mock run must not touch Ollama at all — that is what makes it
// usable with no models installed. The rewrite is a chat-model call, so
// it is always skipped there (it would 404 and silently fall back, which
// works but quietly makes the "no model needed" claim untrue).
skipQueryRewrite: options.skipQueryRewrite || isMock,
// A developer's personal NOMAD.md would silently skew every score.
skipNomadMd: true,
}
if (options.mode === 'oracle') {
pipelineOptions.oracleContext = golden.relevantDocIds.map((docId) => {
const doc = corpusText?.get(docId)
return {
text: doc?.text ?? '',
score: 1,
metadata: { source: doc?.path },
} satisfies RetrievedChunk
})
} else if (options.mode === 'noretrieval') {
// An empty oracle context short-circuits retrieval without injecting
// anything — the model answers from parametric memory alone.
pipelineOptions.oracleContext = []
}
const answers: string[] = []
const scores: GenerationScores[] = []
let retrievedDocIds: string[] = []
let injectedChunks = 0
let error: string | undefined
for (let attempt = 0; attempt < repeats; attempt++) {
try {
const trace = await this.pipeline.buildPrompt(messages, options.model, pipelineOptions)
retrievedDocIds = uniqueDocIds(trace.retrieved)
injectedChunks = trace.injected.length
const context = trace.injected.map((c) => c.text).join('\n\n')
const answer = isMock
? mockAnswer(context)
: await this.generate(options.model, trace.messages, trace.numCtx)
answers.push(answer)
scores.push(
scoreAnswer({
answer,
context,
mustInclude: golden.mustInclude,
mustNotInclude: golden.mustNotInclude,
expectRefusal: golden.expectRefusal,
})
)
} catch (err) {
error = errorText(err)
break
}
}
return {
id: golden.id,
tags: golden.tags,
expectRefusal: golden.expectRefusal,
answers,
scores,
correctness: summarizeRepeats(scores.map((s) => s.correct)),
refusalCorrectness: summarizeRepeats(scores.map((s) => s.refusalCorrect)),
leakageFree: summarizeRepeats(scores.map((s) => s.leakage.length === 0)),
groundedness: summarizeNumeric(scores.map((s) => s.numericGroundedness)),
retrievedDocIds,
injectedChunks,
error,
}
}
private async generate(model: string, messages: OllamaChatMessage[], numCtx?: number): Promise<string> {
const response = await this.ollamaService.chat({
model,
messages,
numCtx,
temperature: EVAL_TEMPERATURE,
seed: EVAL_SEED,
})
return response.message.content.trim()
}
}
/**
* The extractive reference answer: echo the context, or decline when there is
* none. Deterministic, model-free, and an honest ceiling for the current
* retrieval.
*/
export function mockAnswer(context: string): string {
if (!context.trim()) return "I don't have information about that."
return context
}
function uniqueDocIds(chunks: RetrievedChunk[]): string[] {
const ids: string[] = []
const seen = new Set<string>()
for (const chunk of chunks) {
const docId = docIdFromSource(chunk.metadata?.source)
if (docId && !seen.has(docId)) {
seen.add(docId)
ids.push(docId)
}
}
return ids
}
function errorText(error: unknown): string {
return error instanceof Error ? error.message : String(error)
}
const rate = (values: boolean[]): number | null =>
values.length === 0 ? null : values.filter(Boolean).length / values.length
export function aggregateGeneration(cases: GenerationCaseResult[]): GenerationAggregate {
const scored = cases.filter((c) => c.scores.length > 0)
const allScores = scored.flatMap((c) => c.scores)
return {
cases: cases.length,
unstable: scored.filter((c) => c.correctness.unstable).length,
errors: cases.filter((c) => c.error).length,
// Mean pass-rate rather than all-or-nothing, so a case that passes 2 of 3
// is reported as 0.67 instead of being silently rounded either way.
correctness: scored.length === 0 ? null : mean(scored.map((c) => c.correctness.passRate)),
refusalCorrectness:
scored.length === 0 ? null : mean(scored.map((c) => c.refusalCorrectness.passRate)),
leakageRate: rate(allScores.map((s) => s.leakage.length > 0)),
thinkTagLeakRate: rate(allScores.map((s) => s.thinkTagLeak)),
markdownRate: rate(allScores.map((s) => s.markdownFormatted)),
groundedness: summarizeNumeric(allScores.map((s) => s.numericGroundedness)),
meanAnswerLength: allScores.length === 0 ? null : mean(allScores.map((s) => s.length)),
}
}
export function aggregateGenerationByTag(
cases: GenerationCaseResult[]
): Record<string, GenerationAggregate> {
const tags = new Set(cases.flatMap((c) => c.tags))
const out: Record<string, GenerationAggregate> = {}
for (const tag of [...tags].sort()) {
out[tag] = aggregateGeneration(cases.filter((c) => c.tags.includes(tag)))
}
return out
}
const mean = (values: number[]) => values.reduce((a, b) => a + b, 0) / values.length

154
admin/app/services/eval_report_service.ts Normal file
View File

@ -0,0 +1,154 @@
import { inject } from '@adonisjs/core'
import { execFile } from 'node:child_process'
import { mkdir, readFile, writeFile } from 'node:fs/promises'
import { dirname, join, resolve } from 'node:path'
import { promisify } from 'node:util'
import type { GenerationRunResult } from './eval_generation_service.js'
import type { RetrievalRunResult } from './eval_retrieval_service.js'
import { flattenByK, type EvalReport, type ReportMeta } from '../utils/eval/report.js'
const run = promisify(execFile)
export const EVAL_REPORTS_DIR = 'tests/eval/reports'
export const EVAL_BASELINES_DIR = 'tests/eval/baselines'
/**
* Turns run results into durable, comparable report files.
*
* Two formats, on purpose: JSON is what `eval:compare` diffs, and Markdown is
* what a human actually reads when a number moved and they need to know which
* question broke and what the model said.
*/
@inject()
export class EvalReportService {
async buildMeta(
kind: 'retrieval' | 'generation',
corpusFingerprint: string,
params: Record<string, unknown>
): Promise<ReportMeta> {
const [git, nomadVersion, platform] = await Promise.all([
readGitState(),
readNomadVersion(),
readPlatform(),
])
return {
kind,
createdAt: new Date().toISOString(),
corpusFingerprint,
gitSha: git.sha,
gitBranch: git.branch,
gitDirty: git.dirty,
nomadVersion,
platform,
params,
}
}
fromRetrieval(meta: ReportMeta, result: RetrievalRunResult): EvalReport {
const flatten = (agg: RetrievalRunResult['overall']) => ({
...flattenByK('recall', agg.recall),
...flattenByK('hitRate', agg.hitRate),
...flattenByK('precision', agg.precision),
...flattenByK('ndcg', agg.ndcg),
mrr: agg.mrr,
emptyRateOnAnswerable: agg.emptyRateOnAnswerable,
nonEmptyRateOnRefusal: agg.nonEmptyRateOnRefusal,
})
return {
meta,
metrics: flatten(result.overall),
byTag: Object.fromEntries(Object.entries(result.byTag).map(([tag, agg]) => [tag, flatten(agg)])),
cases: result.cases,
}
}
fromGeneration(meta: ReportMeta, result: GenerationRunResult): EvalReport {
const flatten = (agg: GenerationRunResult['overall']) => ({
correctness: agg.correctness,
refusalCorrectness: agg.refusalCorrectness,
leakageRate: agg.leakageRate,
thinkTagLeakRate: agg.thinkTagLeakRate,
markdownRate: agg.markdownRate,
groundedness: agg.groundedness?.mean ?? null,
})
return {
meta,
metrics: flatten(result.overall),
byTag: Object.fromEntries(Object.entries(result.byTag).map(([tag, agg]) => [tag, flatten(agg)])),
// Answers are kept in full. When a score moves, the only question anyone
// actually asks is "what did it say?", and a truncated answer cannot
// answer it.
cases: result.cases,
}
}
/** Write `<slug>.json` and `<slug>.md`; returns the JSON path. */
async write(report: EvalReport, slug: string, markdown: string): Promise<string> {
const dir = resolve(join(process.cwd(), EVAL_REPORTS_DIR))
await mkdir(dir, { recursive: true })
const jsonPath = join(dir, `${slug}.json`)
await writeFile(jsonPath, JSON.stringify(report, null, 2))
await writeFile(join(dir, `${slug}.md`), markdown)
return jsonPath
}
async read(path: string): Promise<EvalReport> {
const parsed = JSON.parse(await readFile(resolve(path), 'utf8'))
if (!parsed?.meta?.kind || !parsed?.metrics) {
throw new Error(`${path} is not an eval report`)
}
return parsed as EvalReport
}
/**
* Baselines are filed under their corpus fingerprint. That is not cosmetic:
* it makes it structurally impossible to overwrite the baseline for one
* corpus with a run against another.
*/
baselinePath(report: EvalReport, name: string): string {
return resolve(
join(process.cwd(), EVAL_BASELINES_DIR, report.meta.corpusFingerprint, `${name}.json`)
)
}
async promoteToBaseline(report: EvalReport, name: string): Promise<string> {
const path = this.baselinePath(report, name)
await mkdir(dirname(path), { recursive: true })
await writeFile(path, JSON.stringify(report, null, 2))
return path
}
}
async function readGitState() {
const git = async (args: string[]) => (await run('git', args)).stdout.trim()
try {
const [sha, branch, status] = await Promise.all([
git(['rev-parse', 'HEAD']),
git(['rev-parse', '--abbrev-ref', 'HEAD']),
git(['status', '--porcelain']),
])
// A dirty tree matters: it means the report does not correspond to any
// commit, so a later "this regressed at sha X" claim would be unfounded.
return { sha, branch, dirty: status.length > 0 }
} catch {
return { sha: null, branch: null, dirty: false }
}
}
async function readNomadVersion(): Promise<string | null> {
try {
const pkg = JSON.parse(await readFile(resolve(join(process.cwd(), '..', 'package.json')), 'utf8'))
return pkg.version ?? null
} catch {
return null
}
}
async function readPlatform() {
const os = await import('node:os')
return {
cpuArchitecture: os.arch(),
osName: `${os.type()} ${os.release()}`,
nodeVersion: process.version,
}
}

153
admin/app/services/eval_retrieval_service.ts Normal file
View File

@ -0,0 +1,153 @@
import { EvalCorpusService } from '#services/eval_corpus_service'
import { RagService } from '#services/rag_service'
import { inject } from '@adonisjs/core'
import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js'
import { RAG_DEFAULT_SCORE_THRESHOLD, RAG_DEFAULT_TOP_K } from '../../constants/ollama.js'
import type { RetrievalStages } from '../../types/rag.js'
import { docIdFromSource } from '../utils/eval/corpus_source.js'
import type { Golden } from '../utils/eval/golden_set.js'
import {
aggregate,
aggregateByTag,
DEFAULT_K_VALUES,
scoreCase,
type RetrievalAggregate,
type RetrievalCase,
type RetrievalCaseResult,
type ScoredChunk,
} from '../utils/eval/retrieval_metrics.js'
export type RetrievalRunOptions = {
topK?: number
scoreThreshold?: number
kValues?: number[]
/** Score the raw dense / reranked / diversified orderings separately. */
ablate?: boolean
}
export type StageAblation = {
dense: RetrievalAggregate
reranked: RetrievalAggregate
diversified: RetrievalAggregate
}
export type RetrievalRunResult = {
params: { topK: number; scoreThreshold: number; kValues: number[] }
overall: RetrievalAggregate
byTag: Record<string, RetrievalAggregate>
cases: RetrievalCaseResult[]
ablation: StageAblation | null
/**
* Chunks that came back without a resolvable eval doc id. Non-zero means the
* collection filter leaked and the run is measuring the wrong corpus.
*/
unresolvedChunks: number
}
/**
* Runs the golden question set through NOMAD's real retrieval path and scores
* the result.
*
* Deliberately does not touch the chat model. The only model call is the
* embedding of each query, whose output is stable, so two runs over the same
* corpus produce identical numbers on any hardware. That is what makes this the
* fast inner loop and the only tier worth gating CI on: a movement here is a
* code change, full stop.
*
* Multi-turn goldens are scored on their raw final message. Resolving the
* coreference would need the chat model, which would make this tier
* non-deterministic — so the multi-turn bucket here reports the honest floor,
* and the rewrite's contribution is measured in the generation tier instead.
*/
@inject()
export class EvalRetrievalService {
constructor(
private ragService: RagService,
private corpusService: EvalCorpusService
) {}
async run(goldens: Golden[], options: RetrievalRunOptions = {}): Promise<RetrievalRunResult> {
const topK = options.topK ?? RAG_DEFAULT_TOP_K
const scoreThreshold = options.scoreThreshold ?? RAG_DEFAULT_SCORE_THRESHOLD
const kValues = options.kValues ?? DEFAULT_K_VALUES
const cases: RetrievalCase[] = []
const denseCases: RetrievalCase[] = []
const rerankedCases: RetrievalCase[] = []
const diversifiedCases: RetrievalCase[] = []
let unresolvedChunks = 0
for (const golden of goldens) {
const stages: RetrievalStages = {}
const docs = await this.ragService.searchSimilarDocuments(
golden.query,
topK,
scoreThreshold,
KB_EVAL_COLLECTION,
options.ablate ? stages : undefined
)
const retrieved: ScoredChunk[] = docs.map((d) => {
const docId = docIdFromSource(d.metadata?.source)
if (!docId) unresolvedChunks++
return { docId, score: d.score, semanticScore: d.metadata?.semantic_score }
})
cases.push(toCase(golden, retrieved))
if (options.ablate) {
denseCases.push(toCase(golden, stageToChunks(stages.dense)))
rerankedCases.push(toCase(golden, stageToChunks(stages.reranked)))
diversifiedCases.push(toCase(golden, stageToChunks(stages.diversified)))
}
}
const results = cases.map((c) => scoreCase(c, kValues))
const aggregateStage = (stageCases: RetrievalCase[]) =>
aggregate(stageCases, stageCases.map((c) => scoreCase(c, kValues)), kValues)
return {
params: { topK, scoreThreshold, kValues },
overall: aggregate(cases, results, kValues),
byTag: aggregateByTag(cases, results, kValues),
cases: results,
ablation: options.ablate
? {
dense: aggregateStage(denseCases),
reranked: aggregateStage(rerankedCases),
diversified: aggregateStage(diversifiedCases),
}
: null,
unresolvedChunks,
}
}
/** Confirm the corpus is actually ingested before reporting a score of zero. */
async assertCorpusReady(): Promise<number> {
const chunks = await this.corpusService.count()
if (chunks === 0) {
throw new Error(
'The eval corpus is not ingested — every score would be zero. Run: node ace eval:corpus --ingest'
)
}
return chunks
}
}
function toCase(golden: Golden, retrieved: ScoredChunk[]): RetrievalCase {
return {
id: golden.id,
tags: golden.tags,
retrieved,
relevantDocIds: golden.relevantDocIds,
expectRefusal: golden.expectRefusal,
}
}
function stageToChunks(stage: Array<{ source?: string; score: number }> | undefined): ScoredChunk[] {
return (stage ?? []).map((entry) => ({
docId: docIdFromSource(entry.source),
score: entry.score,
}))
}

18
admin/app/services/ollama_service.ts
View File

@ -48,6 +48,12 @@ type ChatInput = {
thinkingCapable?: boolean
stream?: boolean
numCtx?: number
// Sampling controls. Left unset for normal chat, which inherits the backend's
// defaults — the historical behaviour. The eval harness sets temperature 0 and
// a fixed seed so repeated runs are as close to comparable as llama.cpp allows
// (batching and GPU non-determinism still move outputs, hence --repeats).
temperature?: number
seed?: number
// Aborts the upstream request when the client disconnects, so an abandoned generation
// doesn't keep decoding server-side and block Ollama's single parallel slot (#1065).
signal?: AbortSignal
@ -350,6 +356,12 @@ export class OllamaService {
if (chatRequest.numCtx) {
params.num_ctx = chatRequest.numCtx
}
if (chatRequest.temperature !== undefined) {
params.temperature = chatRequest.temperature
}
if (chatRequest.seed !== undefined) {
params.seed = chatRequest.seed
}
const response = await this.openai.chat.completions.create(params, { signal: chatRequest.signal })
const choice = response.choices[0]
@ -392,6 +404,12 @@ export class OllamaService {
if (chatRequest.numCtx) {
params.num_ctx = chatRequest.numCtx
}
if (chatRequest.temperature !== undefined) {
params.temperature = chatRequest.temperature
}
if (chatRequest.seed !== undefined) {
params.seed = chatRequest.seed
}
const stream = (await this.openai.chat.completions.create(params, {
signal: chatRequest.signal,

222
admin/app/services/rag_pipeline_service.ts Normal file
View File

@ -0,0 +1,222 @@
import { NomadMdService } from '#services/nomad_md_service'
import { OllamaService } from '#services/ollama_service'
import { RagService } from '#services/rag_service'
import { inject } from '@adonisjs/core'
import logger from '@adonisjs/core/services/logger'
import {
RAG_CONTEXT_LIMITS,
RAG_DEFAULT_SCORE_THRESHOLD,
RAG_DEFAULT_TOP_K,
SYSTEM_PROMPTS,
} from '../../constants/ollama.js'
import type { OllamaChatMessage } from '../../types/ollama.js'
import type { PipelineOptions, PipelineTrace, RetrievedChunk } from '../../types/rag.js'
import {
buildContextBlock,
deriveNumCtx,
getContextLimitsForModel,
trimToContextBudget,
} from '../utils/rag_prompt.js'
/**
* Everything that happens between "a user sent a message" and "a payload goes
* to Ollama": system-prompt assembly, history-aware query rewriting, retrieval,
* model-size-aware context trimming, and the num_ctx decision.
*
* This used to live inline in OllamaController.chat. It was moved here so there
* is exactly one implementation of the prompt-building pipeline — the chat
* endpoint and the eval harness both call `buildPrompt`, so a measurement of
* the harness is a measurement of production, not of a copy that drifts.
*
* The behaviour is a verbatim port. Every quirk preserved below is marked; the
* quirks are worth fixing but each one changes output, and the point of the
* harness is to stop changing output without measuring it.
*/
@inject()
export class RagPipelineService {
constructor(
private ollamaService: OllamaService,
private ragService: RagService,
private nomadMdService: NomadMdService
) {}
/**
* Build the exact message array to send to Ollama, plus a trace of every
* decision made along the way.
*
* The caller passes the conversation as received; this never mutates it.
*/
async buildPrompt(
messages: OllamaChatMessage[],
model: string,
opts: PipelineOptions = {}
): Promise<PipelineTrace> {
const working: OllamaChatMessage[] = [...messages]
// Default formatting prompt, only when the caller supplied no system message.
const hasSystemMessage = working.some((msg) => msg.role === 'system')
if (!hasSystemMessage) {
logger.debug('[RagPipeline] Injecting system prompt')
working.unshift({ role: 'system', content: SYSTEM_PROMPTS.default })
}
// The user-managed NOMAD.md goes in front of the formatting prompt so the
// user's persistent instructions take precedence. Skipped in evals, where a
// developer's personal NOMAD.md would silently skew every score.
if (!opts.skipNomadMd) {
const nomadPrompt = await this.nomadMdService.getSystemPrompt()
if (nomadPrompt) {
logger.debug('[RagPipeline] Injecting NOMAD.md system prompt')
working.unshift({ role: 'system', content: nomadPrompt })
}
}
const trace: PipelineTrace = {
rewrittenQuery: null,
didRewrite: false,
retrieved: [],
injected: [],
messages: working,
numCtx: undefined,
contextLimits: { maxResults: RAG_DEFAULT_TOP_K, maxTokens: 0 },
timings: { rewriteMs: 0, retrievalMs: 0 },
}
// --- Retrieval -------------------------------------------------------
// oracleContext bypasses retrieval entirely (eval `oracle` mode).
let relevantDocs: RetrievedChunk[] = []
if (opts.oracleContext) {
relevantDocs = opts.oracleContext
trace.retrieved = relevantDocs
} else {
const rewriteStart = Date.now()
const { query, didRewrite } = await this.resolveRetrievalQuery(working, model, opts)
trace.timings.rewriteMs = Date.now() - rewriteStart
trace.rewrittenQuery = query
trace.didRewrite = didRewrite
if (query) {
const retrievalStart = Date.now()
relevantDocs = await this.ragService.searchSimilarDocuments(
query,
opts.topK ?? RAG_DEFAULT_TOP_K,
opts.scoreThreshold ?? RAG_DEFAULT_SCORE_THRESHOLD,
opts.collection
)
trace.timings.retrievalMs = Date.now() - retrievalStart
trace.retrieved = relevantDocs
logger.debug(
`[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${query}"`
)
}
}
// --- Context trimming + injection -------------------------------------
if (relevantDocs.length > 0) {
const limits = getContextLimitsForModel(model, RAG_CONTEXT_LIMITS)
trace.contextLimits = limits
const trimmedDocs = trimToContextBudget(relevantDocs, limits)
trace.injected = trimmedDocs
logger.debug(
`[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${model}, maxResults: ${limits.maxResults}, maxTokens: ${limits.maxTokens || 'unlimited'})`
)
const systemMessage: OllamaChatMessage = {
role: 'system',
content: SYSTEM_PROMPTS.rag_context(buildContextBlock(trimmedDocs)),
}
// After any existing system messages, before the first non-system message.
const firstNonSystemIndex = working.findIndex((msg) => msg.role !== 'system')
const insertIndex = firstNonSystemIndex === -1 ? 0 : firstNonSystemIndex
working.splice(insertIndex, 0, systemMessage)
}
trace.numCtx = deriveNumCtx(working)
if (trace.numCtx) {
logger.debug(`[RagPipeline] Large system prompt, requesting num_ctx: ${trace.numCtx}`)
}
return trace
}
/**
* Decide what string to hand to retrieval.
*
* Returns null when the RAG pipeline should be skipped entirely — an empty
* knowledge base, or a conversation with no user message at all.
*/
private async resolveRetrievalQuery(
messages: OllamaChatMessage[],
model: string,
opts: PipelineOptions
): Promise<{ query: string | null; didRewrite: boolean }> {
const lastUserMessage = [...messages].reverse().find((msg) => msg.role === 'user')
try {
// Skip the entire RAG pipeline if there are no documents to search.
const hasDocuments = await this.ragService.hasDocuments()
if (!hasDocuments) {
return { query: null, didRewrite: false }
}
if (opts.skipQueryRewrite) {
return { query: lastUserMessage?.content ?? null, didRewrite: false }
}
// Last 6 messages ≈ 3 turns.
//
// PRESERVED QUIRK: this slice is taken *after* system messages have been
// unshifted, so on short conversations the system prompts land inside the
// window and get labelled "Assistant" in the transcript below. Faithful
// to the original; a candidate fix once the harness can measure it.
const recentMessages = messages.slice(-6)
// Skip rewriting on the very first turn — with only one user message there
// is no prior context to fold in, so the rewrite would just echo the
// message back at the cost of an extra LLM round-trip. From the first
// follow-up onward the rewrite carries entities from earlier turns
// ("the bars" -> "Hershey's bars chocolate poisoning dog"); without it,
// embeddings match nothing and the assistant loses the thread.
const userMessages = recentMessages.filter((msg) => msg.role === 'user')
if (userMessages.length < 2) {
return { query: lastUserMessage?.content ?? null, didRewrite: false }
}
const conversationContext = recentMessages
.map((msg) => {
const role = msg.role === 'user' ? 'User' : 'Assistant'
// Truncate assistant messages to keep the rewrite prompt manageable.
const content =
msg.role === 'assistant'
? msg.content.slice(0, 200) + (msg.content.length > 200 ? '...' : '')
: msg.content
return `${role}: "${content}"`
})
.join('\n')
const response = await this.ollamaService.chat({
model,
messages: [
{ role: 'system', content: SYSTEM_PROMPTS.query_rewrite },
{
role: 'user',
content: `Conversation:\n${conversationContext}\n\nRewritten Query:`,
},
],
})
const rewrittenQuery = response.message.content.trim()
logger.info(`[RAG] Query rewritten: "${rewrittenQuery}"`)
return { query: rewrittenQuery, didRewrite: true }
} catch (error) {
logger.error(
`[RAG] Query rewriting failed: ${error instanceof Error ? error.message : error}`
)
// Fall back to the last user message rather than losing retrieval entirely.
return { query: lastUserMessage?.content ?? null, didRewrite: false }
}
}
}

97
admin/app/services/rag_service.ts
View File

@ -22,7 +22,18 @@ import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision
import { decideContentReindex, type ReindexOutcome } from '../utils/content_reindex_decision.js'
import KbRatioRegistry from '#models/kb_ratio_registry'
import { decideWarnings } from '../utils/kb_warning_decision.js'
import type { FileWarning, FileWarningsResult, StoredFileInfo } from '../../types/rag.js'
import type { FileWarning, FileWarningsResult, RetrievalStages, StoredFileInfo } from '../../types/rag.js'
import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js'
/**
* Qdrant filter that hides the developer eval corpus from every read path that
* enumerates the *user's* knowledge base. The fixtures share this collection
* (NOMAD collections are a payload tag), so without this they would show up as
* broken files in the KB UI and skew the ingest-health warnings.
*/
const EXCLUDE_EVAL_FILTER = {
must_not: [{ key: 'collection', match: { value: KB_EVAL_COLLECTION } }],
}
import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
import { ZIMExtractionService } from './zim_extraction_service.js'
import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
@ -59,6 +70,10 @@ export class RagService {
public static MODEL_CONTEXT_LENGTH = 2048 // nomic-embed-text has 2K token context
public static MAX_SAFE_TOKENS = 1600 // Leave buffer for prefix and tokenization variance
public static TARGET_TOKENS_PER_CHUNK = 1500 // Target 1500 tokens per chunk for embedding
// Overlap between adjacent chunks. Was an inline literal at the chunker call
// site; named here because it shapes what lands in the vector store and so
// must be part of the eval corpus fingerprint.
public static CHUNK_OVERLAP_TOKENS = 150
public static PREFIX_TOKEN_BUDGET = 10 // Reserve ~10 tokens for prefixes
public static CHAR_TO_TOKEN_RATIO = 2 // Conservative chars-per-token estimate; technical docs
// (numbers, symbols, abbreviations) tokenize denser
@ -336,7 +351,7 @@ export class RagService {
// We need to convert our embedding model's token counts to character counts
// since nomic-embed-text tokenizer uses ~3 chars per token
const targetCharsPerChunk = Math.floor(RagService.TARGET_TOKENS_PER_CHUNK * RagService.CHAR_TO_TOKEN_RATIO)
const overlapChars = Math.floor(150 * RagService.CHAR_TO_TOKEN_RATIO)
const overlapChars = Math.floor(RagService.CHUNK_OVERLAP_TOKENS * RagService.CHAR_TO_TOKEN_RATIO)
const chunker = await TokenChunker.create({
chunkSize: targetCharsPerChunk,
@ -840,7 +855,15 @@ export class RagService {
query: string,
limit: number = 5,
scoreThreshold: number = 0.3, // Lower default threshold - was 0.7, now 0.3
collection?: string
collection?: string,
/**
* Optional sink for the intermediate ranked lists. When supplied, the raw
* dense order, the post-rerank order, and the post-diversity order are all
* written here. Purely observational — nothing about the returned result
* changes — and it is what lets the eval harness answer "is the reranking
* heuristic actually helping?" without a second implementation of search.
*/
stagesOut?: RetrievalStages
): Promise<Array<{ text: string; score: number; metadata?: Record<string, any> }>> {
try {
logger.debug(`[RAG] Starting similarity search for query: "${query}"`)
@ -947,6 +970,21 @@ export class RagService {
// Apply source diversity penalty to avoid all results from the same document
const diverseResults = this.applySourceDiversity(rerankedResults)
// Record the three ranked lists for the eval harness's stage ablation.
// Sliced to `limit` so each stage is compared on the window that would
// actually have been injected, not on the wider rerank candidate pool.
if (stagesOut) {
const summarize = (rows: Array<{ text: string; score: number; source?: string }>) =>
rows.slice(0, limit).map((r) => ({ source: r.source, score: r.score }))
stagesOut.dense = summarize(resultsWithMetadata)
stagesOut.reranked = rerankedResults
.slice(0, limit)
.map((r) => ({ source: r.source, score: r.finalScore }))
stagesOut.diversified = diverseResults
.slice(0, limit)
.map((r) => ({ source: r.source, score: r.finalScore }))
}
// Return top N results with enhanced metadata
return diverseResults.slice(0, limit).map((result) => ({
text: result.text,
@ -955,6 +993,10 @@ export class RagService {
chunk_index: result.chunk_index,
created_at: result.created_at,
semantic_score: result.score,
// The originating file/ZIM path. Stored on every point but previously
// dropped here, which made it impossible to map a retrieved chunk back
// to its document — needed for citations and for recall@k scoring.
source: result.source,
// Enhanced ZIM metadata (likely be undefined for non-ZIM content)
article_title: result.article_title,
section_title: result.section_title,
@ -1149,6 +1191,10 @@ export class RagService {
key: 'source',
limit: RagService.FACET_SOURCE_LIMIT,
exact: true,
// Keep the developer eval corpus out of the user's Stored Files list.
// It shares this Qdrant collection but is not the user's content, and
// its fixture paths would render as broken/missing files in the KB UI.
filter: EXCLUDE_EVAL_FILTER,
})
for (const hit of facetResult.hits) {
if (typeof hit.value === 'string') sources.add(hit.value)
@ -1221,7 +1267,10 @@ export class RagService {
})
const collections = new Set<string>()
for (const hit of facetResult.hits) {
if (typeof hit.value === 'string') collections.add(hit.value)
// The reserved eval tag is internal; never offer it in a subject picker.
if (typeof hit.value === 'string' && hit.value !== KB_EVAL_COLLECTION) {
collections.add(hit.value)
}
}
return Array.from(collections).sort()
}
@ -1317,6 +1366,43 @@ export class RagService {
}
}
/**
* Count the points carrying a given `collection` tag.
*
* Distinct from the facet-based counts above, which deliberately exclude
* internal collections. This answers "how many chunks are in exactly this
* collection", which is what the eval harness needs to confirm an ingest
* landed and what a future per-collection UI would want.
*/
public async countChunksInCollection(collection: string): Promise<number> {
await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
const result = await this.qdrant!.count(RagService.CONTENT_COLLECTION_NAME, {
filter: { must: [{ key: 'collection', match: { value: collection } }] },
exact: true,
})
return result.count
}
/**
* Hard-delete every point carrying a given `collection` tag.
*
* Note this is NOT what `deleteKnowledgeCollection` does — that one clears the
* tag and leaves the user's documents in place, because deleting a user's
* content because they renamed a folder would be indefensible. This one
* genuinely removes the points, and exists for the eval corpus, which is
* disposable by construction. Returns the number of points removed.
*/
public async deleteCollectionPoints(collection: string): Promise<number> {
await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
const before = await this.countChunksInCollection(collection)
if (before === 0) return 0
await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, {
wait: true,
filter: { must: [{ key: 'collection', match: { value: collection } }] },
})
return before
}
/**
* Resolve a stored-file `source` to an absolute disk path, but only if the
* path lives under the uploads directory. Mirrors the docs_service traversal
@ -1434,6 +1520,9 @@ export class RagService {
key: 'source',
limit: RagService.FACET_SOURCE_LIMIT,
exact: true,
// Same exclusion as getStoredFiles: eval fixtures are not user files,
// and counting them here would raise bogus zero_chunks warnings.
filter: EXCLUDE_EVAL_FILTER,
})
for (const hit of facetResult.hits) {
if (typeof hit.value === 'string') chunksBySource.set(hit.value, hit.count)

38
admin/app/utils/eval/corpus_source.ts Normal file
View File

@ -0,0 +1,38 @@
import { basename, join, resolve, sep } from 'node:path'
/** Where the frozen corpus lives, relative to the app root. */
export const EVAL_CORPUS_DIR = 'tests/eval/corpus'
/**
* Map a retrieved chunk's `source` payload back to its corpus document id, or
* null if the chunk did not come from the eval corpus.
*
* The eval corpus shares the `nomad_knowledge_base` Qdrant collection with the
* developer's real documents — NOMAD "collections" are a payload tag, not
* separate Qdrant collections — and is isolated by a
* `collection: __nomad_eval__` filter that Qdrant applies during search. That
* filter does hold, but a harness whose every number depends on a filter should
* be able to *prove* the filter held rather than assume it. This function is
* that proof: the caller counts every unresolved chunk and fails the run if the
* count is non-zero.
*
* The check is on the resolved **path**, not the file extension. An earlier
* version accepted any `*.md`, which was worse than useless here: NOMAD embeds
* its own `admin/docs/*.md` into the knowledge base on first run, so a leaked
* `faq.md` would have resolved to the plausible-looking document id "faq", been
* counted as a merely-irrelevant chunk, and quietly depressed precision with no
* indication that anything had gone wrong.
*
* Mirrors the resolve-then-prefix-check guard in `RagService.resolveUploadPath`.
* `corpusDir` is injectable so this stays testable without a real corpus on disk.
*/
export function docIdFromSource(source: unknown, corpusDir?: string): string | null {
if (typeof source !== 'string') return null
const dir = resolve(corpusDir ?? join(process.cwd(), EVAL_CORPUS_DIR))
const abs = resolve(source)
// The trailing separator matters: without it, a sibling directory such as
// "…/corpus-backup" shares a string prefix with "…/corpus" and would pass.
if (!abs.startsWith(dir + sep)) return null
if (!abs.endsWith('.md')) return null
return basename(abs, '.md')
}

254
admin/app/utils/eval/generation_metrics.ts Normal file
View File

@ -0,0 +1,254 @@
/**
* Deterministic scorers for generated answers.
*
* Every metric here is computable offline with no judge model, which is what
* makes the generation tier gateable. An LLM judge can be layered on later for
* the things regex genuinely cannot see (nuanced faithfulness, completeness),
* but nothing in this file needs one — and a number you can compute without a
* model is a number that cannot drift because the judge changed.
*
* These are intentionally *conservative*. Each one is a proxy, and each proxy's
* blind spot is documented where it lives. A proxy you understand the limits of
* beats a score you cannot explain.
*/
/** Answer-level scoring of one generated response against one golden. */
export type GenerationScores = {
/** Every mustInclude pattern matched and no mustNotInclude pattern did. */
correct: boolean
matchedRequired: string[]
missedRequired: string[]
/** Forbidden patterns that appeared. Non-empty means a specific wrong claim. */
hitForbidden: string[]
/** The model declined to answer. */
refused: boolean
/**
* Refusal behaviour was right: declined an out-of-corpus question, or
* answered an answerable one.
*/
refusalCorrect: boolean
/** The answer narrated its own retrieval ("according to Context 1"). */
leakage: string[]
/** Reasoning tags survived into the user-visible answer. */
thinkTagLeak: boolean
/**
* Fraction of the answer's numeric claims that also appear in the injected
* context. null when the answer makes no numeric claims.
*/
numericGroundedness: number | null
/** Numbers asserted by the answer that the context does not support. */
ungroundedNumbers: string[]
/** The answer used at least some markdown structure. */
markdownFormatted: boolean
/** Characters. Useful for spotting a model that answers by writing an essay. */
length: number
}
/**
* Phrases that mean "I am not answering this".
*
* Tuned against NOMAD's actual rag_context prompt, which instructs the model to
* answer from general knowledge rather than hedge — so a hedge here is either a
* correct refusal on an out-of-corpus question or a prompt regression on an
* answerable one. Which of those it is, is exactly what `refusalCorrect` says.
*/
const REFUSAL_PATTERNS: RegExp[] = [
/\bi (?:don'?t|do not) (?:know|have)\b/i,
/\bi(?:'m| am) (?:not able|unable) to\b/i,
/\bi (?:couldn'?t|could not|can'?t|cannot) find\b/i,
/\b(?:no|not enough) (?:information|details|data)\b/i,
/\b(?:does|do) not (?:contain|include|mention|specify|provide|cover)\b/i,
/\b(?:isn'?t|is not|aren'?t|are not) (?:mentioned|specified|covered|available|provided)\b/i,
/\bnot (?:mentioned|specified|covered|stated|documented)\b/i,
/\bunable to (?:answer|determine|find)\b/i,
/\bthere (?:is|'s) no (?:information|mention|record)\b/i,
// "I'd" has no space before the contraction, so this cannot reuse the
// `i (?:would|'d)` shape used above.
/\bi(?:'d| would) need more\b/i,
]
/**
* Phrases that narrate the retrieval machinery.
*
* The rag_context prompt explicitly forbids these ("Never narrate your
* retrieval or reasoning process"), so any hit is a measurable prompt
* regression. This is the cheapest, sharpest signal in the whole harness: pure
* regex, zero ambiguity, and it catches a bad prompt edit on the first run.
*/
const LEAKAGE_PATTERNS: RegExp[] = [
/\b(?:according to|based on|per|from) the (?:provided |retrieved |given |supplied )?context\b/i,
/\bcontext \d+\b/i,
/\bthe knowledge ?base\b/i,
/\bthe (?:provided|retrieved|supplied|given) (?:documents?|passages?|excerpts?|text)\b/i,
/\bthe context (?:does not|doesn'?t|is|was|seems|appears)\b/i,
/\b(?:in|from) the (?:documents?|passages?) (?:provided|above|below)\b/i,
// "I wasn't able to find specific context regarding X, but here's a general
// answer" is the verbatim symptom that started NOMAD's RAG work. It is both a
// refusal and a leak, and it is the single most important string this
// detector has to catch — a prompt change that brings it back must fail
// loudly on the very next run.
/\b(?:no|any|specific|relevant) context\b/i,
/\bsearch results?\b/i,
]
/** Reasoning-model tags that must never reach the user. */
const THINK_TAG = /<\/?(?:think|thought|thinking|reasoning)\b[^>]*>/i
export function detectRefusal(answer: string): boolean {
return REFUSAL_PATTERNS.some((re) => re.test(answer))
}
export function detectLeakage(answer: string): string[] {
return LEAKAGE_PATTERNS.filter((re) => re.test(answer)).map((re) => re.source)
}
export function hasThinkTagLeak(answer: string): boolean {
return THINK_TAG.test(answer)
}
/**
* Loose markdown check: a header, a list, emphasis, a table, or a code fence.
*
* Deliberately loose. SYSTEM_PROMPTS.default asks for markdown "for
* readability", not for a specific structure, so requiring headers would fail
* perfectly good one-sentence answers. This only catches a model that has
* stopped formatting entirely.
*/
export function isMarkdownFormatted(answer: string): boolean {
return /(^|\n)\s{0,3}#{1,6}\s|(^|\n)\s*[-*+]\s|(^|\n)\s*\d+\.\s|\*\*[^*]+\*\*|`[^`]+`|(^|\n)\s*\|/.test(
answer
)
}
/**
* Numeric tokens the answer asserts, normalized for comparison.
*
* Numbers are the highest-value fabrication signal in this domain: a wrong
* boiling time, bleach dose, or canner pressure is a wrong answer with real
* consequences, and it is exactly the kind of specific the rag_context prompt
* forbids inventing.
*
* Thousands separators are stripped so "2,000" and "2000" compare equal.
*/
export function extractNumbers(text: string): string[] {
const matches = text.match(/\d[\d,]*(?:\.\d+)?/g) ?? []
const normalized = matches.map((m) => m.replace(/,/g, '').replace(/\.0+$/, ''))
return [...new Set(normalized)]
}
/**
* Fraction of the answer's numeric claims that the injected context supports.
*
* Returns null when the answer contains no numbers — a qualitative answer is
* not ungrounded, it is just not measurable this way, and folding it in as 1.0
* would quietly inflate the score.
*
* **Known limitation, stated plainly:** this only sees numbers. An answer that
* fabricates a procedure or a proper noun scores a perfect 1.0 here. It is a
* fabrication *detector*, not a faithfulness *guarantee* — the LLM-judge tier
* exists for the rest. Small integers (0-10) are excluded because they appear
* incidentally in almost any prose ("3 layers", "step 2") and would swamp the
* signal with false grounding.
*/
export function numericGroundedness(
answer: string,
context: string
): { score: number | null; ungrounded: string[] } {
const contextNumbers = new Set(extractNumbers(context))
const claimed = extractNumbers(answer).filter((n) => {
const value = Number.parseFloat(n)
return !Number.isNaN(value) && value > 10
})
if (claimed.length === 0) return { score: null, ungrounded: [] }
const ungrounded = claimed.filter((n) => !contextNumbers.has(n))
return { score: (claimed.length - ungrounded.length) / claimed.length, ungrounded }
}
/** Which of a golden's patterns matched, using case-insensitive regex semantics. */
export function matchPatterns(answer: string, patterns: string[]): { matched: string[]; missed: string[] } {
const matched: string[] = []
const missed: string[] = []
for (const pattern of patterns) {
if (new RegExp(pattern, 'i').test(answer)) matched.push(pattern)
else missed.push(pattern)
}
return { matched, missed }
}
export type ScoreAnswerInput = {
answer: string
/** The context text actually injected into the prompt; '' when none was. */
context: string
mustInclude: string[]
mustNotInclude: string[]
expectRefusal: boolean
}
export function scoreAnswer(input: ScoreAnswerInput): GenerationScores {
const { answer, context, mustInclude, mustNotInclude, expectRefusal } = input
const required = matchPatterns(answer, mustInclude)
const forbidden = matchPatterns(answer, mustNotInclude)
const refused = detectRefusal(answer)
const grounding = numericGroundedness(answer, context)
return {
// An out-of-corpus question has no mustInclude patterns, so `correct` there
// is carried entirely by refusalCorrect below rather than by assertions.
correct: required.missed.length === 0 && forbidden.matched.length === 0,
matchedRequired: required.matched,
missedRequired: required.missed,
hitForbidden: forbidden.matched,
refused,
refusalCorrect: expectRefusal ? refused : !refused,
leakage: detectLeakage(answer),
thinkTagLeak: hasThinkTagLeak(answer),
numericGroundedness: grounding.score,
ungroundedNumbers: grounding.ungrounded,
markdownFormatted: isMarkdownFormatted(answer),
length: answer.length,
}
}
// --- aggregation over repeats -------------------------------------------------
export type RepeatStats = {
/** How many of N repeats passed. */
passes: number
repeats: number
passRate: number
/**
* True when the outcome was neither always-pass nor always-fail. Unstable
* cases are excluded from gating — treating a coin flip as a regression is
* how a harness loses the team's trust in one afternoon.
*/
unstable: boolean
}
export function summarizeRepeats(outcomes: boolean[]): RepeatStats {
const repeats = outcomes.length
const passes = outcomes.filter(Boolean).length
return {
passes,
repeats,
passRate: repeats === 0 ? 0 : passes / repeats,
unstable: repeats > 1 && passes > 0 && passes < repeats,
}
}
export type NumericSummary = { mean: number; stddev: number; n: number }
/**
* Mean and population standard deviation, ignoring nulls.
*
* The stddev is not decoration: with temperature 0 it should be near zero, and
* a non-trivial value is the harness telling you the run is noisier than the
* difference you are about to interpret.
*/
export function summarizeNumeric(values: Array<number | null>): NumericSummary | null {
const defined = values.filter((v): v is number => v !== null && Number.isFinite(v))
if (defined.length === 0) return null
const mean = defined.reduce((a, b) => a + b, 0) / defined.length
const variance = defined.reduce((acc, v) => acc + (v - mean) ** 2, 0) / defined.length
return { mean, stddev: Math.sqrt(variance), n: defined.length }
}

190
admin/app/utils/eval/golden_set.ts Normal file
View File

@ -0,0 +1,190 @@
/**
* Loading and validating the golden question set, plus the corpus fingerprint.
*
* Pure and dependency-free so it runs under bare node in unit tests. Callers
* supply file contents; nothing here touches the filesystem except through the
* explicit `readFile`-shaped arguments the service passes in.
*/
/** A prior conversation turn, for multi-turn / coreference cases. */
export type GoldenTurn = { role: 'user' | 'assistant'; content: string }
export type Golden = {
/** Stable identifier. Used as the join key across reports and baselines. */
id: string
/** The user's question, as they would actually type it. */
query: string
/** Conversation history preceding `query`. Empty for single-turn cases. */
turns: GoldenTurn[]
/**
* Documents that genuinely answer the question, by corpus doc id (the
* markdown filename without its extension). Empty for out-of-corpus cases.
*/
relevantDocIds: string[]
/**
* Patterns the answer must contain. Each entry is a case-insensitive
* **regular expression** — plain text is a valid regex, and alternation lets
* one entry accept "3 minutes" or "three minutes" without inflating the list.
*/
mustInclude: string[]
/** Patterns the answer must NOT contain. Same regex semantics. */
mustNotInclude: string[]
/**
* True when the corpus genuinely cannot answer the question and the correct
* behaviour is to decline rather than invent. Scored as refusal-correctness.
*/
expectRefusal: boolean
/** Free-form buckets for per-slice reporting (single-hop, acronym, ...). */
tags: string[]
}
export class GoldenSetError extends Error {}
/**
* Parse a JSONL golden file, validating hard enough that a typo fails loudly at
* load rather than silently scoring zero for the rest of the project's life.
*/
export function parseGoldens(jsonl: string, sourceName = 'goldens'): Golden[] {
const goldens: Golden[] = []
const seen = new Set<string>()
jsonl.split('\n').forEach((rawLine, idx) => {
const line = rawLine.trim()
if (!line || line.startsWith('//')) return
const where = `${sourceName}:${idx + 1}`
let parsed: any
try {
parsed = JSON.parse(line)
} catch (err) {
throw new GoldenSetError(`${where}: not valid JSON — ${(err as Error).message}`)
}
const req = (field: string) => {
if (parsed[field] === undefined) throw new GoldenSetError(`${where}: missing "${field}"`)
return parsed[field]
}
const id = req('id')
if (typeof id !== 'string' || !id) throw new GoldenSetError(`${where}: "id" must be a non-empty string`)
if (seen.has(id)) throw new GoldenSetError(`${where}: duplicate id "${id}"`)
seen.add(id)
const query = req('query')
if (typeof query !== 'string' || !query.trim()) {
throw new GoldenSetError(`${where}: "query" must be a non-empty string`)
}
const strArray = (field: string): string[] => {
const v = parsed[field] ?? []
if (!Array.isArray(v) || v.some((x) => typeof x !== 'string')) {
throw new GoldenSetError(`${where}: "${field}" must be an array of strings`)
}
return v
}
const mustInclude = strArray('mustInclude')
const mustNotInclude = strArray('mustNotInclude')
// Compile every pattern now. A bad regex that only blows up on the one run
// where it finally matches is far worse than one that fails at load.
for (const pattern of [...mustInclude, ...mustNotInclude]) {
try {
new RegExp(pattern, 'i')
} catch (err) {
throw new GoldenSetError(`${where}: invalid regex ${JSON.stringify(pattern)} — ${(err as Error).message}`)
}
}
const turns = (parsed.turns ?? []) as GoldenTurn[]
if (!Array.isArray(turns) || turns.some((t) => t?.role !== 'user' && t?.role !== 'assistant')) {
throw new GoldenSetError(`${where}: "turns" must be an array of {role: user|assistant, content}`)
}
const expectRefusal = Boolean(parsed.expectRefusal)
const relevantDocIds = strArray('relevantDocIds')
if (expectRefusal && relevantDocIds.length > 0) {
throw new GoldenSetError(
`${where}: "${id}" expects a refusal but also lists relevant documents — one of those is wrong`
)
}
if (!expectRefusal && relevantDocIds.length === 0) {
throw new GoldenSetError(
`${where}: "${id}" lists no relevant documents and does not expect a refusal — it can never be scored`
)
}
goldens.push({
id,
query,
turns,
relevantDocIds,
mustInclude,
mustNotInclude,
expectRefusal,
tags: strArray('tags'),
})
})
if (goldens.length === 0) throw new GoldenSetError(`${sourceName}: no goldens found`)
return goldens
}
/**
* Every doc id a golden refers to must exist in the corpus, or recall is being
* computed against a target that can never be hit.
*/
export function assertGoldensMatchCorpus(goldens: Golden[], corpusDocIds: Iterable<string>): void {
const corpus = new Set(corpusDocIds)
const missing = new Set<string>()
for (const g of goldens) {
for (const docId of g.relevantDocIds) if (!corpus.has(docId)) missing.add(docId)
}
if (missing.size > 0) {
throw new GoldenSetError(
`goldens reference documents that are not in the corpus: ${[...missing].sort().join(', ')}`
)
}
}
/**
* Inputs to the corpus fingerprint. Anything that changes what ends up in the
* vector store belongs here — if it changes, prior reports are not comparable.
*/
export type FingerprintInputs = {
/** docId -> raw file contents. */
documents: Map<string, string>
chunkTokens: number
chunkOverlapTokens: number
charToTokenRatio: number
embeddingModel: string
embeddingDimension: number
}
/**
* A stable hash over the corpus and the ingest parameters that shaped it.
*
* Reports carry this. Comparing two reports with different fingerprints is
* meaningless, and the compare command refuses to do it — which is the whole
* point: it makes "I changed the chunk size and the score moved" impossible to
* confuse with "I changed the prompt and the score moved".
*
* `hash` is injected so this stays pure and testable; the service passes a
* node:crypto sha256.
*/
export function computeCorpusFingerprint(
inputs: FingerprintInputs,
hash: (input: string) => string
): string {
const docIds = [...inputs.documents.keys()].sort()
const parts: string[] = [
`chunkTokens=${inputs.chunkTokens}`,
`chunkOverlapTokens=${inputs.chunkOverlapTokens}`,
`charToTokenRatio=${inputs.charToTokenRatio}`,
`embeddingModel=${inputs.embeddingModel}`,
`embeddingDimension=${inputs.embeddingDimension}`,
]
for (const id of docIds) {
parts.push(`doc=${id}\n${inputs.documents.get(id)}`)
}
return hash(parts.join('\n---\n')).slice(0, 16)
}

22
admin/app/utils/eval/quiet.ts Normal file
View File

@ -0,0 +1,22 @@
import logger from '@adonisjs/core/services/logger'
/**
* Turn down the application logger for the duration of an eval run.
*
* RagService logs every keyword extraction, every batch, and the text of every
* chunk at debug level. That is the right default for diagnosing an ingest, and
* completely wrong for a tool whose entire output is a score table — the
* numbers scroll off the top of the terminal before you can read them.
*
* Returns a restore function so the level is put back even if the caller throws.
* Pass `debug: true` (from `--debug`) to leave the logger alone when you are
* actually trying to see the pipeline internals.
*/
export function quietLogging(debug = false): () => void {
if (debug) return () => {}
const previous = logger.level
logger.level = 'warn'
return () => {
logger.level = previous
}
}

254
admin/app/utils/eval/report.ts Normal file
View File

@ -0,0 +1,254 @@
/**
* Report shape, baseline diffing, and the gate decision.
*
* Pure functions over plain objects — the service layer does the file I/O. The
* comparison logic in particular has to be testable without a filesystem,
* because it is the piece that decides whether a pull request is blocked.
*/
export type ReportKind = 'retrieval' | 'generation'
/**
* Provenance. Every field here exists to answer "is this report comparable to
* that one?", and `corpusFingerprint` is the one that can veto the comparison
* outright.
*/
export type ReportMeta = {
kind: ReportKind
createdAt: string
corpusFingerprint: string
gitSha: string | null
gitBranch: string | null
gitDirty: boolean
nomadVersion: string | null
/** Host platform. Recorded for context; never used to justify a score. */
platform: {
cpuArchitecture: string | null
osName: string | null
nodeVersion: string
}
params: Record<string, unknown>
}
/**
* A report is metadata plus a flat bag of named metrics plus the per-case
* detail. Flattening the metrics is deliberate: it means the diff logic does
* not need to know anything about retrieval versus generation, so adding a
* metric later requires no change to the comparison or the gate.
*/
export type EvalReport = {
meta: ReportMeta
/** metric name -> value. null means "not measurable in this run". */
metrics: Record<string, number | null>
/** tag -> metric name -> value. */
byTag: Record<string, Record<string, number | null>>
/** Per-question detail, for the "what did it actually say" question. */
cases: unknown[]
}
/**
* Whether a metric going up is good.
*
* Without this the diff cannot tell an improvement from a regression, and a
* leakage rate falling to zero would be reported as a failure.
*/
export type MetricDirection = 'higher-is-better' | 'lower-is-better'
export const METRIC_DIRECTIONS: Record<string, MetricDirection> = {
// retrieval
'recall@1': 'higher-is-better',
'recall@3': 'higher-is-better',
'recall@5': 'higher-is-better',
'recall@10': 'higher-is-better',
'hitRate@1': 'higher-is-better',
'hitRate@3': 'higher-is-better',
'hitRate@5': 'higher-is-better',
'hitRate@10': 'higher-is-better',
'precision@1': 'higher-is-better',
'precision@3': 'higher-is-better',
'precision@5': 'higher-is-better',
'precision@10': 'higher-is-better',
'ndcg@1': 'higher-is-better',
'ndcg@3': 'higher-is-better',
'ndcg@5': 'higher-is-better',
'ndcg@10': 'higher-is-better',
mrr: 'higher-is-better',
emptyRateOnAnswerable: 'lower-is-better',
nonEmptyRateOnRefusal: 'lower-is-better',
// generation
correctness: 'higher-is-better',
refusalCorrectness: 'higher-is-better',
leakageRate: 'lower-is-better',
thinkTagLeakRate: 'lower-is-better',
markdownRate: 'higher-is-better',
groundedness: 'higher-is-better',
}
export type MetricDelta = {
metric: string
baseline: number | null
current: number | null
delta: number | null
direction: MetricDirection
/** Signed so that positive always means "better", whatever the direction. */
improvement: number | null
regressed: boolean
/** Present in one report but not the other. */
onlyIn?: 'baseline' | 'current'
}
export type ComparisonResult = {
comparable: boolean
/** Why the comparison was refused, when it was. */
incomparableReason?: string
tolerance: number
deltas: MetricDelta[]
regressions: MetricDelta[]
improvements: MetricDelta[]
}
/**
* Default tolerance band.
*
* Not zero, on purpose. Even at temperature 0 the generation tier moves a
* little between runs, and a gate that fires on 0.001 gets switched off within
* a week. 0.02 is roughly "one question in fifty" on a 99-question set — small
* enough to catch a real regression, large enough to ignore a coin flip.
*/
export const DEFAULT_TOLERANCE = 0.02
/**
* Slack for floating-point representation error at the tolerance boundary.
*
* Without it, `0.9 - 0.02` evaluates to a delta of -0.020000000000000018, which
* is "greater than the tolerance" by 1.8e-17 and would block a pull request. A
* gate that fires on the last bit of a double is a gate people learn to ignore.
*/
const BOUNDARY_EPSILON = 1e-9
export function compareReports(
baseline: EvalReport,
current: EvalReport,
tolerance: number = DEFAULT_TOLERANCE
): ComparisonResult {
if (baseline.meta.kind !== current.meta.kind) {
return incomparable(
`cannot compare a ${baseline.meta.kind} report against a ${current.meta.kind} report`,
tolerance
)
}
// The hard veto. Different corpora, chunk sizes, or embedding models mean the
// two numbers were produced by different experiments, and diffing them would
// manufacture a regression (or hide one) out of nothing.
if (baseline.meta.corpusFingerprint !== current.meta.corpusFingerprint) {
return incomparable(
`corpus fingerprint changed (${baseline.meta.corpusFingerprint} -> ${current.meta.corpusFingerprint}). ` +
'The corpus, chunk size, or embedding model differs, so these runs are not measuring the same thing. Re-baseline instead.',
tolerance
)
}
const names = [...new Set([...Object.keys(baseline.metrics), ...Object.keys(current.metrics)])].sort()
const deltas: MetricDelta[] = names.map((metric) => {
const b = baseline.metrics[metric] ?? null
const c = current.metrics[metric] ?? null
const direction = METRIC_DIRECTIONS[metric] ?? 'higher-is-better'
const onlyIn =
!(metric in baseline.metrics) ? ('current' as const)
: !(metric in current.metrics) ? ('baseline' as const)
: undefined
if (b === null || c === null) {
return { metric, baseline: b, current: c, delta: null, direction, improvement: null, regressed: false, onlyIn }
}
const delta = c - b
const improvement = direction === 'higher-is-better' ? delta : -delta
return {
metric,
baseline: b,
current: c,
delta,
direction,
improvement,
regressed: improvement < -(tolerance + BOUNDARY_EPSILON),
onlyIn,
}
})
return {
comparable: true,
tolerance,
deltas,
regressions: deltas.filter((d) => d.regressed),
improvements: deltas.filter(
(d) => d.improvement !== null && d.improvement > tolerance + BOUNDARY_EPSILON
),
}
}
function incomparable(reason: string, tolerance: number): ComparisonResult {
return {
comparable: false,
incomparableReason: reason,
tolerance,
deltas: [],
regressions: [],
improvements: [],
}
}
/** Flatten a `Record<number, number|null>` into `name@k` keys. */
export function flattenByK(
prefix: string,
values: Record<number, number | null>
): Record<string, number | null> {
return Object.fromEntries(Object.entries(values).map(([k, v]) => [`${prefix}@${k}`, v]))
}
const pct = (v: number | null) => (v === null ? 'n/a' : v.toFixed(4))
/** Human-readable summary, worst regressions first. */
export function renderComparisonMarkdown(
baselinePath: string,
currentPath: string,
result: ComparisonResult
): string {
const lines: string[] = ['# Eval comparison', '']
lines.push(`- baseline: \`${baselinePath}\``)
lines.push(`- current: \`${currentPath}\``)
lines.push(`- tolerance: ${result.tolerance}`)
lines.push('')
if (!result.comparable) {
lines.push('## Not comparable', '', result.incomparableReason ?? 'unknown reason', '')
return lines.join('\n')
}
lines.push(
result.regressions.length > 0
? `## ${result.regressions.length} regression(s)`
: '## No regressions'
)
lines.push('')
lines.push('| metric | baseline | current | delta | verdict |')
lines.push('|---|---:|---:|---:|---|')
// Worst first — the thing a developer needs is at the top, not sorted
// alphabetically halfway down a wall of unchanged rows.
const ordered = [...result.deltas].sort((a, b) => (a.improvement ?? 0) - (b.improvement ?? 0))
for (const d of ordered) {
const verdict =
d.onlyIn === 'current' ? 'new'
: d.onlyIn === 'baseline' ? 'removed'
: d.delta === null ? '—'
: d.regressed ? '**REGRESSED**'
: (d.improvement ?? 0) > result.tolerance ? 'improved'
: 'within tolerance'
const arrow = d.delta === null ? 'n/a' : `${d.delta >= 0 ? '+' : ''}${d.delta.toFixed(4)}`
lines.push(`| ${d.metric} | ${pct(d.baseline)} | ${pct(d.current)} | ${arrow} | ${verdict} |`)
}
lines.push('')
return lines.join('\n')
}

323
admin/app/utils/eval/retrieval_metrics.ts Normal file
View File

@ -0,0 +1,323 @@
/**
* Information-retrieval metrics for the RAG eval harness.
*
* Pure functions over ranked results — no I/O, no models, no clock. Everything
* here is deterministic, which is the point: a change in these numbers is
* unambiguously a change in the code, never in the weather.
*
* ## Two levels of measurement, and why both
*
* Retrieval returns *chunks*, but a golden answer lives in a *document*. The
* two questions we care about are different, so they are measured differently:
*
* - **Document level** (recall, MRR, nDCG): "did the answer's document make it
* into the context at all, and how near the top?" Chunks are collapsed to
* their document, keeping each document's best rank.
* - **Chunk level** (precision): "how much of what we injected is noise?" This
* is deliberately *not* deduped — five chunks from one irrelevant document
* cost five slots of a small model's context and should be counted five
* times.
*
* Reporting only one of these hides a real failure mode in the other.
*/
/** One retrieved chunk, reduced to what scoring needs. */
export type ScoredChunk = {
/** Corpus document this chunk came from; null if it could not be resolved. */
docId: string | null
/** The pipeline's final (post-rerank) score. */
score: number
/** The raw pre-rerank cosine score, when available. */
semanticScore?: number
}
export type RetrievalCase = {
id: string
tags: string[]
/** Ranked chunks, best first, exactly as the pipeline returned them. */
retrieved: ScoredChunk[]
/** Documents that genuinely answer the question. */
relevantDocIds: string[]
/** True when the right behaviour is to retrieve nothing useful. */
expectRefusal: boolean
}
/**
* Collapse a ranked chunk list to a ranked document list, keeping each
* document's best (earliest) position. Chunks with no resolvable document are
* dropped — they still occupy a rank in the chunk-level metrics, but they
* cannot be credited to any document.
*/
export function toDocumentRanking(retrieved: ScoredChunk[]): string[] {
const seen = new Set<string>()
const ranking: string[] = []
for (const chunk of retrieved) {
if (!chunk.docId || seen.has(chunk.docId)) continue
seen.add(chunk.docId)
ranking.push(chunk.docId)
}
return ranking
}
/**
* Fraction of the relevant documents that appear within the top `k` chunks.
*
* Returns null when there are no relevant documents (an out-of-corpus case) —
* recall is undefined there, and returning 0 would drag the mean down for
* questions that are *supposed* to retrieve nothing.
*/
export function recallAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null {
if (relevantDocIds.length === 0) return null
const relevant = new Set(relevantDocIds)
const found = new Set<string>()
for (const chunk of retrieved.slice(0, k)) {
if (chunk.docId && relevant.has(chunk.docId)) found.add(chunk.docId)
}
return found.size / relevant.size
}
/**
* 1 if *any* relevant document appears in the top `k`, else 0.
*
* Distinct from recall on multi-hop questions: finding one of two required
* documents is a hit but only 0.5 recall. Reporting both is what separates
* "found something" from "found enough to answer".
*/
export function hitRateAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null {
if (relevantDocIds.length === 0) return null
const relevant = new Set(relevantDocIds)
return retrieved.slice(0, k).some((c) => c.docId && relevant.has(c.docId)) ? 1 : 0
}
/**
* Fraction of the top `k` *chunks* that come from a relevant document.
*
* Not deduped, on purpose — this measures context pollution, and a small model
* drowning in four irrelevant chunks does not care that they share a source.
* Denominator is min(k, retrieved.length) so a pipeline returning 2 good chunks
* is not punished for the 3 it correctly declined to return.
*/
export function precisionAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null {
if (relevantDocIds.length === 0) return null
const window = retrieved.slice(0, k)
if (window.length === 0) return 0
const relevant = new Set(relevantDocIds)
const hits = window.filter((c) => c.docId && relevant.has(c.docId)).length
return hits / window.length
}
/**
* Reciprocal of the rank of the first relevant chunk (1-indexed); 0 if none.
* Averaged over cases this is MRR.
*/
export function reciprocalRank(retrieved: ScoredChunk[], relevantDocIds: string[]): number | null {
if (relevantDocIds.length === 0) return null
const relevant = new Set(relevantDocIds)
const idx = retrieved.findIndex((c) => c.docId && relevant.has(c.docId))
return idx === -1 ? 0 : 1 / (idx + 1)
}
/**
* Normalized discounted cumulative gain at `k`, over the **document** ranking,
* with binary gain.
*
* DCG@k = Σ_{i=1..k} rel_i / log2(i + 1)
* IDCG@k = Σ_{i=1..min(R, k)} 1 / log2(i + 1) where R = |relevant|
* nDCG@k = DCG@k / IDCG@k
*
* The ideal ranking is defined against the *known* number of relevant documents
* rather than against however many happened to be retrieved. That distinction
* matters: normalizing against the retrieved set would score a run that found
* one of three required documents, and ranked it first, as a perfect 1.0.
*
* This is the metric that catches "right documents, wrong order" — a reranking
* regression that leaves recall untouched while pushing the answer to position
* five, where a 1B model with a 2-result budget will never see it.
*/
export function ndcgAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null {
if (relevantDocIds.length === 0) return null
const relevant = new Set(relevantDocIds)
const docRanking = toDocumentRanking(retrieved).slice(0, k)
let dcg = 0
docRanking.forEach((docId, i) => {
if (relevant.has(docId)) dcg += 1 / Math.log2(i + 2) // i is 0-indexed, so rank = i + 1
})
let idcg = 0
for (let i = 0; i < Math.min(relevant.size, k); i++) {
idcg += 1 / Math.log2(i + 2)
}
return idcg === 0 ? 0 : dcg / idcg
}
/**
* Score separation: the gap between what relevant and irrelevant chunks score.
*
* This is how the similarity threshold gets calibrated with evidence instead of
* intuition. If the relevant p10 sits below the irrelevant p90, no threshold
* can cleanly separate them and the honest conclusion is that the *retriever*
* needs work, not the cutoff.
*/
export type ScoreDistribution = {
count: number
min: number
p10: number
median: number
p90: number
max: number
mean: number
}
export function describeScores(scores: number[]): ScoreDistribution | null {
if (scores.length === 0) return null
const sorted = [...scores].sort((a, b) => a - b)
const pct = (p: number) => {
// Nearest-rank percentile: no interpolation, so a reported value is always
// a value that actually occurred.
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))
return sorted[idx]
}
return {
count: sorted.length,
min: sorted[0],
p10: pct(10),
median: pct(50),
p90: pct(90),
max: sorted[sorted.length - 1],
mean: sorted.reduce((a, b) => a + b, 0) / sorted.length,
}
}
/** Mean over the entries that are actually defined; null if none are. */
export function meanOf(values: Array<number | null>): number | null {
const defined = values.filter((v): v is number => v !== null)
if (defined.length === 0) return null
return defined.reduce((a, b) => a + b, 0) / defined.length
}
export type RetrievalCaseResult = {
id: string
tags: string[]
retrievedCount: number
relevantDocIds: string[]
retrievedDocIds: string[]
expectRefusal: boolean
recall: Record<number, number | null>
hitRate: Record<number, number | null>
precision: Record<number, number | null>
ndcg: Record<number, number | null>
reciprocalRank: number | null
/** True when this case retrieved nothing at all. */
empty: boolean
}
export type RetrievalAggregate = {
cases: number
/** Cases with at least one relevant document (i.e. excluding refusal cases). */
answerable: number
recall: Record<number, number | null>
hitRate: Record<number, number | null>
precision: Record<number, number | null>
ndcg: Record<number, number | null>
mrr: number | null
/**
* Fraction of *answerable* questions that retrieved nothing. A non-zero value
* means the score threshold is filtering out real answers.
*/
emptyRateOnAnswerable: number | null
/**
* Fraction of *out-of-corpus* questions that retrieved something anyway. This
* is the other half of the threshold trade-off: context handed to the model
* for a question the corpus cannot answer is exactly what produces a
* confident, wrong reply.
*/
nonEmptyRateOnRefusal: number | null
relevantScores: ScoreDistribution | null
irrelevantScores: ScoreDistribution | null
}
export const DEFAULT_K_VALUES = [1, 3, 5, 10]
export function scoreCase(c: RetrievalCase, kValues: number[] = DEFAULT_K_VALUES): RetrievalCaseResult {
const byK = <T>(fn: (k: number) => T): Record<number, T> =>
Object.fromEntries(kValues.map((k) => [k, fn(k)]))
return {
id: c.id,
tags: c.tags,
retrievedCount: c.retrieved.length,
relevantDocIds: c.relevantDocIds,
retrievedDocIds: toDocumentRanking(c.retrieved),
expectRefusal: c.expectRefusal,
recall: byK((k) => recallAtK(c.retrieved, c.relevantDocIds, k)),
hitRate: byK((k) => hitRateAtK(c.retrieved, c.relevantDocIds, k)),
precision: byK((k) => precisionAtK(c.retrieved, c.relevantDocIds, k)),
ndcg: byK((k) => ndcgAtK(c.retrieved, c.relevantDocIds, k)),
reciprocalRank: reciprocalRank(c.retrieved, c.relevantDocIds),
empty: c.retrieved.length === 0,
}
}
export function aggregate(
cases: RetrievalCase[],
results: RetrievalCaseResult[],
kValues: number[] = DEFAULT_K_VALUES
): RetrievalAggregate {
const answerable = results.filter((r) => !r.expectRefusal)
const refusals = results.filter((r) => r.expectRefusal)
const byK = (pick: (r: RetrievalCaseResult, k: number) => number | null): Record<number, number | null> =>
Object.fromEntries(kValues.map((k) => [k, meanOf(results.map((r) => pick(r, k)))]))
// Split every retrieved chunk's score by whether its document was relevant.
// Refusal cases contribute only to the irrelevant side — by definition
// nothing they retrieve is relevant, and that is precisely the population a
// threshold needs to exclude.
const relevantScores: number[] = []
const irrelevantScores: number[] = []
for (const c of cases) {
const relevant = new Set(c.relevantDocIds)
for (const chunk of c.retrieved) {
const bucket = chunk.docId && relevant.has(chunk.docId) ? relevantScores : irrelevantScores
bucket.push(chunk.semanticScore ?? chunk.score)
}
}
return {
cases: results.length,
answerable: answerable.length,
recall: byK((r, k) => r.recall[k]),
hitRate: byK((r, k) => r.hitRate[k]),
precision: byK((r, k) => r.precision[k]),
ndcg: byK((r, k) => r.ndcg[k]),
mrr: meanOf(results.map((r) => r.reciprocalRank)),
emptyRateOnAnswerable:
answerable.length === 0 ? null : answerable.filter((r) => r.empty).length / answerable.length,
nonEmptyRateOnRefusal:
refusals.length === 0 ? null : refusals.filter((r) => !r.empty).length / refusals.length,
relevantScores: describeScores(relevantScores),
irrelevantScores: describeScores(irrelevantScores),
}
}
/** Aggregate restricted to cases carrying a given tag, for per-slice reporting. */
export function aggregateByTag(
cases: RetrievalCase[],
results: RetrievalCaseResult[],
kValues: number[] = DEFAULT_K_VALUES
): Record<string, RetrievalAggregate> {
const tags = new Set(results.flatMap((r) => r.tags))
const byId = new Map(cases.map((c) => [c.id, c]))
const out: Record<string, RetrievalAggregate> = {}
for (const tag of [...tags].sort()) {
const tagged = results.filter((r) => r.tags.includes(tag))
out[tag] = aggregate(
tagged.map((r) => byId.get(r.id)!).filter(Boolean),
tagged,
kValues
)
}
return out
}

123
admin/app/utils/rag_prompt.ts Normal file
View File

@ -0,0 +1,123 @@
/**
* Pure prompt-budgeting helpers for the chat RAG pipeline.
*
* These live here rather than in RagPipelineService so they can be exercised
* under bare `node --experimental-strip-types` with no MySQL, Redis, Qdrant, or
* Ollama — the same reason `kb_ratio_lookup.ts` is shaped this way. The service
* supplies the config; everything below is a function of its arguments.
*/
/**
* Chars-per-token estimate used when budgeting the *prompt* (context trimming
* and the num_ctx ladder).
*
* NOTE: RagService.CHAR_TO_TOKEN_RATIO is 2, not 3.5 — the two halves of the
* system disagree about what a token costs, and RagService's own doc-comment
* says 3. That divergence is real and known; reconciling it changes chunk size
* and therefore retrieval results, so it is deliberately left alone here and
* tracked as its own measured change rather than folded into a refactor.
*/
export const PROMPT_CHARS_PER_TOKEN = 3.5
/**
* num_ctx is only requested once the system prompt is large enough to risk
* overflowing Ollama's silent 2048 default. Below the trigger we send nothing
* and inherit the server default.
*/
export const NUM_CTX_TRIGGER_TOKENS = 3000
export const NUM_CTX_RESPONSE_HEADROOM = 2048
export const NUM_CTX_LADDER = [8192, 16384, 32768, 65536]
export type ContextLimits = { maxResults: number; maxTokens: number }
export type ContextLimitTier = { maxParams: number; maxResults: number; maxTokens: number }
/** The minimum shape these helpers need; the real chunks carry more. */
export type BudgetableChunk = { text: string; metadata?: Record<string, any> }
export type BudgetableMessage = { role: 'system' | 'user' | 'assistant'; content: string }
/**
* Determine RAG context limits from the model size encoded in its name.
* Parses size indicators like "1b", "3b", "8b", "70b".
*
* PRESERVED QUIRK: an unparseable model name is treated as 8B. That is a guess,
* and for a name like "phi3" or a custom tag it can hand a small model far more
* context than it can actually use. Faithful to the pre-extraction behaviour.
*/
export function getContextLimitsForModel(
modelName: string,
tiers: readonly ContextLimitTier[]
): ContextLimits {
// e.g. "llama3.2:3b", "qwen2.5:1.5b", "gemma:7b"
const sizeMatch = modelName.match(/(\d+\.?\d*)[bB]/)
const paramBillions = sizeMatch ? Number.parseFloat(sizeMatch[1]) : 8 // default to 8B if unknown
for (const tier of tiers) {
if (paramBillions <= tier.maxParams) {
return { maxResults: tier.maxResults, maxTokens: tier.maxTokens }
}
}
return { maxResults: 5, maxTokens: 0 }
}
/**
* Apply the model-size context budget: cap the result count, then cap total
* characters.
*
* The first (most relevant) result is always kept — the token cap only gates
* subsequent results, so a single oversized chunk never starves the model of
* context entirely.
*/
export function trimToContextBudget<T extends BudgetableChunk>(
docs: T[],
limits: ContextLimits
): T[] {
const byCount = docs.slice(0, limits.maxResults)
if (limits.maxTokens <= 0) return byCount
const charCap = limits.maxTokens * PROMPT_CHARS_PER_TOKEN
let totalChars = 0
return byCount.filter((doc, idx) => {
totalChars += doc.text.length
return idx === 0 || totalChars <= charCap
})
}
/**
* Render retrieved chunks into the block the rag_context prompt wraps.
*
* Each block is labelled with its source title when one is available — a
* neutral, honest provenance signal — but never with the raw relevance score.
* nomic cosine scores for genuinely relevant passages sit around 0.4-0.6, and
* surfacing e.g. "42%" primes the model to distrust correct context. Scores
* stay in the logs and in the eval report.
*/
export function buildContextBlock(docs: BudgetableChunk[]): string {
return docs
.map((doc, idx) => {
const title = doc.metadata?.full_title || doc.metadata?.article_title
const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]`
return `${label}\n${doc.text}`
})
.join('\n\n')
}
/**
* Request a context window big enough to hold the system messages, but only
* once they are large enough to be at risk.
*
* Ollama respects num_ctx per request; LM Studio ignores it gracefully. Below
* the trigger we send nothing and inherit the server default — which for Ollama
* is a silent 2048.
*/
export function deriveNumCtx(messages: BudgetableMessage[]): number | undefined {
const systemChars = messages
.filter((m) => m.role === 'system')
.reduce((sum, m) => sum + m.content.length, 0)
const estimatedSystemTokens = Math.ceil(systemChars / PROMPT_CHARS_PER_TOKEN)
if (estimatedSystemTokens <= NUM_CTX_TRIGGER_TOKENS) return undefined
const needed = estimatedSystemTokens + NUM_CTX_RESPONSE_HEADROOM
return NUM_CTX_LADDER.find((n) => n >= needed) ?? NUM_CTX_LADDER[NUM_CTX_LADDER.length - 1]
}

127
admin/commands/eval/compare.ts Normal file
View File

@ -0,0 +1,127 @@
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
/**
* Diff an eval report against a baseline and fail on regression.
*
* This is the "did my change help?" command, and the one CI would call.
*
* node ace eval:compare tests/eval/baselines/<fp>/retrieval.json tests/eval/reports/latest.json
* node ace eval:compare base.json current.json --tolerance=0.05
* node ace eval:compare --promote=retrieval tests/eval/reports/latest.json
*/
export default class EvalCompare extends BaseCommand {
static commandName = 'eval:compare'
static description = 'Diff an eval report against a baseline; exits non-zero on regression'
@args.string({ description: 'Baseline report JSON (or the report to promote, with --promote)' })
declare baseline: string
@args.string({ description: 'Current report JSON', required: false })
declare current: string
@flags.string({ description: 'Regression tolerance per metric (default: 0.02)' })
declare tolerance: string
@flags.string({
description: 'Promote the given report to a committed baseline under this name, then exit',
})
declare promote: string
@flags.boolean({ description: 'Print the full metric table, not just the regressions' })
declare verbose: boolean
/**
* Deliberately does not boot the application.
*
* Comparing two report files is pure I/O — it needs no MySQL, no Redis, no
* Qdrant, and no Ollama. Keeping it that way is what lets CI run the gate on
* a committed baseline without standing up the whole stack, and it avoids the
* app's noisy Redis teardown on exit.
*/
static options: CommandOptions = {
startApp: false,
}
async run() {
const { EvalReportService } = await import('#services/eval_report_service')
const { compareReports, DEFAULT_TOLERANCE } = await import('../../app/utils/eval/report.js')
const service = new EvalReportService()
try {
if (this.promote) {
const report = await service.read(this.baseline)
const path = await service.promoteToBaseline(report, this.promote)
this.logger.success(`Promoted to baseline: ${path}`)
this.logger.info(`Corpus fingerprint: ${report.meta.corpusFingerprint}`)
this.logger.info('Commit this file so the whole team gates against the same numbers.')
return
}
if (!this.current) {
this.logger.error('Two report paths are required (or use --promote=<name> with one).')
this.exitCode = 1
return
}
const [baseline, current] = await Promise.all([
service.read(this.baseline),
service.read(this.current),
])
const tolerance = this.tolerance ? Number.parseFloat(this.tolerance) : DEFAULT_TOLERANCE
const result = compareReports(baseline, current, tolerance)
if (!result.comparable) {
// Refusing is the correct outcome, not a failure of the tool — diffing
// runs from different corpora would manufacture a regression.
this.logger.error('Reports are not comparable:')
this.logger.error(` ${result.incomparableReason}`)
this.exitCode = 1
return
}
this.logger.info(
`baseline ${short(baseline.meta.gitSha)} (${baseline.meta.createdAt}) -> current ${short(current.meta.gitSha)} (${current.meta.createdAt})`
)
this.logger.info(`corpus ${current.meta.corpusFingerprint} · tolerance ${tolerance}`)
if (current.meta.gitDirty) {
this.logger.warning('Current report was produced from a dirty working tree.')
}
this.logger.info('')
const rows = this.verbose
? [...result.deltas].sort((a, b) => (a.improvement ?? 0) - (b.improvement ?? 0))
: [...result.regressions, ...result.improvements]
if (rows.length === 0) {
this.logger.success('No metric moved outside the tolerance band.')
} else {
for (const d of rows) {
const line = ` ${d.metric.padEnd(24)} ${fmt(d.baseline)} -> ${fmt(d.current)} ${signed(d.delta)}`
if (d.regressed) this.logger.error(`${line} REGRESSED`)
else if ((d.improvement ?? 0) > tolerance) this.logger.success(`${line} improved`)
else this.logger.info(line)
}
}
this.logger.info('')
if (result.regressions.length > 0) {
this.logger.error(
`${result.regressions.length} metric(s) regressed beyond the ${tolerance} tolerance.`
)
this.exitCode = 1
} else {
this.logger.success(
`No regressions. ${result.improvements.length} metric(s) improved beyond tolerance.`
)
}
} catch (error) {
this.logger.error(error instanceof Error ? error.message : String(error))
this.exitCode = 1
}
}
}
const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(4).padStart(6))
const signed = (v: number | null) => (v === null ? ' ' : `${v >= 0 ? '+' : ''}${v.toFixed(4)}`)
const short = (sha: string | null) => (sha ? sha.slice(0, 7) : 'unknown')

155
admin/commands/eval/corpus.ts Normal file
View File

@ -0,0 +1,155 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
/**
* Manage the frozen evaluation corpus.
*
* # Validate the corpus and goldens without touching the vector store
* node ace eval:corpus --check
*
* # Show what is currently ingested
* node ace eval:corpus --status
*
* # Wipe and rebuild (the normal path; safe — never touches user content)
* node ace eval:corpus --ingest
*
* # Remove the eval corpus entirely
* node ace eval:corpus --reset
*/
export default class EvalCorpus extends BaseCommand {
static commandName = 'eval:corpus'
static description = 'Manage the frozen RAG evaluation corpus (ingest, reset, status, check)'
@flags.boolean({ description: 'Wipe and re-ingest the corpus into the reserved eval collection' })
declare ingest: boolean
@flags.boolean({ description: 'Remove every eval corpus chunk from the vector store' })
declare reset: boolean
@flags.boolean({ description: 'Show the fingerprint and current chunk count' })
declare status: boolean
@flags.boolean({
description: 'Validate the corpus and goldens only — no Qdrant, no Ollama, no writes',
})
declare check: boolean
@flags.boolean({ description: 'Leave application debug logging on (very noisy)' })
declare debug: boolean
static options: CommandOptions = {
startApp: true,
}
async run() {
const { EvalCorpusService } = await import('#services/eval_corpus_service')
const { quietLogging } = await import('../../app/utils/eval/quiet.js')
const service = await this.app.container.make(EvalCorpusService)
const restoreLogging = quietLogging(this.debug)
try {
await this.dispatch(service)
} finally {
restoreLogging()
}
}
private async dispatch(service: any) {
// Default to --status so a bare invocation is always safe.
if (!this.ingest && !this.reset && !this.check) this.status = true
if (this.check) {
return this.runCheck(service)
}
if (this.reset) {
const removed = await service.reset()
this.logger.success(`Removed ${removed} eval corpus chunks`)
return
}
if (this.ingest) {
return this.runIngest(service)
}
const [fingerprint, chunks, goldens] = await Promise.all([
service.fingerprint(),
service.count(),
service.loadGoldens(),
])
this.logger.info(`Fingerprint: ${fingerprint}`)
this.logger.info(`Chunks in KB: ${chunks}`)
this.logger.info(`Goldens: ${goldens.length}`)
if (chunks === 0) {
this.logger.warning('Corpus is not ingested. Run: node ace eval:corpus --ingest')
}
}
/**
* Validation-only path. Deliberately avoids the container-resolved services'
* network dependencies so a contributor can check their golden edits without
* a running stack.
*/
private async runCheck(service: any) {
try {
const corpus = await service.loadCorpus()
const goldens = await service.loadGoldens()
const fingerprint = await service.fingerprint()
const tagCounts = new Map<string, number>()
for (const g of goldens) {
for (const tag of g.tags) tagCounts.set(tag, (tagCounts.get(tag) ?? 0) + 1)
}
// A document no golden asks about is dead weight in the corpus: it costs
// ingest time and adds distractor noise nobody chose deliberately.
const referenced = new Set<string>(goldens.flatMap((g: any) => g.relevantDocIds))
const unreferenced = corpus
.map((d: any) => d.docId)
.filter((id: string) => !referenced.has(id))
this.logger.success(`Corpus OK: ${corpus.length} documents, ${goldens.length} goldens`)
this.logger.info(`Fingerprint: ${fingerprint}`)
this.logger.info(
`Tags: ${[...tagCounts.entries()]
.sort((a, b) => b[1] - a[1])
.map(([t, n]) => `${t}=${n}`)
.join(' ')}`
)
const refusals = goldens.filter((g: any) => g.expectRefusal).length
this.logger.info(`Refusal cases: ${refusals} / ${goldens.length}`)
if (unreferenced.length > 0) {
this.logger.warning(`Documents no golden references: ${unreferenced.join(', ')}`)
}
} catch (error) {
this.logger.error(error instanceof Error ? error.message : String(error))
this.exitCode = 1
}
}
private async runIngest(service: any) {
this.logger.info('Rebuilding the eval corpus (user content is never touched)...')
// Validate before destroying anything — a typo in a golden should not cost
// you the ingest you were about to run.
await service.loadGoldens()
const summary = await service.ingest((docId: string, index: number, total: number) => {
this.logger.info(` [${index}/${total}] ${docId}`)
})
this.logger.info('')
if (summary.removedBeforeIngest > 0) {
this.logger.info(`Removed ${summary.removedBeforeIngest} chunks from the previous ingest`)
}
this.logger.success(`Ingested ${summary.documents} documents into ${summary.chunks} chunks`)
this.logger.info(`Fingerprint: ${summary.fingerprint}`)
if (summary.failures.length > 0) {
this.logger.error(`${summary.failures.length} document(s) failed to ingest:`)
for (const f of summary.failures) this.logger.error(` ${f.docId}: ${f.reason}`)
// A partial corpus produces scores that look real and are not.
this.logger.error('The corpus is incomplete — results from this state are not comparable.')
this.exitCode = 1
}
}
}

238
admin/commands/eval/generation.ts Normal file
View File

@ -0,0 +1,238 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import type {
GenerationAggregate,
GenerationMode,
} from '../../app/services/eval_generation_service.js'
const MODES: GenerationMode[] = ['oracle', 'e2e', 'noretrieval']
/**
* Score generated answers against the golden set.
*
* The `--mode` flag is the whole point of this command:
*
* oracle perfect context injected by construction -> isolates the MODEL
* e2e real retrieval -> the real product
* noretrieval no context at all -> parametric baseline
*
* Run all three on the same model and the ambiguous complaint "the AI gave a
* bad answer" decomposes into a number that can be acted on.
*
* node ace eval:generation --model=llama3.2:latest --mode=oracle
* node ace eval:generation --model=llama3:8b --all-modes
* node ace eval:generation --model=mock # no Ollama needed
*/
export default class EvalGeneration extends BaseCommand {
static commandName = 'eval:generation'
static description = 'Score generated answers, with oracle/e2e/noretrieval ablation'
@flags.string({ description: 'Ollama model to evaluate, or "mock" for the extractive ceiling' })
declare model: string
@flags.string({ description: `One of: ${MODES.join(', ')} (default: e2e)` })
declare mode: string
@flags.boolean({ description: 'Run all three modes and print the decomposition' })
declare allModes: boolean
@flags.string({ description: 'Repeats per question, for stability (default: 3)' })
declare repeats: string
@flags.string({ description: 'Only run goldens carrying this tag' })
declare tag: string
@flags.string({ description: 'Limit to the first N goldens (for a quick smoke run)' })
declare limit: string
@flags.boolean({ description: 'Print every failing question with the answer the model gave' })
declare verbose: boolean
@flags.boolean({ description: 'Leave application debug logging on (very noisy)' })
declare debug: boolean
static options: CommandOptions = {
startApp: true,
}
async run() {
const { EvalCorpusService } = await import('#services/eval_corpus_service')
const { EvalGenerationService, MOCK_MODEL } = await import('#services/eval_generation_service')
const { quietLogging } = await import('../../app/utils/eval/quiet.js')
const corpusService = await this.app.container.make(EvalCorpusService)
const generationService = await this.app.container.make(EvalGenerationService)
const restoreLogging = quietLogging(this.debug)
try {
if (!this.model) {
this.logger.error('--model is required (use --model=mock to run without Ollama)')
this.exitCode = 1
return
}
const modes: GenerationMode[] = this.allModes
? MODES
: [(this.mode as GenerationMode) || 'e2e']
for (const mode of modes) {
if (!MODES.includes(mode)) {
this.logger.error(`Unknown mode "${mode}". Expected one of: ${MODES.join(', ')}`)
this.exitCode = 1
return
}
}
let goldens = await corpusService.loadGoldens()
if (this.tag) goldens = goldens.filter((g) => g.tags.includes(this.tag))
if (this.limit) goldens = goldens.slice(0, Number.parseInt(this.limit, 10))
if (goldens.length === 0) {
this.logger.error('No goldens matched the given filters')
this.exitCode = 1
return
}
// e2e and oracle both need the corpus present; noretrieval does not, but
// requiring it uniformly keeps the three modes comparable.
const chunks = await corpusService.count()
if (chunks === 0 && modes.some((m) => m !== 'noretrieval')) {
this.logger.error('The eval corpus is not ingested. Run: node ace eval:corpus --ingest')
this.exitCode = 1
return
}
const fingerprint = await corpusService.fingerprint()
const repeats = this.repeats ? Number.parseInt(this.repeats, 10) : 3
this.logger.info(
`Corpus ${fingerprint} · ${goldens.length} goldens · model=${this.model} · repeats=${repeats}`
)
if (this.model === MOCK_MODEL) {
this.logger.info(
'Mock model: answers are the injected context verbatim — this is the extractive ceiling for the current retrieval, not a real model.'
)
}
this.logger.info('')
const summaries: Array<{ mode: GenerationMode; agg: GenerationAggregate }> = []
for (const mode of modes) {
const result = await generationService.run(goldens, {
mode,
model: this.model,
repeats,
onProgress: (id, index, total) => {
if (this.verbose) this.logger.info(` [${index}/${total}] ${mode}: ${id}`)
},
})
summaries.push({ mode, agg: result.overall })
this.printAggregate(mode, result.overall, result.elapsedMs)
if (result.overall.errors > 0) {
this.logger.error(
`${result.overall.errors} question(s) errored talking to the model — the scores above are incomplete.`
)
const firstError = result.cases.find((c) => c.error)
if (firstError) this.logger.error(` first error (${firstError.id}): ${firstError.error}`)
this.exitCode = 1
}
if (this.verbose) this.printFailures(result.cases)
}
if (summaries.length > 1) this.printDecomposition(summaries)
} catch (error) {
this.logger.error(error instanceof Error ? error.message : String(error))
this.exitCode = 1
} finally {
restoreLogging()
}
}
private printAggregate(mode: GenerationMode, agg: GenerationAggregate, elapsedMs: number) {
this.logger.info(`=== ${mode.toUpperCase()} (${(elapsedMs / 1000).toFixed(1)}s) ===`)
this.logger.info(` correctness ${fmt(agg.correctness)}`)
this.logger.info(` refusal correct ${fmt(agg.refusalCorrectness)}`)
this.logger.info(` leakage rate ${fmt(agg.leakageRate)} (lower is better)`)
this.logger.info(` think-tag leak ${fmt(agg.thinkTagLeakRate)} (should be 0)`)
this.logger.info(` markdown formatted ${fmt(agg.markdownRate)}`)
this.logger.info(
` numeric grounding ${agg.groundedness ? `${agg.groundedness.mean.toFixed(3)} (n=${agg.groundedness.n})` : ' n/a'}`
)
this.logger.info(
` mean answer length ${agg.meanAnswerLength === null ? 'n/a' : Math.round(agg.meanAnswerLength)} chars`
)
if (agg.unstable > 0) {
this.logger.warning(
` ${agg.unstable} question(s) flipped between repeats — excluded from gating, do not read them as a regression.`
)
}
this.logger.info('')
}
/**
* The reason this command exists: turn three scores into an attribution.
*/
private printDecomposition(summaries: Array<{ mode: GenerationMode; agg: GenerationAggregate }>) {
const get = (mode: GenerationMode) => summaries.find((s) => s.mode === mode)?.agg.correctness ?? null
const oracle = get('oracle')
const e2e = get('e2e')
const none = get('noretrieval')
this.logger.info('=== Decomposition ===')
if (oracle !== null && e2e !== null) {
const cost = oracle - e2e
this.logger.info(` retrieval cost ${cost.toFixed(3)} (oracle ${oracle.toFixed(3)} - e2e ${e2e.toFixed(3)})`)
this.logger.info(` model ceiling ${oracle.toFixed(3)} (what this model manages with perfect context)`)
}
if (e2e !== null && none !== null) {
this.logger.info(` RAG contribution ${(e2e - none).toFixed(3)} (e2e ${e2e.toFixed(3)} - noretrieval ${none.toFixed(3)})`)
}
if (oracle !== null && e2e !== null) {
// The actual triage rule, stated so nobody has to re-derive it.
const cost = oracle - e2e
if (oracle < 0.6) {
this.logger.warning(
' Oracle is low: this model struggles even with perfect context. Retrieval work will not fix it — try a larger model.'
)
} else if (cost > 0.15) {
this.logger.warning(
' Large retrieval cost: the model can use good context but is not being given it. Work on retrieval.'
)
} else {
this.logger.success(' Model and retrieval are both holding up on this corpus.')
}
}
if (none !== null && e2e !== null && e2e <= none) {
this.logger.warning(
' RAG is not adding anything over the bare model on this set — check that retrieval is actually reaching the prompt.'
)
}
}
private printFailures(cases: Array<any>) {
const failures = cases.filter((c) => c.correctness.passRate < 1 || c.refusalCorrectness.passRate < 1)
if (failures.length === 0) return
this.logger.info('--- failures ---')
for (const f of failures) {
this.logger.info(` ${f.id} correct=${f.correctness.passes}/${f.correctness.repeats} refusal=${f.refusalCorrectness.passes}/${f.refusalCorrectness.repeats}`)
const score = f.scores[0]
if (score?.missedRequired?.length) {
this.logger.info(` missed: ${score.missedRequired.join(' | ')}`)
}
if (score?.hitForbidden?.length) {
this.logger.info(` said forbidden: ${score.hitForbidden.join(' | ')}`)
}
if (score?.leakage?.length) {
this.logger.info(` narrated retrieval`)
}
if (f.answers[0]) {
const preview = f.answers[0].replace(/\s+/g, ' ').slice(0, 220)
this.logger.info(` said: ${preview}${f.answers[0].length > 220 ? '…' : ''}`)
}
}
this.logger.info('')
}
}
const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3))

178
admin/commands/eval/matrix.ts Normal file
View File

@ -0,0 +1,178 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import type { GenerationAggregate } from '../../app/services/eval_generation_service.js'
type Row = {
model: string
oracle: GenerationAggregate | null
e2e: GenerationAggregate | null
error?: string
}
/**
* Build the model capability table.
*
* This is the artifact that answers a GitHub issue. When a user reports "the
* AI gave me a bad answer", look up their model in this table:
*
* - scoring at or near its row -> the model is at its ceiling. The honest
* answer is "run a larger model", not a bug.
* - scoring well below its row -> something is wrong with their config or
* our code, and it is worth investigating.
*
* Without it, every quality concern is unfalsifiable.
*
* node ace eval:matrix --models=llama3.2:latest,llama3:8b --limit=30
* node ace eval:matrix --models=... --promote # commit as the reference table
*/
export default class EvalMatrix extends BaseCommand {
static commandName = 'eval:matrix'
static description = 'Score several models side by side to build the capability reference table'
@flags.string({ description: 'Comma-separated Ollama model names' })
declare models: string
@flags.string({ description: 'Repeats per question (default: 1 — the matrix is already long)' })
declare repeats: string
@flags.string({ description: 'Limit to the first N goldens' })
declare limit: string
@flags.string({ description: 'Only run goldens carrying this tag' })
declare tag: string
@flags.boolean({ description: 'Write the table to tests/eval/baselines/<fingerprint>/matrix.json' })
declare promote: boolean
@flags.boolean({ description: 'Leave application debug logging on (very noisy)' })
declare debug: boolean
static options: CommandOptions = {
startApp: true,
}
async run() {
const { EvalCorpusService } = await import('#services/eval_corpus_service')
const { EvalGenerationService } = await import('#services/eval_generation_service')
const { quietLogging } = await import('../../app/utils/eval/quiet.js')
const { mkdir, writeFile } = await import('node:fs/promises')
const { dirname, join, resolve } = await import('node:path')
const corpusService = await this.app.container.make(EvalCorpusService)
const generationService = await this.app.container.make(EvalGenerationService)
const restoreLogging = quietLogging(this.debug)
try {
if (!this.models) {
this.logger.error('--models is required, e.g. --models=llama3.2:latest,llama3:8b')
this.exitCode = 1
return
}
const models = this.models.split(',').map((m) => m.trim()).filter(Boolean)
let goldens = await corpusService.loadGoldens()
if (this.tag) goldens = goldens.filter((g) => g.tags.includes(this.tag))
if (this.limit) goldens = goldens.slice(0, Number.parseInt(this.limit, 10))
const fingerprint = await corpusService.fingerprint()
const repeats = this.repeats ? Number.parseInt(this.repeats, 10) : 1
this.logger.info(
`Corpus ${fingerprint} · ${goldens.length} goldens · ${models.length} models · repeats=${repeats}`
)
this.logger.info('Running oracle and e2e for each model. This takes a while.')
this.logger.info('')
const rows: Row[] = []
for (const model of models) {
this.logger.info(`--- ${model} ---`)
const row: Row = { model, oracle: null, e2e: null }
try {
// Only oracle and e2e: those two are what the triage rule needs, and
// adding noretrieval would half again the runtime of an already long
// command for a number that does not change the verdict.
for (const mode of ['oracle', 'e2e'] as const) {
const started = Date.now()
const result = await generationService.run(goldens, { mode, model, repeats })
row[mode] = result.overall
this.logger.info(
` ${mode.padEnd(6)} correctness=${fmt(result.overall.correctness)} refusal=${fmt(result.overall.refusalCorrectness)} (${((Date.now() - started) / 1000).toFixed(0)}s)`
)
if (result.overall.errors > 0) {
this.logger.warning(` ${result.overall.errors} question(s) errored in ${mode}`)
}
}
} catch (error) {
row.error = error instanceof Error ? error.message : String(error)
this.logger.error(` failed: ${row.error}`)
}
rows.push(row)
}
this.logger.info('')
this.printTable(rows)
if (this.promote) {
const path = resolve(join(process.cwd(), 'tests/eval/baselines', fingerprint, 'matrix.json'))
await mkdir(dirname(path), { recursive: true })
await writeFile(
path,
JSON.stringify(
{
corpusFingerprint: fingerprint,
createdAt: new Date().toISOString(),
goldens: goldens.length,
repeats,
tag: this.tag ?? null,
rows,
},
null,
2
)
)
this.logger.success(`Capability table written: ${path}`)
this.logger.info('Commit it — this is the reference a support triage compares against.')
}
} catch (error) {
this.logger.error(error instanceof Error ? error.message : String(error))
this.exitCode = 1
} finally {
restoreLogging()
}
}
private printTable(rows: Row[]) {
this.logger.info('=== Capability matrix ===')
this.logger.info('')
const width = Math.max(20, ...rows.map((r) => r.model.length + 2))
this.logger.info(
`${'model'.padEnd(width)}${'oracle'.padStart(8)}${'e2e'.padStart(8)}${'ret.cost'.padStart(10)}${'refusal'.padStart(9)}${'leakage'.padStart(9)}${'ground'.padStart(8)}`
)
for (const row of rows) {
if (row.error) {
this.logger.info(`${row.model.padEnd(width)} ERROR: ${row.error}`)
continue
}
const cost =
row.oracle?.correctness !== null && row.oracle?.correctness !== undefined &&
row.e2e?.correctness !== null && row.e2e?.correctness !== undefined
? row.oracle.correctness - row.e2e.correctness
: null
this.logger.info(
row.model.padEnd(width) +
fmt(row.oracle?.correctness ?? null).padStart(8) +
fmt(row.e2e?.correctness ?? null).padStart(8) +
(cost === null ? ' n/a' : `${cost >= 0 ? '+' : ''}${cost.toFixed(3)}`).padStart(10) +
fmt(row.e2e?.refusalCorrectness ?? null).padStart(9) +
fmt(row.e2e?.leakageRate ?? null).padStart(9) +
fmt(row.e2e?.groundedness?.mean ?? null).padStart(8)
)
}
this.logger.info('')
this.logger.info('oracle = correctness with perfect context (the model\'s ceiling)')
this.logger.info('e2e = correctness with real retrieval (what a user gets)')
this.logger.info('ret.cost = oracle - e2e (what imperfect retrieval costs this model)')
this.logger.info('leakage = rate of narrating retrieval; lower is better')
}
}
const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3))

276
admin/commands/eval/retrieval.ts Normal file
View File

@ -0,0 +1,276 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import type { RetrievalAggregate } from '../../app/utils/eval/retrieval_metrics.js'
/**
* Score NOMAD's retrieval against the frozen golden set.
*
* No chat model is involved, so this is deterministic and hardware-independent:
* a movement in these numbers is a code change, not a slow machine or an unlucky
* sample. It is the fast inner loop for anything touching chunking, embedding,
* thresholds, or reranking.
*
* node ace eval:retrieval
* node ace eval:retrieval --ablate # is the reranker helping?
* node ace eval:retrieval --threshold=0.5 # sweep the cutoff
* node ace eval:retrieval --tag=multi-hop # one slice only
*/
export default class EvalRetrieval extends BaseCommand {
static commandName = 'eval:retrieval'
static description = 'Score RAG retrieval against the golden set (deterministic, no chat model)'
@flags.string({ description: 'Chunks to retrieve per query (default: the production value)' })
declare topK: string
@flags.string({ description: 'Minimum similarity score (default: the production value)' })
declare threshold: string
@flags.boolean({ description: 'Also score the raw dense, reranked, and diversified orderings' })
declare ablate: boolean
@flags.string({ description: 'Only run goldens carrying this tag' })
declare tag: string
@flags.boolean({ description: 'Print each failing question and what it retrieved' })
declare verbose: boolean
@flags.boolean({ description: 'Leave application debug logging on (very noisy)' })
declare debug: boolean
@flags.boolean({ description: 'Write a JSON + Markdown report to tests/eval/reports/' })
declare report: boolean
static options: CommandOptions = {
startApp: true,
}
async run() {
const { EvalCorpusService } = await import('#services/eval_corpus_service')
const { EvalRetrievalService } = await import('#services/eval_retrieval_service')
const { EvalReportService } = await import('#services/eval_report_service')
const { quietLogging } = await import('../../app/utils/eval/quiet.js')
const corpusService = await this.app.container.make(EvalCorpusService)
const retrievalService = await this.app.container.make(EvalRetrievalService)
const reportService = await this.app.container.make(EvalReportService)
const restoreLogging = quietLogging(this.debug)
try {
const chunks = await retrievalService.assertCorpusReady()
const fingerprint = await corpusService.fingerprint()
let goldens = await corpusService.loadGoldens()
if (this.tag) {
goldens = goldens.filter((g) => g.tags.includes(this.tag))
if (goldens.length === 0) {
this.logger.error(`No goldens carry the tag "${this.tag}"`)
this.exitCode = 1
return
}
}
this.logger.info(`Corpus ${fingerprint} · ${chunks} chunks · ${goldens.length} goldens`)
this.logger.info('')
const started = Date.now()
const result = await retrievalService.run(goldens, {
topK: this.topK ? Number.parseInt(this.topK, 10) : undefined,
scoreThreshold: this.threshold ? Number.parseFloat(this.threshold) : undefined,
ablate: this.ablate,
})
const elapsed = ((Date.now() - started) / 1000).toFixed(1)
this.logger.info(
`Params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} (${elapsed}s)`
)
this.logger.info('')
this.printAggregate('OVERALL', result.overall, result.params.kValues)
if (result.unresolvedChunks > 0) {
// Every retrieved chunk should belong to the eval corpus. Anything else
// means the collection filter leaked and the numbers above describe a
// corpus nobody chose.
this.logger.error(
`${result.unresolvedChunks} retrieved chunk(s) did not belong to the eval corpus — the collection filter leaked.`
)
this.exitCode = 1
}
this.printThresholdGuidance(result.overall)
if (result.ablation) {
this.logger.info('')
this.logger.info('=== Stage ablation (does each heuristic earn its place?) ===')
const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
const row = (name: string, agg: RetrievalAggregate) =>
this.logger.info(
` ${name.padEnd(14)} recall@${k}=${fmt(agg.recall[k])} ndcg@${k}=${fmt(agg.ndcg[k])} mrr=${fmt(agg.mrr)} prec@${k}=${fmt(agg.precision[k])}`
)
row('dense only', result.ablation.dense)
row('+ rerank', result.ablation.reranked)
row('+ diversity', result.ablation.diversified)
this.explainAblation(result.ablation, k)
}
this.logger.info('')
this.logger.info('=== By tag ===')
for (const [tag, agg] of Object.entries(result.byTag).sort()) {
const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
this.logger.info(
` ${tag.padEnd(20)} n=${String(agg.cases).padStart(3)} recall@${k}=${fmt(agg.recall[k])} ndcg@${k}=${fmt(agg.ndcg[k])}`
)
}
const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
const misses = result.cases.filter((c) => !c.expectRefusal && (c.recall[k] ?? 1) < 1)
this.logger.info('')
this.logger.info(`${misses.length} of ${result.overall.answerable} answerable questions missed at k=${k}`)
if (this.verbose && misses.length > 0) {
this.logger.info('')
this.logger.info('=== Misses ===')
for (const miss of misses) {
this.logger.info(` ${miss.id}`)
this.logger.info(` wanted: ${miss.relevantDocIds.join(', ') || '(none)'}`)
this.logger.info(` retrieved: ${miss.retrievedDocIds.join(', ') || '(nothing)'}`)
}
} else if (misses.length > 0) {
this.logger.info('Re-run with --verbose to see which questions and what they retrieved.')
}
if (this.report) {
const meta = await reportService.buildMeta('retrieval', fingerprint, {
...result.params,
tag: this.tag ?? null,
})
const doc = reportService.fromRetrieval(meta, result)
const slug = `retrieval-${meta.createdAt.replace(/[:.]/g, '-')}`
const path = await reportService.write(doc, slug, renderRetrievalMarkdown(doc, result, misses))
this.logger.info('')
this.logger.success(`Report written: ${path}`)
}
} catch (error) {
this.logger.error(error instanceof Error ? error.message : String(error))
this.exitCode = 1
} finally {
restoreLogging()
}
}
private printAggregate(label: string, agg: RetrievalAggregate, kValues: number[]) {
this.logger.info(`=== ${label} (${agg.answerable} answerable of ${agg.cases}) ===`)
const row = (name: string, values: Record<number, number | null>) =>
this.logger.info(
` ${name.padEnd(10)} ${kValues.map((k) => `@${k}=${fmt(values[k])}`).join(' ')}`
)
row('recall', agg.recall)
row('hit rate', agg.hitRate)
row('precision', agg.precision)
row('ndcg', agg.ndcg)
this.logger.info(` mrr ${fmt(agg.mrr)}`)
}
/**
* Turn the score distributions into an actual recommendation. The raw
* percentiles are the evidence; this is the reading of them, which is what
* the threshold constants have never had.
*/
private printThresholdGuidance(agg: RetrievalAggregate) {
const rel = agg.relevantScores
const irr = agg.irrelevantScores
this.logger.info('')
this.logger.info('=== Score separation (how to calibrate the threshold) ===')
if (rel) {
this.logger.info(
` relevant chunks n=${rel.count} min=${rel.min.toFixed(3)} p10=${rel.p10.toFixed(3)} median=${rel.median.toFixed(3)} p90=${rel.p90.toFixed(3)}`
)
}
if (irr) {
this.logger.info(
` irrelevant chunks n=${irr.count} min=${irr.min.toFixed(3)} p10=${irr.p10.toFixed(3)} median=${irr.median.toFixed(3)} p90=${irr.p90.toFixed(3)}`
)
}
if (rel && irr) {
if (rel.p10 > irr.p90) {
this.logger.success(
` Clean separation: a threshold between ${irr.p90.toFixed(3)} and ${rel.p10.toFixed(3)} splits them.`
)
} else {
this.logger.warning(
` Overlapping: relevant p10 (${rel.p10.toFixed(3)}) sits below irrelevant p90 (${irr.p90.toFixed(3)}).`
)
this.logger.warning(
' No cutoff separates these cleanly — the retriever needs work, not the threshold.'
)
}
}
if (agg.emptyRateOnAnswerable !== null && agg.emptyRateOnAnswerable > 0) {
this.logger.warning(
` ${pct(agg.emptyRateOnAnswerable)} of answerable questions retrieved nothing — threshold may be too high.`
)
}
if (agg.nonEmptyRateOnRefusal !== null && agg.nonEmptyRateOnRefusal > 0) {
this.logger.warning(
` ${pct(agg.nonEmptyRateOnRefusal)} of out-of-corpus questions retrieved something anyway — that context is what produces confident wrong answers.`
)
}
}
private explainAblation(ablation: { dense: RetrievalAggregate; reranked: RetrievalAggregate; diversified: RetrievalAggregate }, k: number) {
const verdict = (name: string, before: number | null, after: number | null) => {
if (before === null || after === null) return
const delta = after - before
if (Math.abs(delta) < 1e-6) {
this.logger.warning(` ${name} changed nothing at k=${k} — it is complexity with no measured benefit.`)
} else if (delta < 0) {
this.logger.warning(` ${name} made ndcg@${k} worse by ${Math.abs(delta).toFixed(4)}.`)
} else {
this.logger.success(` ${name} improved ndcg@${k} by ${delta.toFixed(4)}.`)
}
}
verdict('Reranking', ablation.dense.ndcg[k], ablation.reranked.ndcg[k])
verdict('Source diversity', ablation.reranked.ndcg[k], ablation.diversified.ndcg[k])
}
}
const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3))
const pct = (v: number) => `${(v * 100).toFixed(0)}%`
/**
* The human-readable half of a report. Leads with the misses, because when a
* number moves the next question is always "which questions?" and a table of
* aggregates cannot answer it.
*/
function renderRetrievalMarkdown(doc: any, result: any, misses: any[]): string {
const lines: string[] = ['# Retrieval eval', '']
lines.push(`- corpus: \`${doc.meta.corpusFingerprint}\``)
lines.push(`- commit: \`${doc.meta.gitSha ?? 'unknown'}\`${doc.meta.gitDirty ? ' (dirty tree)' : ''}`)
lines.push(`- when: ${doc.meta.createdAt}`)
lines.push(`- params: topK=${result.params.topK} threshold=${result.params.scoreThreshold}`)
lines.push('')
lines.push('## Metrics', '')
lines.push('| metric | value |', '|---|---:|')
for (const [name, value] of Object.entries(doc.metrics)) {
lines.push(`| ${name} | ${value === null ? 'n/a' : (value as number).toFixed(4)} |`)
}
lines.push('')
lines.push(`## Misses (${misses.length})`, '')
if (misses.length === 0) {
lines.push('None.')
} else {
for (const m of misses) {
lines.push(`### \`${m.id}\``)
lines.push(`- wanted: ${m.relevantDocIds.join(', ') || '_none_'}`)
lines.push(`- retrieved: ${m.retrievedDocIds.join(', ') || '_nothing_'}`)
lines.push('')
}
}
lines.push('## By tag', '')
lines.push('| tag | recall@5 | ndcg@5 |', '|---|---:|---:|')
for (const [tag, metrics] of Object.entries(doc.byTag).sort()) {
const m = metrics as Record<string, number | null>
const cell = (v: number | null | undefined) => (v === null || v === undefined ? 'n/a' : v.toFixed(4))
lines.push(`| ${tag} | ${cell(m['recall@5'])} | ${cell(m['ndcg@5'])} |`)
}
return lines.join('\n') + '\n'
}

19
admin/constants/kb_collections.ts
View File

@ -21,6 +21,21 @@ export const KB_COLLECTIONS = [
export type KbCollection = (typeof KB_COLLECTIONS)[number]
/**
* Reserved collection tag for the developer evaluation corpus (`ace eval:*`).
*
* The eval fixtures live in the same Qdrant collection as everything else —
* NOMAD "collections" are a payload tag, not separate Qdrant collections — so
* this tag is what keeps them out of the user's Knowledge Base UI and out of
* ordinary chat retrieval. Every read path that enumerates user content filters
* it out, and `sanitizeCollectionName` refuses to mint it, so a user cannot
* create a colliding tag by accident.
*
* The leading/trailing underscores are deliberate: they make the tag obviously
* internal if it ever does surface in a log or a raw Qdrant query.
*/
export const KB_EVAL_COLLECTION = '__nomad_eval__'
/** Hard cap on a user-created tag's length, enforced client- and server-side. */
export const KB_COLLECTION_NAME_MAX_LENGTH = 40
@ -36,5 +51,9 @@ export function sanitizeCollectionName(raw: string | null | undefined): string |
if (!raw) return null
const trimmed = raw.trim().toLowerCase()
if (!trimmed) return null
// The eval tag is reserved. Treating a collision as "uncategorized" rather
// than throwing keeps this a pure normalizer, and the user's documents stay
// visible in the KB instead of vanishing into a hidden internal collection.
if (trimmed === KB_EVAL_COLLECTION) return null
return trimmed.slice(0, KB_COLLECTION_NAME_MAX_LENGTH)
}

9
admin/constants/ollama.ts
View File

@ -76,6 +76,15 @@ export const RAG_CONTEXT_LIMITS: { maxParams: number; maxResults: number; maxTok
{ maxParams: Infinity, maxResults: 5, maxTokens: 0 }, // 13B+ (no cap)
]
/**
* Retrieval defaults for the chat pipeline. These were previously inline
* literals at the `searchSimilarDocuments` call site in OllamaController, which
* made them impossible to sweep or record in a report. They are named here so
* the pipeline and the eval harness read the same numbers.
*/
export const RAG_DEFAULT_TOP_K = 5
export const RAG_DEFAULT_SCORE_THRESHOLD = 0.3
export const SYSTEM_PROMPTS = {
default: `
Format all responses using markdown for better readability. Vanilla markdown or GitHub-flavored markdown is preferred.

2
admin/package.json
View File

@ -10,6 +10,8 @@
"build": "node ace build",
"dev": "node ace serve --hmr",
"test": "node ace test",
"test:unit": "node --import ts-node-maintained/register/esm --test \"tests/unit/**/*.spec.ts\"",
"test:eval": "node --import ts-node-maintained/register/esm --test \"tests/unit/eval_*.spec.ts\" \"tests/unit/rag_pipeline_*.spec.ts\"",
"lint": "eslint .",
"format": "prettier --write .",
"gen:curated-data": "node --experimental-strip-types scripts/generate_curated_data.ts && prettier --write app/data/conditions.ts app/data/natural_remedies.ts app/data/home_remedies.ts",

352
admin/tests/eval/README.md Normal file
View File

@ -0,0 +1,352 @@
# NOMAD AI Quality Harness
A reproducible way to measure whether NOMAD's RAG pipeline is getting better or
worse, and to tell a **code regression** apart from **a small model on modest
hardware being asked too much**.
This is a developer tool. There is no UI, nothing is user-facing, and none of it
runs in production.
> **Throughput is out of scope.** tokens/sec, time-to-first-token, and the NOMAD
> Score belong to `node ace benchmark:run`. Nothing here is comparable across
> machines and nothing here should ever be, because every developer and every
> user has different hardware. This measures *quality* only.
---
## The one-minute version
```bash
node ace eval:corpus --ingest # once, and after any corpus edit
node ace eval:retrieval --ablate # seconds, no chat model, deterministic
node ace eval:generation --model=<model> --all-modes # minutes; answers "code or model?"
```
---
## Why the three modes matter
`eval:generation --all-modes` runs every question three ways. That is what turns
"the AI gave a bad answer" into something you can act on:
| Mode | Context the model gets | What a low score means |
|---|---|---|
| `oracle` | The golden's own documents, injected verbatim | **The model.** Retrieval was perfect by construction, so this is the model's ceiling with this prompt. |
| `e2e` | Whatever real retrieval found | The actual product experience. |
| `noretrieval` | Nothing | The model's parametric baseline — what it knows without NOMAD. |
Read the decomposition it prints:
- **`oracle` is low** → the model cannot use good context. No amount of retrieval
work will fix it. The honest answer to the user is "run a larger model."
- **`oracle - e2e` is large** → the model *can* use good context but is not being
given it. This is a retrieval bug, and it is ours.
- **`e2e - noretrieval` is near zero** → RAG is not contributing. Check that
retrieval is actually reaching the prompt.
There is a fourth reference line, `--model=mock`, which needs no Ollama at all.
It answers by echoing the injected context, so it is the **extractive ceiling**:
the best a perfect model could do given the current retrieval. A real model
below the mock line is the bottleneck; a mock line that is itself low means
retrieval is.
---
## Commands
### `eval:corpus`
```bash
node ace eval:corpus --check # validate corpus + goldens, no services needed
node ace eval:corpus --status # fingerprint and current chunk count
node ace eval:corpus --ingest # wipe and rebuild (always a full rebuild)
node ace eval:corpus --reset # remove every eval chunk
```
Ingest goes through NOMAD's **real** `RagService.embedAndStoreText`, so chunk
size, the token-estimate ratio, the `search_document:` prefix, and the embedding
model are all inside the measurement. Change any of them and the score moves —
which is the point.
### Isolation: how the eval corpus stays out of your knowledge base
**The eval corpus is not a separate Qdrant collection.** It lives in the same
`nomad_knowledge_base` collection as your real documents, tagged with the
reserved payload value `collection: __nomad_eval__`. That is not a shortcut —
NOMAD "collections" *are* payload tags rather than separate Qdrant collections,
so using the tag means the harness exercises the same filter path production
chat uses, with no production code changed to accommodate it.
Three separate mechanisms keep the two apart:
**1. Writes are scoped.** Ingest only ever adds points carrying the eval tag.
`--reset` deletes by that filter and nothing else. Your documents are never
written, re-tagged, or deleted. (On the machine this was built on: 306 points
before ingest, 335 after — exactly the 29 eval chunks, nothing else moved.)
**2. Reads are filtered server-side.** Every eval query passes
`must: [{ key: 'collection', match: { value: '__nomad_eval__' } }]`, and Qdrant
applies it during search against a keyword payload index, so your documents
never compete for a result slot. Verified directly against a knowledge base
containing unrelated content — the same query returns a real user document at
similarity **0.83** unfiltered, and only eval documents at ~0.50 filtered:
```
unfiltered: 0.831 evolution_of_steam_locomotive.txt
filtered: 0.522 water-river-song.md [eval]
```
**3. A leak would fail the run, loudly.** Every retrieved chunk's `source` is
resolved back to a corpus document by path. Anything outside
`tests/eval/corpus/` is counted as an unresolved chunk, and a non-zero count
prints an error and exits 1 rather than quietly reporting a score. The check is
on the resolved path, not the file extension — NOMAD embeds its own
`admin/docs/*.md` into the knowledge base on first run, so an extension check
would have accepted a leaked `faq.md` as the plausible document id "faq".
See `tests/unit/eval_source_guard.spec.ts`.
**The one thing the tag does not isolate** is the *physical* collection: your
documents and the eval fixtures share an HNSW index. That has no effect on
correctness (the filter is applied during search) and results were verified
byte-identical across runs, but if you want true physical separation the change
is to thread a collection name through `_ensureCollection`,
`embedAndStoreText`, and `searchSimilarDocuments`. That was deliberately not
done, because it means touching three production methods for a test-only
benefit that the payload filter already delivers.
### `eval:retrieval`
```bash
node ace eval:retrieval
node ace eval:retrieval --ablate # is the reranker earning its complexity?
node ace eval:retrieval --threshold=0.5 # sweep the cutoff
node ace eval:retrieval --tag=multi-hop
node ace eval:retrieval --verbose # show every miss and what it retrieved
node ace eval:retrieval --report # write JSON + Markdown to reports/
```
Embedding is the only model call, and its output is stable, so **this tier is
deterministic and hardware-independent**. Two runs produce byte-identical
numbers. A movement here is a code change, full stop — which makes it the only
tier worth gating CI on.
Multi-turn goldens are scored on their raw final message, because resolving the
coreference needs the chat model and would make the tier non-deterministic. That
bucket therefore reports the honest floor; the rewrite's contribution shows up in
the generation tier instead.
### `eval:generation`
```bash
node ace eval:generation --model=mock # no Ollama required
node ace eval:generation --model=llama3.2:latest --all-modes
node ace eval:generation --model=llama3:8b --repeats=5 --verbose
node ace eval:generation --model=... --tag=out-of-corpus
```
Runs at `temperature: 0` with a fixed seed. That reduces variance but does not
eliminate it — llama.cpp batching and GPU scheduling still move outputs — so
`--repeats` defaults to 3 and any question whose pass/fail flips across repeats
is reported as **unstable** and excluded from gating. Do not read an unstable
question as a regression.
Before scoring, the harness evicts other resident models and burns one throwaway
generation, borrowed from `BenchmarkService` for the same reason it added them:
a cold first run behaves differently.
### `eval:matrix`
```bash
node ace eval:matrix --models=qwen2.5:0.5b,llama3.2:latest,llama3:8b --limit=25
node ace eval:matrix --models=... --promote
```
Produces the **capability table** — the artifact that answers a support ticket.
When a user reports a bad answer, look up their model:
- scoring at or near its row → the model is at its ceiling, not a bug
- scoring well below its row → their config or our code, worth investigating
### `eval:compare`
```bash
node ace eval:retrieval --report
node ace eval:compare <report.json> --promote=retrieval # set the baseline
node ace eval:compare tests/eval/baselines/<fp>/retrieval.json <new-report.json>
```
Exits non-zero when any metric regresses beyond the tolerance (default `0.02`).
It boots no services, so CI can run it against a committed baseline without
standing up MySQL, Redis, Qdrant, or Ollama.
**It refuses to compare reports with different corpus fingerprints.** That is
correct behaviour, not a limitation: if the corpus, chunk size, or embedding
model changed, the two runs measured different things and diffing them would
manufacture a regression. Re-baseline instead.
---
## The corpus and the goldens
- `corpus/*.md` — 28 short documents across NOMAD's real domains.
- `goldens/*.jsonl` — 99 questions, one JSON object per line.
The corpus is built with deliberate traps, not just easy questions:
| Tag | What it tests |
|---|---|
| `distractor` | `water-river-song.md` is a poem that shares vocabulary with real water questions and answers none of them. This is exactly the "poetic, tangential passage" failure `SYSTEM_PROMPTS.rag_context` rule 1 defends against. |
| `near-miss` | Water-bath vs pressure canning: two documents that look alike and give opposite advice. |
| `out-of-corpus` | Questions the corpus genuinely cannot answer, including adversarial ones about topics the corpus *partly* covers (the TR-88's warranty). The right answer is to decline. |
| `fictional` | The Thornfield protocol and the TR-88 pump do not exist. No model can know them, so a correct answer proves retrieval worked rather than that the model memorised the internet. |
| `acronym` / `acronym-control` | The same question asked with an acronym and spelled out, to measure what `preprocessQuery`'s 28-entry glossary expansion actually buys. |
| `chunk-boundary` | Facts buried late in the one long document, which is the only one that splits into multiple chunks. |
| `multi-hop` | Answers requiring two documents (elevation table + boiling times). |
| `multi-turn` | A pronoun in the second turn, the only thing that exercises `rewriteQueryWithContext`. |
### Golden format
```jsonc
{
"id": "water-boil-altitude-01",
"query": "How long do I need to boil water at high altitude?",
"turns": [], // prior messages for multi-turn cases
"relevantDocIds": ["water-boiling"], // corpus filenames without .md
"mustInclude": ["\\b(3|three) minutes?"], // case-insensitive REGEX
"mustNotInclude": ["distill"],
"expectRefusal": false, // true for out-of-corpus
"tags": ["single-hop", "numeric"]
}
```
`mustInclude` and `mustNotInclude` entries are **regular expressions**, so one
entry can accept "3 minutes" or "three minutes" without inflating the list.
Every pattern is compiled at load, so a bad regex fails immediately rather than
on the one run where it finally matters.
Validation is strict on purpose. A golden that lists a document not in the
corpus, or that expects a refusal while also naming relevant documents, is
rejected at load — those mistakes are otherwise invisible and just quietly lower
the score forever.
### Editing the corpus
Any edit changes the fingerprint, which invalidates every existing baseline.
That is deliberate. After editing:
```bash
node ace eval:corpus --check # validate first
node ace eval:corpus --ingest # rebuild
node ace eval:retrieval --report
node ace eval:compare <report.json> --promote=retrieval
```
Note the safety facts in the corpus are real. If you add documents, keep any
health, water, or food-safety content accurate — invent only clearly-fictional
non-safety things (place names, equipment model numbers) when you need something
unguessable.
---
## What the metrics mean
**Retrieval** — measured at two levels, because they answer different questions.
Document level (recall, hit rate, MRR, nDCG) collapses chunks to their source
document: *did the answer's document reach the context?* Chunk level (precision)
does not dedupe: *how much of what we injected is noise?* — five chunks from one
irrelevant document cost a small model five slots.
`nDCG` normalizes against the *known* number of relevant documents, not against
whatever was retrieved. It is the metric that catches "right documents, wrong
order", a reranking regression that leaves recall untouched while pushing the
answer to position five where a 1B model's 2-result budget will never see it.
The implementation is cross-checked against `pytrec_eval` (TREC's reference
implementation) — see `tests/unit/eval_retrieval_metrics.spec.ts`.
**Generation** — all deterministic, no judge model required:
| Metric | What it catches |
|---|---|
| `correctness` | The `mustInclude` / `mustNotInclude` assertions. |
| `refusalCorrectness` | Declining out-of-corpus questions *and* not hedging on answerable ones. This is the "Sorry, I wasn't able to find specific context" symptom, measured. |
| `leakageRate` | Narrating retrieval ("according to Context 1", "the knowledge base"), which `rag_context` rule 4 forbids. Pure regex, zero ambiguity, catches a bad prompt edit on the first run. |
| `groundedness` | Fraction of the answer's numeric claims that appear in the injected context. |
| `thinkTagLeakRate` | Reasoning tags reaching the user. Should always be 0. |
**Groundedness only sees numbers, and only numbers above 10.** An answer that
fabricates a procedure or a proper noun scores a perfect 1.0. It is a
fabrication *detector*, not a faithfulness guarantee. Numbers are the right
first target for this domain — a wrong bleach dose or canner pressure is a wrong
answer with consequences — but do not read a high score as "the answer is
faithful". Small integers are excluded because "3 layers" and "step 2" appear in
any prose and would swamp the signal.
---
## Known limitations
Read these before trusting a number.
1. **The corpus is small, so retrieval recall has little headroom.** 28
documents produce 29 chunks; retrieving the top 5 means retrieving 17% of the
entire corpus on every query. Real NOMAD knowledge bases hold millions of
chunks. `recall@5` therefore sits near 0.99 and cannot detect a modest
retrieval regression. The metrics that *do* have headroom on this corpus are
`recall@1`, `precision@k`, `nonEmptyRateOnRefusal`, and the score
distributions. To make recall discriminating, add substantially more
distractor documents, or ingest `install/wikipedia_en_100_mini_*.zim` under
the eval tag as a harder tier.
2. **`oracle` is not guaranteed to beat `e2e`.** Oracle injects the golden's
whole documents; e2e injects up to `maxResults` retrieved chunks, which can
include a genuinely helpful extra document. Treat small inversions as noise
unless they survive `--repeats=5`.
3. **Temperature 0 is not determinism.** The generation tier still moves between
runs. Always report `--repeats` ≥ 3 before concluding anything, and ignore
questions the harness flags as unstable.
4. **No LLM judge yet.** Faithfulness beyond numeric grounding, completeness,
and answer relevance are not measured. `autoevals` (MIT, TypeScript, talks to
Ollama through the same `/v1` endpoint `OllamaService` already uses) is the
intended addition, reported in a separate section and never mixed into the
deterministic scores — a weak local judge grading a weak local model is not
something to gate on.
---
## Layout
```
tests/eval/
corpus/ 28 markdown fixtures — the frozen knowledge base
goldens/ 99 questions as JSONL
baselines/ <corpus-fingerprint>/*.json — COMMITTED; the gate compares against these
reports/ run artifacts — gitignored
```
Baselines are filed under their corpus fingerprint so it is structurally
impossible to overwrite one corpus's baseline with a run against another.
Implementation:
- `app/utils/eval/*` — pure functions (metrics, golden parsing, report diffing).
No I/O, no models, no services. 153 tests, run with:
```bash
npm run test:eval # this harness only — should always be green
npm run test:unit # every tests/unit spec
```
`test:unit` currently reports **6 pre-existing failures** unrelated to this
harness: `drug_interactions`, `drug_ingest_status`, and `drug_labels` are
written against `@japa/runner` rather than `node:test`, and
`app_auto_update`, `content_auto_update`, and `content_auto_update_backoff`
import services that need a booted application. Both groups belong in the Japa
suite (`node ace test`, which needs MySQL and Redis). They fail identically
before and after any change here — use `test:eval` when you want a signal you
can trust.
- `app/services/eval_*_service.ts` — orchestration; these need Qdrant and Ollama.
- `commands/eval/*` — the CLI.
- `app/services/rag_pipeline_service.ts` — the prompt pipeline, shared with the
chat endpoint. The harness measures production code, not a copy of it.

4732
admin/tests/eval/baselines/51e642964facf251/retrieval.json Normal file
View File

File diff suppressed because it is too large Load Diff

32
admin/tests/eval/corpus/equipment-tr88-pump.md Normal file
View File

@ -0,0 +1,32 @@
# Model TR-88 Manual Transfer Pump
The TR-88 is a hand-operated diaphragm pump intended for moving potable water
between storage containers where no power is available.
## Specifications
- Rated flow: **4.5 liters per minute** at 60 strokes per minute
- Maximum suction lift: **6.5 meters**
- Wetted materials: polypropylene body, EPDM diaphragm, stainless steel hardware
- Dry weight: 1.9 kg
- Inlet and outlet: 25 mm barbed
## Maintenance
Replace the EPDM diaphragm every **500 operating hours** or every three years,
whichever comes first. The diaphragm is the only wear part that will strand the
pump; everything else degrades gracefully.
Flush with clean water after every use. The TR-88 tolerates silty water but the
check valves will stick if sediment is allowed to dry inside the chambers.
## Known limitation
The TR-88 must not be used with any fluid above **40°C**. The EPDM diaphragm
softens above that point and will deform permanently against the seat, after
which the pump will not hold prime.
## Spares kit
The TR-88 spares kit, part number **TR88-SK2**, contains two diaphragms, a set of
four check valve umbrellas, and two barb O-rings.

29
admin/tests/eval/corpus/firstaid-bleeding-control.md Normal file
View File

@ -0,0 +1,29 @@
# Controlling Severe Bleeding
Uncontrolled bleeding kills faster than almost anything else you can treat in
the field. Minutes matter.
## Direct pressure first
Press hard, directly on the wound, with whatever clean material you have. Do not
lift the dressing to check — that breaks the clot you just formed. Add layers on
top instead.
## Tourniquets
If direct pressure does not control arterial bleeding from a limb, apply a
tourniquet. Place it **2 to 3 inches above the wound**, never directly over a
joint. Tighten until the bleeding stops, which will hurt.
**Write the time of application** on the tourniquet or on the patient's
forehead. This single piece of information drives every decision made downstream
of you.
Do not loosen or remove a tourniquet once it is applied. That is a decision for
someone with surgical capability.
## Wound packing
For a deep wound in a junctional area where a tourniquet cannot be placed — the
groin, the armpit, the neck — pack the wound tightly with gauze and hold firm
pressure for a full **3 minutes** before reassessing.

24
admin/tests/eval/corpus/firstaid-burns.md Normal file
View File

@ -0,0 +1,24 @@
# Treating Burns
## Immediate care
Cool the burn under **cool running water for 20 minutes**. This works for up to
three hours after the injury, so it is still worth doing late.
Do **not** use ice, do not use butter or oil, and do not break blisters. Ice
causes further tissue damage in an area that has already lost its ability to
regulate.
Remove rings, watches, and tight clothing near the burn before swelling starts.
## Covering
Cover with a clean, non-adherent dressing or cling film laid on loosely — never
wrapped tightly around a limb.
## When it is beyond field care
Seek definitive care for any burn that is larger than the patient's palm, any
burn that is white, leathery, or painless (which indicates full thickness), and
any burn to the face, hands, feet, genitals, or across a joint. Burns that
circle a limb entirely are an emergency regardless of size.

26
admin/tests/eval/corpus/firstaid-hypothermia.md Normal file
View File

@ -0,0 +1,26 @@
# Recognizing and Treating Hypothermia
Hypothermia begins when core body temperature drops below **95°F (35°C)**.
## Stages
**Mild** — shivering, clumsiness, slurred speech, and poor judgement. The person
often insists they are fine. Confusion is the tell.
**Moderate** — shivering stops. This is not improvement; it means the body has
run out of the energy to shiver. Consciousness clouds.
**Severe** — shivering absent, muscles rigid, pulse and breathing very slow and
hard to detect.
## Treatment
Get the person out of the wind and out of wet clothing. Insulate them from the
ground, which steals heat faster than the air does. Apply heat to the torso —
armpits, chest, groin — not to the extremities.
Handle a severely hypothermic person **gently**. Rough movement can trigger a
fatal cardiac arrhythmia in a cold heart.
Give warm sweet drinks only if they are fully alert and can swallow. No alcohol:
it feels warming because it dilates surface vessels, which dumps core heat.

25
admin/tests/eval/corpus/firstaid-ifak-contents.md Normal file
View File

@ -0,0 +1,25 @@
# Building an IFAK
An **IFAK** is an Individual First Aid Kit. It is not a boo-boo kit. Its purpose
is to treat life-threatening trauma on one person — you — long enough to reach
real care.
## Core contents
- One **CAT or SOF-T tourniquet**, carried where you can reach it one-handed
- Two packages of hemostatic or plain compressed gauze for wound packing
- One pressure bandage
- One vented chest seal, ideally two
- One nasopharyngeal airway with lubricant
- Trauma shears
- Nitrile gloves
- A permanent marker for writing the tourniquet time
## Carry rules
Carry the IFAK on your body, not in your pack. A kit in a bag you dropped when
you were injured is a kit you do not have.
Everything in the kit should be something you have actually practiced with. A
tourniquet you have never applied under stress is a tourniquet you will apply
too loosely.

23
admin/tests/eval/corpus/food-canning-pressure.md Normal file
View File

@ -0,0 +1,23 @@
# Pressure Canning Low-Acid Foods
Low-acid foods — vegetables, meats, poultry, seafood, and most soups — must be
pressure canned. A boiling water bath cannot reach a high enough temperature to
destroy Clostridium botulinum spores.
## Temperature and pressure
Pressure canning holds the jars at **240°F (116°C)**. At sea level this
corresponds to **10 psi** on a weighted gauge or **11 psi** on a dial gauge.
Increase the pressure with altitude. On a weighted gauge, move from 10 psi to
15 psi above 1,000 feet. On a dial gauge, add roughly 1 psi for each additional
2,000 feet of elevation.
## Do not improvise
Processing times come from tested recipes and are specific to the food, the jar
size, and the pack style. Do not shorten a time, do not scale a recipe up, and
do not substitute a different vegetable into a tested recipe.
Have a dial gauge tested for accuracy once a year. A gauge reading 2 psi high
means every batch that season was under-processed.

23
admin/tests/eval/corpus/food-canning-waterbath.md Normal file
View File

@ -0,0 +1,23 @@
# Water Bath Canning
Water bath canning submerges filled jars in boiling water. The contents reach
212°F (100°C) at sea level and no higher, which is why this method is limited to
a specific class of foods.
## Only high-acid foods
Water bath canning is safe **only** for high-acid foods — those with a pH of
**4.6 or below**. That means most fruits, jams and jellies, pickles, and
properly acidified tomatoes.
Tomatoes sit close to the line. Modern varieties are often above pH 4.6, so
tested recipes call for added bottled lemon juice or citric acid. Use bottled
lemon juice, not fresh: fresh juice varies in acidity.
## Method
Cover the jars with at least one inch of water. Start timing when the water
reaches a full boil, and add time for altitude — 5 additional minutes for
1,001 to 3,000 feet, and more above that.
Never use this method for green beans, corn, meat, or soup.

20
admin/tests/eval/corpus/food-dehydrating.md Normal file
View File

@ -0,0 +1,20 @@
# Dehydrating Food
Dehydration removes moisture so that bacteria, yeasts, and molds cannot grow.
Properly dried and stored food keeps for a year or more.
## Temperatures
Dry vegetables at **125°F (52°C)** and fruits at **135°F (57°C)**. Meat for
jerky is different: heat it to an internal **160°F (71°C)** before or after
drying to kill pathogens, then dry at 130-140°F.
Drying too hot causes case hardening — the outside seals while the inside is
still wet, and the piece spoils from within.
## Conditioning
After drying, pack the pieces loosely in a sealed jar for **7 days** and shake it
daily. This is called conditioning, and it equalizes residual moisture between
pieces. If you see any condensation on the glass, the batch needs more drying
time.

30
admin/tests/eval/corpus/food-mylar-oxygen.md Normal file
View File

@ -0,0 +1,30 @@
# Mylar Bags and Oxygen Absorbers
Mylar is a metallized polyester film. It blocks light and, unlike bare plastic
buckets, it is an effective oxygen and moisture barrier.
## Thickness
Use bags of at least **5 mil** thickness for long-term storage. Thinner bags
puncture on the sharp edges of dry pasta and rice.
## Sizing oxygen absorbers
Oxygen absorbers are rated in cubic centimeters (cc) of oxygen they can take up.
Size them to the bag, not the food:
- 1 gallon bag: **300cc**
- 5 gallon bag: **2000cc**
When in doubt, over-size. An absorber that is too large costs a few cents; one
that is too small leaves oxygen behind and the batch degrades.
Absorbers begin working the moment air hits them. Open only what you will use
within about 15 minutes and keep the rest in a sealed jar.
## Do not use with
Do not use oxygen absorbers with sugar or brown sugar — removing the oxygen
turns them into a solid brick. Do not use them for anything with a moisture
content above about 10%; low oxygen plus moisture is the exact condition
botulism spores prefer.

22
admin/tests/eval/corpus/food-storage-rotation.md Normal file
View File

@ -0,0 +1,22 @@
# Rotating Stored Food
A store you never rotate is a store that quietly expires.
## First in, first out
Run your pantry on **FIFO** — first in, first out. New purchases go to the back;
the oldest stock comes forward and gets eaten first. Date every container with a
marker on the day it enters the pantry, not the day it was manufactured.
The single most common failure is a deep shelf. If you cannot see the back row,
you will not rotate it. Use shallow shelving, gravity-fed can racks, or bins you
can pull all the way out.
## Shelf life by category
Wheat, white rice, rolled oats, and dried beans stored cool and dry in sealed
containers with oxygen absorbers last **20 to 30 years**. Brown rice does not —
its oils go rancid in about 6 months at room temperature.
Canned goods stay safe indefinitely if the seal holds, but quality declines
after 2 to 5 years.

27
admin/tests/eval/corpus/gear-bug-out-bag.md Normal file
View File

@ -0,0 +1,27 @@
# The Bug Out Bag
A **BOB** — bug out bag — is a pre-packed kit that sustains one person for
**72 hours** while moving on foot to a predetermined destination. It is not a
wilderness survival kit and it is not everything you own.
## Weight
Target **20% of body weight** loaded, and treat 25% as a hard ceiling. Most
first attempts come in far too heavy, because every individual item seems
reasonable in isolation. Weigh the packed bag and walk five miles with it. That
walk will remove more weight than any packing list.
## Contents by priority
1. **Water** — one liter carried, plus a filter and chemical backup
2. **Shelter** — tarp or bivy, and a sleeping bag rated for the season
3. **Fire** — ferro rod plus a lighter plus tinder; three methods
4. **Food** — 72 hours of no-cook calories
5. **First aid** — a real trauma kit, not a blister kit
6. **Navigation** — paper map and compass, plus a charged phone
7. **Documents** — copies of ID, deeds, prescriptions, on paper and encrypted
## The destination
A bug out bag with no destination is camping gear. Decide in advance where you
are going, who is expecting you, and what the two alternate routes are.

21
admin/tests/eval/corpus/gear-edc.md Normal file
View File

@ -0,0 +1,21 @@
# Everyday Carry
**EDC** is every day carry: the small set of items on your person during a normal
day. Its value is that it is *always* present, which no larger kit can claim.
## A minimal set
- A light. A single-AAA or rechargeable pocket light. This is the item people
use most and pack last.
- A knife or multitool.
- A means of making fire.
- A phone with offline maps downloaded.
- A tourniquet, if you carry anything medical at all. It is the only item that
reverses a death in the timeframe you can act in.
- Cash in small bills.
## The test
The test of an EDC item is whether it is on you right now. An item you leave at
home when the clothes are inconvenient is not EDC — it is a bag item, and it
should be planned for as one.

31
admin/tests/eval/corpus/navigation-map-compass.md Normal file
View File

@ -0,0 +1,31 @@
# Map and Compass
## Declination
True north and magnetic north are not the same, and the angle between them is
**declination**. It varies by location and drifts over time. A map printed with a
declination diagram from 1998 is telling you about 1998.
Set the declination on the compass housing if it adjusts. If it does not, write
the correction on the compass with a paint pen so you are not doing arithmetic
under stress.
## Taking a bearing
Lay the compass edge along your intended line of travel on the map, rotate the
housing until the orienting lines match the map's north-south grid, then lift the
compass and turn your body until the needle sits in the shed. Walk the direction
of travel arrow.
## Pacing
Know your pace count — the number of double-steps you take to cover 100 meters.
For most adults it is between **60 and 70** on flat ground, and it lengthens
downhill and shortens uphill and under load. Measure yours on a known distance;
do not assume an average.
## Handrails and catching features
Navigate along linear features — a stream, a ridge, a fence line — and pick a
"catching feature" beyond your target that tells you when you have gone too far.
This is more reliable than trying to hit a small point precisely.

31
admin/tests/eval/corpus/power-battery-chemistry.md Normal file
View File

@ -0,0 +1,31 @@
# Battery Chemistry for Storage Systems
## Cycle life
The number that matters is cycles at a given depth of discharge.
- **LiFePO4** (lithium iron phosphate): **3,000 to 5,000 cycles** at 80% depth of
discharge
- **AGM** (absorbed glass mat lead-acid): **300 to 500 cycles** at 50% depth of
discharge
- **Flooded lead-acid**: 500 to 1,200 cycles at 50%, but requires watering and
ventilation
LiFePO4 costs more per amp-hour up front and far less per usable cycle.
## Temperature
LiFePO4 must not be **charged below freezing (32°F / 0°C)**. Charging a cold
lithium cell plates metallic lithium onto the anode, which is permanent and
eventually causes an internal short. Discharging below freezing is fine, just
reduced in capacity. Any decent battery management system enforces this, but
cheap cells often ship without one.
Lead-acid tolerates cold charging but loses capacity, and a discharged lead-acid
battery can freeze solid and split its case.
## Usable capacity
A 100Ah LiFePO4 battery gives you about 80Ah usable. A 100Ah AGM gives you about
50Ah before you start destroying it. Compare on usable capacity, never on
nameplate.

28
admin/tests/eval/corpus/power-emp-protection.md Normal file
View File

@ -0,0 +1,28 @@
# EMP and Faraday Cages
**EMP** stands for electromagnetic pulse. A sufficiently strong pulse induces
currents in conductors, and small unshielded electronics with long attached leads
are the most vulnerable.
## What a Faraday cage does
A Faraday cage is a continuous conductive enclosure. Incoming fields induce
currents in the shell, which cancel the field inside. Three things determine
whether it works:
1. **Continuity.** The conductive layer must be unbroken. A seam that is taped
rather than conductively bonded is a hole.
2. **Insulation.** Contents must not touch the conductive shell. A cardboard box
inside a metal trash can is the classic arrangement.
3. **Closure.** The lid must make contact all the way around. Copper tape along
the rim is a cheap fix.
A galvanized steel trash can with a tight lid, contents boxed in cardboard, is a
genuinely effective cage. It does not need to be grounded to protect its
contents — grounding matters for personnel safety, not for shielding.
## What to put in it
Spares, not primaries: a backup handheld radio, a spare charge controller, an
old laptop with offline references, a spare inverter board. Anything already
plugged into long wiring during the event is likely lost regardless.

28
admin/tests/eval/corpus/power-solar-sizing.md Normal file
View File

@ -0,0 +1,28 @@
# Sizing a Small Solar System
Work backwards from your loads, never forwards from a panel you happened to buy.
## Step one: measure the load
Add up watt-hours per day for everything you intend to run. A device drawing 20
watts for 6 hours is 120 watt-hours. Measure real devices with a meter; nameplate
ratings are near-useless.
## Step two: size the battery
Divide daily watt-hours by battery voltage to get amp-hours. Then divide by the
usable depth of discharge — **50%** for lead-acid, **80%** for LiFePO4. Then
multiply by the number of days of autonomy you want to survive without sun.
## Step three: size the panels
Divide daily watt-hours by your location's **peak sun hours** — typically 3 to 5,
not the number of daylight hours. Then add **25%** for charge controller
inefficiency, wiring loss, panel soiling, and the fact that panels never hit
their rated output in the real world.
## Charge controllers
An MPPT controller extracts roughly 20-30% more energy than a PWM controller from
the same panel, and the gap widens in cold weather and low light. PWM is only
sensible on very small systems where the controller cost dominates.

37
admin/tests/eval/corpus/protocol-thornfield-cache.md Normal file
View File

@ -0,0 +1,37 @@
# The Thornfield Cache Rotation Protocol
The Thornfield protocol is a scheme for rotating buried supply caches so that no
single cache is ever both stale and unmonitored. It was written up by the
Thornfield Valley mutual aid group and has since been adopted by several
neighboring groups.
## The rule of three
Every cache site is one of exactly three states: **primed**, **resting**, or
**pulled**. A group maintains three sites and one site is in each state at all
times.
- A **primed** cache is freshly stocked and sealed. It is checked but never
opened.
- A **resting** cache has been in the ground longest and is next in line.
- A **pulled** cache has been recovered, its contents inspected and consumed or
redistributed, and the site is left empty and marked as burned.
## The rotation interval
Sites rotate every **11 months**. The odd interval is deliberate: a 12 month
cycle means every inspection happens in the same season, and the group never
learns how the site behaves in a wet spring or a hard freeze.
## Marking
Sites are recorded by a three-word reference and never by coordinates on any
device. The canonical Thornfield marker is a **cairn of five stones** set 12
paces magnetic north of the cache itself, which looks like nothing to a passer-by
and is unambiguous to someone who knows the count.
## Who holds the map
No single person holds the complete site list. The protocol splits it: each of
three custodians knows two of the three sites. Any two custodians together can
reconstruct the full list; any one alone cannot.

72
admin/tests/eval/corpus/radio-antenna-basics.md Normal file
View File

@ -0,0 +1,72 @@
# Antenna Fundamentals for Field Operators
The antenna matters more than the radio. A 5 watt handheld with a good antenna
at height will outperform a 50 watt mobile feeding a compromised antenna at
ground level, every time. If you have limited money and limited time, spend both
on the antenna system.
## Why length matters
An antenna is a resonant structure. It radiates most efficiently when its
physical length bears a specific relationship to the wavelength of the signal it
is carrying. Wavelength in meters is found by dividing 300 by the frequency in
megahertz. At 146 MHz, near the center of the 2 meter band, a full wavelength is
just over two meters.
Most practical antennas are a fraction of a wavelength. A quarter-wave whip is
the most common compromise on handhelds because it is short enough to carry and
still works acceptably against a ground plane.
## The rubber duck problem
The short flexible antenna that ships with a handheld is a helically wound
quarter-wave that has been physically shortened. Shortening an antenna costs
efficiency. A stock rubber duck typically radiates a fraction of the power that
reaches it, with the remainder lost as heat in the loading coil and in ground
losses through your hand and body.
Replacing it with a full-length whip is usually the single cheapest improvement
available to a handheld operator. A roll-up J-pole hung from a tree branch is
better still, and costs almost nothing to make from twin-lead.
## Ground planes and counterpoises
A quarter-wave antenna is only half of the radiating structure. The other half is
the ground plane — the conductive surface the antenna works against. On a vehicle
that is the metal roof. On a handheld it is your hand, your arm, and your body,
which is why holding a radio differently changes your signal report.
A counterpoise fixes this. It is nothing more than a wire, cut to a quarter
wavelength, attached to the ground side of the antenna connector and allowed to
hang. On a 2 meter handheld a counterpoise is about **19.5 inches** long, and
adding one is frequently worth more than a new radio.
## Feedline loss
Coaxial cable loses signal, and the loss increases with frequency and with
length. RG-58 is convenient and lossy. LMR-400 is stiff, expensive, and much
better. On a short handheld setup feedline barely matters; on a 100 foot run to a
rooftop antenna at 440 MHz it can eat most of your transmitted power before it
reaches the antenna.
Loss works in both directions. A lossy feedline degrades what you hear just as
much as what you send.
## SWR and what it actually tells you
Standing wave ratio measures how much power is reflected back down the feedline
instead of being radiated. A perfect match reads 1:1. Most operators aim to stay
below **2:1**, above which many solid-state transmitters begin folding back their
output power to protect the finals.
A low SWR does not mean a good antenna. A dummy load has a perfect 1:1 SWR and
radiates nothing at all. SWR tells you about the match, not about the radiation.
This confuses people endlessly: they trim an antenna for the lowest possible SWR
and end up with something that matches beautifully and gets out poorly.
## Height
Above roughly 50 MHz, propagation is essentially line of sight, extended a little
by atmospheric refraction. Every foot of height buys range, and height beats
power by a wide margin. Ten feet of mast is worth more than doubling your
transmitter output.

25
admin/tests/eval/corpus/radio-gmrs-frs.md Normal file
View File

@ -0,0 +1,25 @@
# GMRS and FRS Radios
Two license classes cover most short-range handheld radio use in the United
States, and people confuse them constantly.
## FRS
**FRS** is the Family Radio Service. It requires **no license**. Power is capped
at **2 watts** on channels 1-7 and 15-22, and at 0.5 watts on channels 8-14.
Antennas are fixed and cannot be replaced. Realistic range is well under a mile
in terrain, despite what the blister pack claims.
## GMRS
**GMRS** is the General Mobile Radio Service. It **requires a license**, but
there is **no examination** — you pay a fee and the license is issued. One
license covers your entire immediate family, and it is valid for **10 years**.
GMRS allows up to **50 watts** on the main channels, removable antennas, and
repeater use, which is what actually extends range beyond line of sight.
## Sharing channels
FRS channels 1-7 and 15-22 overlap with GMRS frequencies, so an FRS handheld and
a GMRS handheld can talk to each other on those channels.

29
admin/tests/eval/corpus/radio-ham-bands.md Normal file
View File

@ -0,0 +1,29 @@
# Amateur Radio License Classes
Amateur radio — **ham** radio — requires passing a written examination. There are
three current US license classes, each granting more spectrum.
## Technician
The entry class. **35 questions**, and it grants full privileges on all amateur
bands above 30 MHz, which covers the 2 meter and 70 centimeter bands most
handhelds use. Limited HF privileges on 10 meters and small CW segments.
## General
**35 questions** beyond Technician. Opens most of the HF spectrum, which is what
makes long-distance and over-the-horizon communication possible.
## Amateur Extra
**50 questions**. Grants the remaining HF segments, mostly the lower portions of
each band where the DX contests live.
There is **no Morse code requirement** for any current US class; that was dropped
in 2007.
## Emergency use
Any station may use any means of radiocommunication at its disposal in a genuine
emergency involving immediate safety of life or property. That provision exists
for real emergencies, not for skipping the exam.

19
admin/tests/eval/corpus/regions-elevation-table.md Normal file
View File

@ -0,0 +1,19 @@
# Settlement Elevation Reference
Elevations for the settlements in the regional mutual aid network, surveyed at
the town marker. Use these when adjusting any procedure that depends on
elevation — canning pressure, boiling time, and engine tuning all change with
altitude.
| Settlement | Elevation (feet) | Elevation (meters) |
|-----------------|------------------|--------------------|
| Ashmere Landing | 240 | 73 |
| Thornfield | 1,180 | 360 |
| Calder Junction | 3,400 | 1,036 |
| Kestrel Ridge | 8,900 | 2,713 |
| Vantry Flats | 620 | 189 |
| Bellhollow | 5,150 | 1,570 |
Kestrel Ridge is the highest occupied site in the network and the only one above
the treeline. Bellhollow and Calder Junction sit in the band where altitude
adjustments start to matter but are easy to forget.

25
admin/tests/eval/corpus/sanitation-waste.md Normal file
View File

@ -0,0 +1,25 @@
# Sanitation When Plumbing Fails
More people die after a disaster from sanitation failures than from the event
itself. This is the least glamorous preparation and close to the most important.
## The twin-bucket system
Use two buckets: one for liquid waste, one for solid. Separating them is the
whole trick — mixed waste is what produces the smell and the pathogen load.
Line the solid bucket with a heavy bag and cover each use with a scoop of an
absorbent carbon material: sawdust, peat moss, coconut coir, or shredded paper.
Seal and remove the bag when it is roughly two-thirds full.
## Siting a latrine
If you must dig, place a latrine at least **200 feet (60 meters)** from any water
source, and downhill from it. Dig it at least 6 inches deep, cover after each
use, and do not site it anywhere water pools after rain.
## Handwashing
A tippy-tap — a jug on a string with a foot lever — uses a fraction of the water
of pouring and is the single highest-value sanitation build. Soap and running
water beat hand sanitizer against the organisms that matter here.

27
admin/tests/eval/corpus/shelter-cold-layering.md Normal file
View File

@ -0,0 +1,27 @@
# Layering for Cold Weather
Layering manages moisture as much as it manages temperature. Sweat is the enemy;
wet insulation is nearly worthless.
## The three layers
**Base layer** — wicks moisture off the skin. Merino wool or synthetic. Never
cotton: cotton holds water against the body and conducts heat away roughly 25
times faster than dry air.
**Mid layer** — insulates by trapping air. Fleece, down, or synthetic fill. Down
is lighter and more compressible; synthetic keeps insulating when wet.
**Shell layer** — blocks wind and precipitation. Wind stripping the warm air out
of your insulation is what actually makes you cold on a dry day.
## Vent before you sweat
Open zips, remove the hat, or shed the mid layer *before* you start sweating, not
after. Once the base layer is soaked you will not dry it out in the field.
## Extremities
Heat is lost fastest from the head, neck, hands, and feet. A hat and a neck
gaiter do more per gram than any other item. If your hands are cold, put on a
hat — the body restricts blood flow to the extremities to defend the core.

27
admin/tests/eval/corpus/water-boiling.md Normal file
View File

@ -0,0 +1,27 @@
# Boiling Water to Make It Safe
Boiling is the most reliable way to kill bacteria, viruses, and protozoa in
questionable water. It does not remove chemical contaminants, heavy metals, or
salt.
## How long to boil
Bring the water to a **rolling boil for 1 minute** at normal elevations. At
elevations **above 6,500 feet (2,000 meters)**, boil for **3 minutes** — water
boils at a lower temperature as altitude increases, so it needs longer at the
boil to reach the same level of pathogen kill.
Let the water cool naturally. Do not add ice to speed cooling; ice made from
untreated water re-contaminates the batch.
## Improving the taste
Boiled water tastes flat because boiling drives off dissolved air. Pour it back
and forth between two clean containers a few times to re-aerate it, or add a
small pinch of salt per quart.
## Before you boil
If the water is cloudy, let it settle and pour it through a clean cloth or
coffee filter first. Boiling cloudy water still works, but sediment shields
some organisms and makes the result unpleasant to drink.

27
admin/tests/eval/corpus/water-chemical-treatment.md Normal file
View File

@ -0,0 +1,27 @@
# Chemical Water Treatment
Chemical disinfection is lighter and faster than boiling when fuel is short.
It is less effective against protozoa such as Cryptosporidium than boiling or
filtration.
## Unscented household bleach
Use plain, unscented household bleach containing 5-6% sodium hypochlorite. Add
**8 drops per gallon** of clear water (about 1/8 teaspoon). Double that to 16
drops per gallon if the water is cloudy or very cold. Stir and let it stand for
**30 minutes** before drinking. The water should have a faint chlorine smell; if
it does not, repeat the dose and wait another 15 minutes.
Never use scented, color-safe, or additive-bearing bleach.
## Chlorine dioxide
Chlorine dioxide tablets are effective against bacteria, viruses, and — unlike
bleach — Cryptosporidium, but they need a long contact time: **4 hours** for
Cryptosporidium at cold temperatures. Follow the package dosing exactly.
## Iodine
Iodine works on bacteria and viruses but is unreliable against Cryptosporidium.
It is not suitable for pregnant people, anyone with thyroid conditions, or for
continuous use beyond a few weeks.

25
admin/tests/eval/corpus/water-filtration.md Normal file
View File

@ -0,0 +1,25 @@
# Water Filters: What They Do and Do Not Remove
A filter is rated by its pore size, measured in microns. The rating tells you
what physically cannot pass through.
## Pore sizes
A filter with an **absolute 0.2 micron** rating removes bacteria and protozoa,
including Giardia and Cryptosporidium. It does **not** remove viruses, which are
far smaller. In regions where viral contamination is a concern, filter first and
then disinfect chemically or by boiling.
"Nominal" ratings are marketing numbers and allow a percentage of larger
particles through. Look for the word "absolute".
## What filters never remove
Filters do not remove dissolved salts, heavy metals, or most chemicals. Activated
carbon elements improve taste and adsorb some organic compounds, but they are not
a substitute for testing when contamination is suspected.
## Maintenance
Backflush hollow-fiber filters before storage and never let them freeze — ice
crystals rupture the fibers and the damage is invisible from the outside.

18
admin/tests/eval/corpus/water-river-song.md Normal file
View File

@ -0,0 +1,18 @@
# The River Song
*A traditional verse, collected in the highlands.*
The river runs clean where the tall pines drink,
and the cold water sings on the stone;
I filled my cup at the mountain's brink
and boiled the night down to the bone.
Three days of rain and the fords ran high,
the crossing lost to the flood —
we watched the clean water hurrying by
and tasted the silt and the mud.
They sang this at the spring gathering, when the snowmelt came down and the
lower fields flooded. The verse is older than the settlement itself, and nobody
remembers who wrote it. It is about patience, and about the distance between
water you can see and water you can drink.

22
admin/tests/eval/corpus/water-storage.md Normal file
View File

@ -0,0 +1,22 @@
# Storing Water
Store at least **one gallon per person per day**, with a two-week supply as a
practical target for a fixed location.
## Containers
Use food-grade containers marked with recycling code 2 (HDPE). Never reuse
containers that held milk or fruit juice — residual sugars and proteins support
bacterial growth no matter how well you rinse.
Keep containers off bare concrete. Plasticizers migrate and the floor stays cold
enough to cause condensation cycles.
## Rotation
Commercially bottled water carries a manufacturer date; rotate it every two
years. Water you bottled yourself from a treated municipal supply should be
rotated every **six months**.
Store away from sunlight and away from gasoline, pesticides, or solvents —
vapors permeate plastic over time.

99
admin/tests/eval/goldens/core.jsonl Normal file
View File

@ -0,0 +1,99 @@
{"id": "water-boil-time-01", "query": "How long should I boil water to make it safe to drink?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["\\b(1|one) minute"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-boil-altitude-01", "query": "How long do I need to boil water at high altitude?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["\\b(3|three) minutes?"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-boil-altitude-threshold-01", "query": "Above what elevation do I need to boil water longer?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["6,?500|2,?000\\s*(m|meter)"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-boil-taste-01", "query": "Why does boiled water taste flat and how do I fix it?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["aerat|pour|air"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "water-bleach-dose-01", "query": "How much unscented bleach do I add to a gallon of clear water?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["8 drops"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-bleach-cloudy-01", "query": "How much bleach should I use if the water is cloudy?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["16 drops"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-bleach-wait-01", "query": "After adding bleach to water, how long do I wait before drinking it?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["30 minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-cldioxide-crypto-01", "query": "How long do chlorine dioxide tablets need to work against Cryptosporidium?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["4 hours"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-filter-pore-01", "query": "What filter pore size removes Giardia and Cryptosporidium?", "turns": [], "relevantDocIds": ["water-filtration"], "mustInclude": ["0\\.2"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-filter-virus-01", "query": "Will a 0.2 micron water filter remove viruses?", "turns": [], "relevantDocIds": ["water-filtration"], "mustInclude": ["virus"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "water-filter-freeze-01", "query": "Why must I keep a hollow fiber water filter from freezing?", "turns": [], "relevantDocIds": ["water-filtration"], "mustInclude": ["fiber|fibre|ruptur|crack|damage"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "water-storage-amount-01", "query": "How much drinking water should I store per person per day?", "turns": [], "relevantDocIds": ["water-storage"], "mustInclude": ["\\b(1|one) gallon"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-storage-rotation-01", "query": "How often should I rotate water I bottled myself from the tap?", "turns": [], "relevantDocIds": ["water-storage"], "mustInclude": ["six months|6 months"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "water-storage-containers-01", "query": "Which containers should I never reuse for storing drinking water?", "turns": [], "relevantDocIds": ["water-storage"], "mustInclude": ["milk|juice"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "food-pressure-temp-01", "query": "What temperature does pressure canning reach?", "turns": [], "relevantDocIds": ["food-canning-pressure"], "mustInclude": ["240"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-pressure-psi-01", "query": "What pressure should a weighted gauge canner run at, at sea level?", "turns": [], "relevantDocIds": ["food-canning-pressure"], "mustInclude": ["10 ?psi"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-greenbeans-method-01", "query": "Can I water bath can green beans?", "turns": [], "relevantDocIds": ["food-canning-pressure", "food-canning-waterbath"], "mustInclude": ["pressure"], "mustNotInclude": [], "expectRefusal": false, "tags": ["near-miss", "multi-hop"]}
{"id": "food-waterbath-ph-01", "query": "What pH do foods need to be safe for water bath canning?", "turns": [], "relevantDocIds": ["food-canning-waterbath"], "mustInclude": ["4\\.6"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-tomato-acid-01", "query": "Why do tomato canning recipes call for added lemon juice?", "turns": [], "relevantDocIds": ["food-canning-waterbath"], "mustInclude": ["acid|pH|4\\.6"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "food-dehydrate-veg-temp-01", "query": "What temperature should I dehydrate vegetables at?", "turns": [], "relevantDocIds": ["food-dehydrating"], "mustInclude": ["125"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-jerky-temp-01", "query": "What internal temperature does meat need to reach when making jerky?", "turns": [], "relevantDocIds": ["food-dehydrating"], "mustInclude": ["160"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-conditioning-01", "query": "What is conditioning dried food and how long does it take?", "turns": [], "relevantDocIds": ["food-dehydrating"], "mustInclude": ["(7|seven) days"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-fifo-01", "query": "What does FIFO mean for pantry rotation?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["first in,? first out"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]}
{"id": "food-fifo-spelled-01", "query": "How should I organize my pantry so the oldest food gets eaten first?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["first in,? first out|oldest|front|rotat"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]}
{"id": "food-brownrice-01", "query": "How long does brown rice last in storage?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["6 months|six months"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-wheat-shelflife-01", "query": "How long will wheat and white rice keep in sealed containers?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["20|30 years"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-oxyabsorber-5gal-01", "query": "What size oxygen absorber do I need for a 5 gallon mylar bag?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["2,?000"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-oxyabsorber-1gal-01", "query": "What size oxygen absorber goes in a 1 gallon mylar bag?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["300"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-mylar-thickness-01", "query": "How thick should mylar bags be for long term food storage?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["5 ?mil"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "food-oxyabsorber-sugar-01", "query": "Can I use oxygen absorbers with sugar?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["brick|hard|solid"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "firstaid-tq-placement-01", "query": "Where do I place a tourniquet relative to the wound?", "turns": [], "relevantDocIds": ["firstaid-bleeding-control"], "mustInclude": ["2 ?(to|-|\u2013) ?3 inch|above"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "firstaid-tq-time-01", "query": "What should I write down after applying a tourniquet?", "turns": [], "relevantDocIds": ["firstaid-bleeding-control"], "mustInclude": ["time"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "firstaid-woundpack-01", "query": "How long do I hold pressure after packing a wound?", "turns": [], "relevantDocIds": ["firstaid-bleeding-control"], "mustInclude": ["(3|three) minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "firstaid-ifak-acronym-01", "query": "What does IFAK stand for?", "turns": [], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["individual first aid kit"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]}
{"id": "firstaid-ifak-spelled-01", "query": "What should go in an individual first aid kit?", "turns": [], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["tourniquet"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]}
{"id": "firstaid-ifak-carry-01", "query": "Where should I carry my IFAK?", "turns": [], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["body|person|on you"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]}
{"id": "firstaid-hypothermia-temp-01", "query": "At what core body temperature does hypothermia begin?", "turns": [], "relevantDocIds": ["firstaid-hypothermia"], "mustInclude": ["95|35"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "firstaid-hypothermia-shivering-01", "query": "What does it mean when a cold person stops shivering?", "turns": [], "relevantDocIds": ["firstaid-hypothermia"], "mustInclude": ["moderate|worse|energy|serious|not.{0,15}improv"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "firstaid-hypothermia-handling-01", "query": "Why must a severely hypothermic person be handled gently?", "turns": [], "relevantDocIds": ["firstaid-hypothermia"], "mustInclude": ["arrhythmia|cardiac|heart"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "firstaid-burn-cooling-01", "query": "How long should I cool a burn under running water?", "turns": [], "relevantDocIds": ["firstaid-burns"], "mustInclude": ["20 minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "firstaid-burn-ice-01", "query": "What should I use to cool a burn?", "turns": [], "relevantDocIds": ["firstaid-burns"], "mustInclude": ["cool running water|cool water"], "mustNotInclude": ["\\buse ice\\b|apply ice"], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "radio-gmrs-license-01", "query": "Do I need to pass a test to get a GMRS license?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["no exam|no examination|without.{0,20}exam|not.{0,20}exam"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]}
{"id": "radio-gmrs-term-01", "query": "How long is a GMRS license valid?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["10 years|ten years"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "radio-frs-power-01", "query": "What is the maximum power for an FRS radio?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["2 ?watt"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]}
{"id": "radio-gmrs-power-01", "query": "How much power can a GMRS radio transmit?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["50 ?watt"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]}
{"id": "radio-frs-license-01", "query": "Does the Family Radio Service require a license?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["no licen|not require|without a licen"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]}
{"id": "radio-tech-questions-01", "query": "How many questions are on the Technician amateur radio exam?", "turns": [], "relevantDocIds": ["radio-ham-bands"], "mustInclude": ["\\b35\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "radio-extra-questions-01", "query": "How many questions are on the Amateur Extra exam?", "turns": [], "relevantDocIds": ["radio-ham-bands"], "mustInclude": ["\\b50\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "radio-morse-01", "query": "Do I still need to learn Morse code to get a ham license?", "turns": [], "relevantDocIds": ["radio-ham-bands"], "mustInclude": ["no longer|not required|dropped|2007|no morse"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]}
{"id": "radio-counterpoise-01", "query": "How long should a counterpoise be for a 2 meter handheld?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["19\\.5"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary", "numeric"]}
{"id": "radio-swr-threshold-01", "query": "What SWR should I stay below on a solid state transmitter?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["2:1|2 ?to ?1"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary", "numeric"]}
{"id": "radio-swr-meaning-01", "query": "Does a low SWR mean I have a good antenna?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["dummy load|not mean|does not|match"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary"]}
{"id": "radio-wavelength-01", "query": "How do I calculate wavelength from frequency?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["300"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "radio-height-vs-power-01", "query": "Is it better to raise my antenna or increase transmit power?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["height"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary"]}
{"id": "power-leadacid-dod-01", "query": "What depth of discharge should I design for with lead acid batteries?", "turns": [], "relevantDocIds": ["power-solar-sizing", "power-battery-chemistry"], "mustInclude": ["50 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]}
{"id": "power-lifepo4-dod-01", "query": "How deeply can I discharge a LiFePO4 battery?", "turns": [], "relevantDocIds": ["power-solar-sizing", "power-battery-chemistry"], "mustInclude": ["80 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]}
{"id": "power-derate-01", "query": "How much margin should I add when sizing solar panels?", "turns": [], "relevantDocIds": ["power-solar-sizing"], "mustInclude": ["25 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "power-mppt-01", "query": "How much better is an MPPT charge controller than PWM?", "turns": [], "relevantDocIds": ["power-solar-sizing"], "mustInclude": ["20|30"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]}
{"id": "power-peaksun-01", "query": "What are peak sun hours and what value should I use?", "turns": [], "relevantDocIds": ["power-solar-sizing"], "mustInclude": ["3|4|5"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "power-lifepo4-cycles-01", "query": "How many cycles does a LiFePO4 battery last?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["3,?000|5,?000"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "power-agm-cycles-01", "query": "How many cycles will an AGM battery give me?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["300|500"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]}
{"id": "power-lifepo4-cold-01", "query": "Can I charge a lithium iron phosphate battery below freezing?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["32|0 ?\u00b0?C|freezing"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "power-usable-capacity-01", "query": "How many usable amp hours do I get from a 100Ah LiFePO4 battery?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["\\b80\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "power-emp-acronym-01", "query": "What does EMP stand for?", "turns": [], "relevantDocIds": ["power-emp-protection"], "mustInclude": ["electromagnetic pulse"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]}
{"id": "power-emp-spelled-01", "query": "How do I protect spare electronics from an electromagnetic pulse?", "turns": [], "relevantDocIds": ["power-emp-protection"], "mustInclude": ["faraday|trash can|conductive|enclos"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]}
{"id": "power-faraday-ground-01", "query": "Does a Faraday cage need to be grounded to protect what is inside it?", "turns": [], "relevantDocIds": ["power-emp-protection"], "mustInclude": ["not need|no need|not required|does not|doesn't"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "gear-bob-acronym-01", "query": "How many hours should a BOB sustain me?", "turns": [], "relevantDocIds": ["gear-bug-out-bag"], "mustInclude": ["72|three days|3 days"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]}
{"id": "gear-bob-spelled-01", "query": "How many hours should a bug out bag sustain me?", "turns": [], "relevantDocIds": ["gear-bug-out-bag"], "mustInclude": ["72|three days|3 days"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control", "numeric"]}
{"id": "gear-bob-weight-01", "query": "How heavy should my bug out bag be?", "turns": [], "relevantDocIds": ["gear-bug-out-bag"], "mustInclude": ["20 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "gear-edc-acronym-01", "query": "What does EDC mean?", "turns": [], "relevantDocIds": ["gear-edc"], "mustInclude": ["every ?day carry"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]}
{"id": "shelter-cotton-01", "query": "Why should I avoid cotton base layers in cold weather?", "turns": [], "relevantDocIds": ["shelter-cold-layering"], "mustInclude": ["25 times|conduct|holds water|wet|moist"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "shelter-layers-01", "query": "What are the three clothing layers for cold weather?", "turns": [], "relevantDocIds": ["shelter-cold-layering"], "mustInclude": ["base", "mid", "shell"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "shelter-cold-hands-01", "query": "My hands are cold even in gloves. What should I do?", "turns": [], "relevantDocIds": ["shelter-cold-layering"], "mustInclude": ["hat|head|core"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "sanitation-latrine-distance-01", "query": "How far from a water source should I dig a latrine?", "turns": [], "relevantDocIds": ["sanitation-waste"], "mustInclude": ["200 ?(feet|ft)|60 ?(meters|metres|m)\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "sanitation-buckets-01", "query": "Why separate liquid and solid waste in a bucket toilet?", "turns": [], "relevantDocIds": ["sanitation-waste"], "mustInclude": ["smell|odou?r|pathogen"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "nav-declination-01", "query": "What is magnetic declination?", "turns": [], "relevantDocIds": ["navigation-map-compass"], "mustInclude": ["true north|magnetic north|angle|differ"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]}
{"id": "nav-pacecount-01", "query": "What is a typical pace count for 100 meters?", "turns": [], "relevantDocIds": ["navigation-map-compass"], "mustInclude": ["60|70"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]}
{"id": "fict-thornfield-interval-01", "query": "How often do Thornfield cache sites rotate?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["11 months|eleven months"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]}
{"id": "fict-thornfield-states-01", "query": "What are the three states of a Thornfield cache site?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["primed", "resting", "pulled"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]}
{"id": "fict-thornfield-marker-01", "query": "How is a Thornfield cache site marked on the ground?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["five stones|5 stones|cairn"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]}
{"id": "fict-thornfield-custody-01", "query": "Who holds the full list of Thornfield cache sites?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["no single|two of the three|split|nobody|no one"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]}
{"id": "fict-tr88-flow-01", "query": "What is the rated flow of the TR-88 transfer pump?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["4\\.5"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]}
{"id": "fict-tr88-diaphragm-01", "query": "How often should the TR-88 diaphragm be replaced?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["500"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]}
{"id": "fict-tr88-maxtemp-01", "query": "What is the maximum fluid temperature for the TR-88 pump?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["40"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]}
{"id": "fict-tr88-spares-01", "query": "What is the part number for the TR-88 spares kit?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["TR88-SK2"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]}
{"id": "multihop-kestrel-boil-01", "query": "How long should I boil water at Kestrel Ridge?", "turns": [], "relevantDocIds": ["regions-elevation-table", "water-boiling"], "mustInclude": ["(3|three) minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]}
{"id": "multihop-ashmere-boil-01", "query": "How long should I boil water at Ashmere Landing?", "turns": [], "relevantDocIds": ["regions-elevation-table", "water-boiling"], "mustInclude": ["\\b(1|one) minute"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]}
{"id": "multihop-bellhollow-canning-01", "query": "Do I need to adjust my canner pressure at Bellhollow?", "turns": [], "relevantDocIds": ["regions-elevation-table", "food-canning-pressure"], "mustInclude": ["15 ?psi|increase|adjust|higher|yes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop"]}
{"id": "distract-river-drink-01", "query": "Is river water clean enough to drink straight from the stream?", "turns": [], "relevantDocIds": ["water-boiling", "water-filtration", "water-chemical-treatment"], "mustInclude": ["boil|filter|treat|disinfect"], "mustNotInclude": ["\\bpoem\\b|\\bverse\\b|\\bsong\\b|tall pines|highland"], "expectRefusal": false, "tags": ["distractor"]}
{"id": "distract-river-legit-01", "query": "What does the river song say happened after three days of rain?", "turns": [], "relevantDocIds": ["water-river-song"], "mustInclude": ["flood|high|ford"], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}
{"id": "ooc-tire-pressure-01", "query": "What is the recommended tire pressure for a Toyota Hilux?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus"]}
{"id": "ooc-bgp-01", "query": "How do I configure BGP route reflectors on a Juniper router?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus"]}
{"id": "ooc-thornfield-founded-01", "query": "What year was the Thornfield Valley mutual aid group founded?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus", "adversarial"]}
{"id": "ooc-tr88-warranty-01", "query": "What is the warranty period on the TR-88 pump?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus", "adversarial"]}
{"id": "ooc-tr92-01", "query": "How much does the Model TR-92 pump weigh?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus", "adversarial"]}
{"id": "multiturn-canning-temp-01", "query": "What temperature does it reach?", "turns": [{"role": "user", "content": "Tell me about pressure canning low acid foods."}, {"role": "assistant", "content": "Pressure canning is required for low-acid foods such as vegetables, meats, and soups, because a boiling water bath cannot destroy botulism spores."}], "relevantDocIds": ["food-canning-pressure"], "mustInclude": ["240"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn"]}
{"id": "multiturn-ifak-carry-01", "query": "Where should I carry it?", "turns": [{"role": "user", "content": "What should I put in an IFAK?"}, {"role": "assistant", "content": "An IFAK holds trauma gear: a tourniquet, hemostatic gauze, a pressure bandage, a chest seal, an airway, shears, and gloves."}], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["body|person|on you"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn"]}
{"id": "multiturn-tr88-diaphragm-01", "query": "How often does the diaphragm need replacing?", "turns": [{"role": "user", "content": "Tell me about the TR-88 transfer pump."}, {"role": "assistant", "content": "The TR-88 is a hand-operated diaphragm pump for moving potable water between containers without power."}], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["500"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn", "fictional"]}
{"id": "multiturn-boil-altitude-01", "query": "What about up at Kestrel Ridge?", "turns": [{"role": "user", "content": "How long should I boil water to make it safe?"}, {"role": "assistant", "content": "Bring it to a rolling boil for one minute at normal elevations."}], "relevantDocIds": ["regions-elevation-table", "water-boiling"], "mustInclude": ["(3|three) minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn", "multi-hop"]}

12
admin/tests/eval/reports/.gitignore vendored Normal file
View File

@ -0,0 +1,12 @@
# Eval reports are run artifacts: they carry a timestamp, a host, and a git SHA,
# and a new one is written every time someone runs the harness. Committing them
# would add noise to every RAG pull request without making anything reproducible.
#
# Baselines are the opposite — those live in ../baselines/<corpus-fingerprint>/
# and ARE committed, because the whole team needs to gate against the same
# numbers. Promote a report with:
#
# node ace eval:compare <report.json> --promote=<name>
*.json
*.md
!.gitignore

294
admin/tests/unit/eval_generation_metrics.spec.ts Normal file
View File

@ -0,0 +1,294 @@
/**
* Tests for the deterministic answer scorers.
*
* The refusal and leakage detectors are regex families, and a regex family is
* only as good as the phrasings it was tested against — so the cases below are
* real model phrasings, including the near-misses that must NOT trip them.
* A false positive on leakage would flag a perfectly good answer as a prompt
* regression, which is the fastest way to make people ignore the harness.
*
* npm run test:unit
*/
import * as assert from 'node:assert/strict'
import { test } from 'node:test'
import {
detectLeakage,
detectRefusal,
extractNumbers,
hasThinkTagLeak,
isMarkdownFormatted,
matchPatterns,
numericGroundedness,
scoreAnswer,
summarizeNumeric,
summarizeRepeats,
} from '../../app/utils/eval/generation_metrics.js'
// --- refusal detection ---------------------------------------------------------
const REFUSALS = [
"I don't know the answer to that.",
'I do not have that information.',
"I'm not able to answer that question.",
'I am unable to determine that.',
"I couldn't find anything about that.",
'There is no information available on that topic.',
'That is not mentioned anywhere.',
'The documentation does not specify a warranty period.',
"That isn't covered here.",
"I'd need more detail to answer that.",
]
for (const [i, text] of REFUSALS.entries()) {
test(`detects refusal phrasing #${i + 1}: ${text.slice(0, 40)}`, () => {
assert.equal(detectRefusal(text), true)
})
}
const NON_REFUSALS = [
'Boil the water for three minutes above 6,500 feet.',
'A tourniquet goes 2 to 3 inches above the wound.',
// "not" appears, but the answer is substantive — must not trip the detector.
'Do not use ice on a burn; cool it under running water for 20 minutes.',
'Water bath canning is not suitable for green beans, so use a pressure canner.',
'You know the water is safe once it has held a rolling boil.',
]
for (const [i, text] of NON_REFUSALS.entries()) {
test(`does not misread a substantive answer as a refusal #${i + 1}`, () => {
assert.equal(detectRefusal(text), false, `false positive on: ${text}`)
})
}
// --- leakage detection -----------------------------------------------------------
const LEAKS = [
'According to the context, boil for three minutes.',
'Based on the provided context, use 8 drops per gallon.',
'Context 1 describes the boiling procedure.',
'The knowledge base does not cover that.',
'The retrieved documents mention a 500 hour interval.',
"The context doesn't address your question.",
'I found no relevant context for this.',
'From the search results, the pressure is 10 psi.',
]
for (const [i, text] of LEAKS.entries()) {
test(`detects retrieval narration #${i + 1}`, () => {
assert.ok(detectLeakage(text).length > 0, `missed leakage in: ${text}`)
})
}
const CLEAN = [
'Boil the water for three minutes.',
'The TR-88 has a rated flow of 4.5 litres per minute.',
// "context" used in an ordinary sense, not as retrieval narration.
'In a wilderness context, a tourniquet is the priority.',
'Store one gallon per person per day.',
]
for (const [i, text] of CLEAN.entries()) {
test(`does not flag a clean answer as leakage #${i + 1}`, () => {
assert.deepEqual(detectLeakage(text), [], `false positive on: ${text}`)
})
}
// --- think tags -------------------------------------------------------------------
test('detects surviving reasoning tags', () => {
assert.equal(hasThinkTagLeak('<think>hmm</think>The answer is 3 minutes.'), true)
assert.equal(hasThinkTagLeak('<thinking>x</thinking> answer'), true)
})
test('does not flag ordinary prose containing the word think', () => {
assert.equal(hasThinkTagLeak('I think you should boil it longer.'), false)
})
// --- markdown ---------------------------------------------------------------------
test('recognises common markdown structures', () => {
assert.equal(isMarkdownFormatted('## Heading\n\ntext'), true)
assert.equal(isMarkdownFormatted('- one\n- two'), true)
assert.equal(isMarkdownFormatted('1. first'), true)
assert.equal(isMarkdownFormatted('use **bold** here'), true)
assert.equal(isMarkdownFormatted('| a | b |'), true)
})
test('flags an answer with no formatting at all', () => {
assert.equal(isMarkdownFormatted('Just a plain sentence with no structure.'), false)
})
// --- numbers and grounding ----------------------------------------------------------
test('extracts and normalizes numbers', () => {
assert.deepEqual(extractNumbers('2,000cc and 300cc'), ['2000', '300'])
})
test('treats 2,000 and 2000 as the same number', () => {
const { score, ungrounded } = numericGroundedness('Use a 2,000cc absorber.', 'Use a 2000cc absorber.')
assert.equal(score, 1)
assert.deepEqual(ungrounded, [])
})
test('flags a number the context does not support', () => {
// The classic fabrication: a plausible figure that appears nowhere.
const { score, ungrounded } = numericGroundedness(
'Replace the diaphragm every 250 hours.',
'Replace the EPDM diaphragm every 500 operating hours.'
)
assert.equal(score, 0)
assert.deepEqual(ungrounded, ['250'])
})
test('scores partial grounding', () => {
const { score } = numericGroundedness('Boil 3 minutes above 9000 feet.', 'Boil 3 minutes above 6500 feet.')
// 3 is excluded as a small integer; 9000 is claimed and unsupported.
assert.equal(score, 0)
})
test('ignores small integers, which appear incidentally in any prose', () => {
// "3 layers" and "step 2" would otherwise dominate the score with noise.
const { score } = numericGroundedness('There are 3 layers and 2 rules.', 'Wear a base, mid, and shell layer.')
assert.equal(score, null)
})
test('returns null when the answer makes no numeric claims', () => {
const { score } = numericGroundedness('Cool the burn under running water.', 'Cool for twenty minutes.')
assert.equal(score, null)
})
test('grounding is null, not 1.0, for qualitative answers', () => {
// Folding these in as perfect would quietly inflate the aggregate.
assert.equal(numericGroundedness('Use a pressure canner.', 'context').score, null)
})
// --- pattern matching -----------------------------------------------------------------
test('matchPatterns reports both sides', () => {
const { matched, missed } = matchPatterns('boil for three minutes', [
String.raw`\b(3|three) minutes?`,
'never appears',
])
assert.equal(matched.length, 1)
assert.deepEqual(missed, ['never appears'])
})
test('pattern matching is case insensitive', () => {
assert.equal(matchPatterns('BOIL FOR 3 MINUTES', ['3 minutes']).matched.length, 1)
})
// --- scoreAnswer ------------------------------------------------------------------------
test('a correct answer passes every assertion', () => {
const s = scoreAnswer({
answer: 'Boil the water for **3 minutes** above 6,500 feet.',
context: 'boil for 3 minutes above 6,500 feet',
mustInclude: [String.raw`\b(3|three) minutes?`],
mustNotInclude: ['distill'],
expectRefusal: false,
})
assert.equal(s.correct, true)
assert.deepEqual(s.missedRequired, [])
assert.deepEqual(s.hitForbidden, [])
assert.equal(s.refusalCorrect, true)
assert.deepEqual(s.leakage, [])
})
test('a forbidden phrase fails the case even when the required one matched', () => {
const s = scoreAnswer({
answer: 'Boil for 3 minutes, or distill it.',
context: 'boil for 3 minutes',
mustInclude: ['3 minutes'],
mustNotInclude: ['distill'],
expectRefusal: false,
})
assert.equal(s.correct, false)
assert.deepEqual(s.hitForbidden, ['distill'])
})
test('declining an out-of-corpus question is scored correct', () => {
const s = scoreAnswer({
answer: "I don't have information about the warranty period.",
context: '',
mustInclude: [],
mustNotInclude: [],
expectRefusal: true,
})
assert.equal(s.refused, true)
assert.equal(s.refusalCorrect, true)
})
test('inventing an answer to an out-of-corpus question is scored incorrect', () => {
// This is the failure that matters most: a confident, fabricated reply.
const s = scoreAnswer({
answer: 'The TR-88 carries a 2 year warranty.',
context: '',
mustInclude: [],
mustNotInclude: [],
expectRefusal: true,
})
assert.equal(s.refused, false)
assert.equal(s.refusalCorrect, false)
})
test('hedging on an answerable question is scored incorrect', () => {
// The exact regression that started NOMAD's RAG work: the model has good
// context and hedges anyway.
const s = scoreAnswer({
answer: "I couldn't find specific context, but generally you boil water.",
context: 'boil for 3 minutes',
mustInclude: [],
mustNotInclude: [],
expectRefusal: false,
})
assert.equal(s.refusalCorrect, false)
assert.ok(s.leakage.length > 0, 'should also flag the retrieval narration')
})
// --- repeat aggregation --------------------------------------------------------------------
test('all-pass is stable', () => {
const s = summarizeRepeats([true, true, true])
assert.equal(s.passRate, 1)
assert.equal(s.unstable, false)
})
test('all-fail is stable', () => {
assert.equal(summarizeRepeats([false, false, false]).unstable, false)
})
test('a mixed outcome is flagged unstable and kept out of gating', () => {
const s = summarizeRepeats([true, false, true])
assert.equal(s.passes, 2)
assert.ok(Math.abs(s.passRate - 2 / 3) < 1e-9)
assert.equal(s.unstable, true)
})
test('a single repeat is never called unstable', () => {
assert.equal(summarizeRepeats([true]).unstable, false)
assert.equal(summarizeRepeats([false]).unstable, false)
})
test('numeric summary reports mean and stddev', () => {
const s = summarizeNumeric([1, 1, 1])!
assert.equal(s.mean, 1)
assert.equal(s.stddev, 0)
assert.equal(s.n, 3)
})
test('numeric summary ignores nulls', () => {
const s = summarizeNumeric([1, null, 3])!
assert.equal(s.mean, 2)
assert.equal(s.n, 2)
})
test('numeric summary is null when there is nothing to summarize', () => {
assert.equal(summarizeNumeric([null, null]), null)
})
test('stddev is non-zero when runs disagree, which is the noise warning', () => {
const s = summarizeNumeric([0, 1])!
assert.equal(s.mean, 0.5)
assert.equal(s.stddev, 0.5)
})

227
admin/tests/unit/eval_golden_set.spec.ts Normal file
View File

@ -0,0 +1,227 @@
/**
* Tests for golden-set parsing, validation, and the corpus fingerprint.
*
* The validation here is the difference between "the eval scored 0.62" and "the
* eval scored 0.62 because a golden had a typo'd doc id and could never match".
* Every failure mode below is one that would otherwise be silent.
*
* npm run test:unit
*/
import * as assert from 'node:assert/strict'
import { test } from 'node:test'
import {
assertGoldensMatchCorpus,
computeCorpusFingerprint,
GoldenSetError,
parseGoldens,
} from '../../app/utils/eval/golden_set.js'
const line = (obj: Record<string, unknown>) =>
JSON.stringify({
id: 'x',
query: 'q?',
relevantDocIds: ['doc-a'],
mustInclude: [],
mustNotInclude: [],
expectRefusal: false,
tags: [],
...obj,
})
/**
* Deterministic stand-in for sha256. Returns 64 hex chars like the real thing,
* so the truncation behaviour under test is the behaviour that ships — a short
* fake would make the length assertion vacuous.
*/
const fakeHash = (s: string) => {
let a = 0x811c9dc5
const out: string[] = []
for (let round = 0; round < 8; round++) {
for (const ch of s) a = Math.imul(a ^ ch.charCodeAt(0), 0x01000193) >>> 0
a = (a ^ round) >>> 0
out.push(a.toString(16).padStart(8, '0'))
}
return out.join('')
}
// --- parsing -----------------------------------------------------------------
test('parses a well-formed golden', () => {
const [g] = parseGoldens(line({ id: 'water-01', query: 'How long?', tags: ['numeric'] }))
assert.equal(g.id, 'water-01')
assert.equal(g.query, 'How long?')
assert.deepEqual(g.tags, ['numeric'])
assert.deepEqual(g.turns, [])
})
test('skips blank lines and // comments', () => {
const jsonl = ['', '// a note', line({ id: 'a' }), ' ', line({ id: 'b' })].join('\n')
assert.equal(parseGoldens(jsonl).length, 2)
})
test('rejects duplicate ids', () => {
const jsonl = [line({ id: 'dup' }), line({ id: 'dup' })].join('\n')
assert.throws(() => parseGoldens(jsonl), /duplicate id "dup"/)
})
test('rejects malformed JSON with the line number', () => {
assert.throws(() => parseGoldens('{not json'), /goldens:1: not valid JSON/)
})
test('rejects a missing query', () => {
assert.throws(() => parseGoldens(JSON.stringify({ id: 'a', relevantDocIds: ['d'] })), /missing "query"/)
})
test('rejects an empty query', () => {
assert.throws(() => parseGoldens(line({ query: ' ' })), /"query" must be a non-empty string/)
})
// --- the two contradictions that would silently score wrong -------------------
test('rejects a refusal case that also lists relevant documents', () => {
// One of the two is wrong, and either way the case is scored against a
// contradiction. Better to fail at load than to quietly grade nonsense.
assert.throws(
() => parseGoldens(line({ expectRefusal: true, relevantDocIds: ['doc-a'] })),
/expects a refusal but also lists relevant documents/
)
})
test('rejects a non-refusal case with no relevant documents', () => {
assert.throws(
() => parseGoldens(line({ expectRefusal: false, relevantDocIds: [] })),
/can never be scored/
)
})
test('accepts a refusal case with no relevant documents', () => {
const [g] = parseGoldens(line({ expectRefusal: true, relevantDocIds: [] }))
assert.equal(g.expectRefusal, true)
assert.deepEqual(g.relevantDocIds, [])
})
// --- regex validation ---------------------------------------------------------
test('compiles mustInclude patterns at load so a bad regex fails immediately', () => {
assert.throws(() => parseGoldens(line({ mustInclude: ['(unclosed'] })), /invalid regex/)
})
test('compiles mustNotInclude patterns too', () => {
assert.throws(() => parseGoldens(line({ mustNotInclude: ['[z-a]'] })), /invalid regex/)
})
test('accepts alternation, which is how one entry covers "3 minutes" and "three minutes"', () => {
const [g] = parseGoldens(line({ mustInclude: [String.raw`\b(3|three) minutes?`] }))
assert.match('boil for three minutes', new RegExp(g.mustInclude[0], 'i'))
assert.match('boil for 3 minutes', new RegExp(g.mustInclude[0], 'i'))
})
// --- turns --------------------------------------------------------------------
test('accepts multi-turn history', () => {
const [g] = parseGoldens(
line({ turns: [{ role: 'user', content: 'first' }, { role: 'assistant', content: 'reply' }] })
)
assert.equal(g.turns.length, 2)
})
test('rejects a turn with a bogus role', () => {
assert.throws(() => parseGoldens(line({ turns: [{ role: 'system', content: 'x' }] })), /"turns" must be/)
})
test('rejects an empty golden file rather than silently reporting a perfect score', () => {
assert.throws(() => parseGoldens('\n\n// only comments\n'), /no goldens found/)
})
// --- corpus cross-check --------------------------------------------------------
test('accepts goldens whose documents all exist', () => {
const goldens = parseGoldens(line({ relevantDocIds: ['doc-a', 'doc-b'] }))
assert.doesNotThrow(() => assertGoldensMatchCorpus(goldens, ['doc-a', 'doc-b', 'doc-c']))
})
test('rejects a golden pointing at a document that is not in the corpus', () => {
// A typo'd doc id makes recall unhittable for that question forever, and the
// only symptom is a slightly lower score. This is the check that catches it.
const goldens = parseGoldens(line({ relevantDocIds: ['doc-typo'] }))
assert.throws(
() => assertGoldensMatchCorpus(goldens, ['doc-a']),
(err: unknown) => err instanceof GoldenSetError && /doc-typo/.test((err as Error).message)
)
})
// --- fingerprint ----------------------------------------------------------------
const inputs = (over: Record<string, unknown> = {}) => ({
documents: new Map([
['a', 'alpha'],
['b', 'beta'],
]),
chunkTokens: 1500,
chunkOverlapTokens: 150,
charToTokenRatio: 2,
embeddingModel: 'nomic-embed-text:v1.5',
embeddingDimension: 768,
...over,
})
test('fingerprint is stable across runs', () => {
assert.equal(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(inputs(), fakeHash))
})
test('fingerprint ignores document insertion order', () => {
const reversed = inputs({
documents: new Map([
['b', 'beta'],
['a', 'alpha'],
]),
})
assert.equal(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(reversed, fakeHash))
})
test('fingerprint changes when a document changes', () => {
const edited = inputs({ documents: new Map([['a', 'alpha!'], ['b', 'beta']]) })
assert.notEqual(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(edited, fakeHash))
})
test('fingerprint changes when a document is added', () => {
const added = inputs({ documents: new Map([['a', 'alpha'], ['b', 'beta'], ['c', 'gamma']]) })
assert.notEqual(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(added, fakeHash))
})
test('fingerprint changes when the chunk size changes', () => {
// This is the one that matters most: re-chunking invalidates every prior
// score, and the fingerprint is what stops us comparing across the change.
assert.notEqual(
computeCorpusFingerprint(inputs(), fakeHash),
computeCorpusFingerprint(inputs({ chunkTokens: 512 }), fakeHash)
)
})
test('fingerprint changes when the token-estimate ratio changes', () => {
// CHAR_TO_TOKEN_RATIO feeds the chunker as a character count, so changing it
// silently re-chunks the corpus. It must be in the fingerprint.
assert.notEqual(
computeCorpusFingerprint(inputs(), fakeHash),
computeCorpusFingerprint(inputs({ charToTokenRatio: 3 }), fakeHash)
)
})
test('fingerprint changes when the embedding model changes', () => {
assert.notEqual(
computeCorpusFingerprint(inputs(), fakeHash),
computeCorpusFingerprint(inputs({ embeddingModel: 'mxbai-embed-large' }), fakeHash)
)
})
test('fingerprint is short enough to paste into a filename', () => {
assert.equal(computeCorpusFingerprint(inputs(), fakeHash).length, 16)
})
test('document contents cannot collide by concatenation', () => {
// Naive joining lets {a: "xy", b: ""} hash the same as {a: "x", b: "y"}.
const one = inputs({ documents: new Map([['a', 'xy'], ['b', '']]) })
const two = inputs({ documents: new Map([['a', 'x'], ['b', 'y']]) })
assert.notEqual(computeCorpusFingerprint(one, fakeHash), computeCorpusFingerprint(two, fakeHash))
})

172
admin/tests/unit/eval_report.spec.ts Normal file
View File

@ -0,0 +1,172 @@
/**
* Tests for baseline diffing and the regression gate.
*
* This is the logic that decides whether a pull request is blocked, so the
* cases below focus on the two ways a gate loses trust: firing on noise, and
* comparing runs that were never comparable in the first place.
*
* npm run test:unit
*/
import * as assert from 'node:assert/strict'
import { test } from 'node:test'
import {
compareReports,
DEFAULT_TOLERANCE,
flattenByK,
renderComparisonMarkdown,
type EvalReport,
} from '../../app/utils/eval/report.js'
const report = (
metrics: Record<string, number | null>,
over: Partial<EvalReport['meta']> = {}
): EvalReport => ({
meta: {
kind: 'retrieval',
createdAt: '2026-08-12T00:00:00.000Z',
corpusFingerprint: 'abc123',
gitSha: 'deadbeef',
gitBranch: 'dev',
gitDirty: false,
nomadVersion: '1.34.0',
platform: { cpuArchitecture: 'x64', osName: 'Linux', nodeVersion: 'v24' },
params: {},
...over,
},
metrics,
byTag: {},
cases: [],
})
// --- the comparability veto ------------------------------------------------------
test('refuses to compare across corpus fingerprints', () => {
// Diffing runs against different corpora would manufacture a regression out
// of a chunk-size change. Refusing is the correct behaviour.
const result = compareReports(
report({ mrr: 0.9 }),
report({ mrr: 0.5 }, { corpusFingerprint: 'different' })
)
assert.equal(result.comparable, false)
assert.match(result.incomparableReason!, /fingerprint changed/)
assert.deepEqual(result.regressions, [])
})
test('refuses to compare a retrieval report against a generation report', () => {
const result = compareReports(report({ mrr: 0.9 }), report({ correctness: 0.9 }, { kind: 'generation' }))
assert.equal(result.comparable, false)
assert.match(result.incomparableReason!, /retrieval report against a generation/)
})
test('compares happily when the fingerprints match', () => {
assert.equal(compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 })).comparable, true)
})
// --- direction awareness -----------------------------------------------------------
test('a drop in a higher-is-better metric is a regression', () => {
const r = compareReports(report({ 'recall@5': 0.9 }), report({ 'recall@5': 0.5 }))
assert.equal(r.regressions.length, 1)
assert.equal(r.regressions[0].metric, 'recall@5')
})
test('a rise in a lower-is-better metric is a regression', () => {
// leakageRate going up means the model started narrating retrieval again.
const r = compareReports(report({ leakageRate: 0.0 }), report({ leakageRate: 0.4 }))
assert.equal(r.regressions.length, 1)
assert.equal(r.regressions[0].metric, 'leakageRate')
})
test('a fall in a lower-is-better metric is an improvement, not a regression', () => {
const r = compareReports(report({ leakageRate: 0.4 }), report({ leakageRate: 0.0 }))
assert.deepEqual(r.regressions, [])
assert.equal(r.improvements.length, 1)
})
test('emptyRateOnAnswerable is treated as lower-is-better', () => {
const r = compareReports(report({ emptyRateOnAnswerable: 0.1 }), report({ emptyRateOnAnswerable: 0.6 }))
assert.equal(r.regressions.length, 1)
})
test('an unknown metric defaults to higher-is-better rather than being ignored', () => {
const r = compareReports(report({ somethingNew: 0.9 }), report({ somethingNew: 0.1 }))
assert.equal(r.regressions.length, 1)
})
// --- the tolerance band ---------------------------------------------------------------
test('a movement inside the tolerance band is neither a regression nor an improvement', () => {
// A gate that fires on 0.001 gets switched off within a week.
const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 - DEFAULT_TOLERANCE / 2 }))
assert.deepEqual(r.regressions, [])
assert.deepEqual(r.improvements, [])
})
test('a movement exactly at the tolerance is not yet a regression', () => {
const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 - DEFAULT_TOLERANCE }))
assert.deepEqual(r.regressions, [])
})
test('tolerance is configurable', () => {
const loose = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.85 }), 0.1)
const tight = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.85 }), 0.01)
assert.deepEqual(loose.regressions, [])
assert.equal(tight.regressions.length, 1)
})
// --- nulls and asymmetric metric sets ---------------------------------------------------
test('a null on either side is never a regression', () => {
// "not measurable in this run" must not read as "collapsed to zero".
assert.deepEqual(compareReports(report({ mrr: 0.9 }), report({ mrr: null })).regressions, [])
assert.deepEqual(compareReports(report({ mrr: null }), report({ mrr: 0.1 })).regressions, [])
})
test('a metric present only in the current report is marked new, not regressed', () => {
const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9, ndcgNew: 0.1 }))
const added = r.deltas.find((d) => d.metric === 'ndcgNew')!
assert.equal(added.onlyIn, 'current')
assert.equal(added.regressed, false)
})
test('a metric dropped from the current report is marked removed', () => {
const r = compareReports(report({ mrr: 0.9, gone: 0.5 }), report({ mrr: 0.9 }))
const removed = r.deltas.find((d) => d.metric === 'gone')!
assert.equal(removed.onlyIn, 'baseline')
assert.equal(removed.regressed, false)
})
// --- helpers and rendering -------------------------------------------------------------
test('flattenByK produces name@k keys', () => {
assert.deepEqual(flattenByK('recall', { 1: 0.5, 5: 0.9 }), { 'recall@1': 0.5, 'recall@5': 0.9 })
})
test('markdown leads with the regression count', () => {
const r = compareReports(report({ 'recall@5': 0.9 }), report({ 'recall@5': 0.4 }))
const md = renderComparisonMarkdown('base.json', 'cur.json', r)
assert.match(md, /## 1 regression\(s\)/)
assert.match(md, /\*\*REGRESSED\*\*/)
})
test('markdown says so plainly when nothing regressed', () => {
const md = renderComparisonMarkdown('b.json', 'c.json', compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 })))
assert.match(md, /## No regressions/)
})
test('markdown explains an incomparable pair instead of printing an empty table', () => {
const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 }, { corpusFingerprint: 'other' }))
const md = renderComparisonMarkdown('b.json', 'c.json', r)
assert.match(md, /## Not comparable/)
assert.match(md, /fingerprint changed/)
})
test('markdown orders the worst regression first', () => {
const r = compareReports(
report({ small: 0.9, big: 0.9 }),
report({ small: 0.85, big: 0.2 })
)
const md = renderComparisonMarkdown('b.json', 'c.json', r)
assert.ok(md.indexOf('| big |') < md.indexOf('| small |'), 'worst regression should come first')
})

264
admin/tests/unit/eval_retrieval_metrics.spec.ts Normal file
View File

@ -0,0 +1,264 @@
/**
* Tests for the retrieval metrics.
*
* Expected values are hand-computed from the formulas in the module doc, not
* captured from a run — a snapshot test of a wrong implementation just freezes
* the wrong answer. nDCG in particular is easy to get subtly wrong (log base,
* off-by-one in the rank, what the ideal ranking is normalized against), and a
* silently wrong metric is worse than no metric.
*
* npm run test:unit
*/
import * as assert from 'node:assert/strict'
import { test } from 'node:test'
import {
aggregate,
aggregateByTag,
describeScores,
hitRateAtK,
meanOf,
ndcgAtK,
precisionAtK,
recallAtK,
reciprocalRank,
scoreCase,
toDocumentRanking,
type RetrievalCase,
type ScoredChunk,
} from '../../app/utils/eval/retrieval_metrics.js'
/** Build a ranked chunk list from doc ids, best first. */
const chunks = (...docIds: Array<string | null>): ScoredChunk[] =>
docIds.map((docId, i) => ({ docId, score: 1 - i * 0.05, semanticScore: 0.9 - i * 0.05 }))
const close = (actual: number | null, expected: number, msg?: string) => {
assert.ok(actual !== null, msg ?? 'expected a value, got null')
assert.ok(Math.abs(actual - expected) < 1e-9, `${msg ?? ''} expected ${expected}, got ${actual}`)
}
// --- document ranking ---------------------------------------------------------
test('document ranking keeps each document at its best rank', () => {
assert.deepEqual(toDocumentRanking(chunks('a', 'b', 'a', 'c', 'b')), ['a', 'b', 'c'])
})
test('document ranking drops chunks with no resolvable document', () => {
assert.deepEqual(toDocumentRanking(chunks('a', null, 'b')), ['a', 'b'])
})
// --- recall -------------------------------------------------------------------
test('recall@k finds both documents of a multi-hop question', () => {
close(recallAtK(chunks('a', 'x', 'b'), ['a', 'b'], 5), 1)
})
test('recall@k is partial when only one required document is in the window', () => {
// 'b' sits at rank 3, outside k=2. One of two relevant docs found.
close(recallAtK(chunks('a', 'x', 'b'), ['a', 'b'], 2), 0.5)
})
test('recall@k counts distinct documents, not chunks', () => {
// Three chunks, all from 'a'. That is one document found, not three.
close(recallAtK(chunks('a', 'a', 'a'), ['a', 'b'], 5), 0.5)
})
test('recall is null for out-of-corpus cases rather than 0', () => {
// Scoring these as 0 would drag the mean down for questions that are
// supposed to retrieve nothing — punishing correct behaviour.
assert.equal(recallAtK(chunks('x'), [], 5), null)
})
test('recall is 0 when nothing relevant was retrieved', () => {
close(recallAtK(chunks('x', 'y'), ['a'], 5), 0)
})
// --- hit rate -----------------------------------------------------------------
test('hit rate is 1 when any relevant document appears', () => {
assert.equal(hitRateAtK(chunks('x', 'a'), ['a', 'b'], 5), 1)
})
test('hit rate and recall diverge on multi-hop, which is the point of having both', () => {
const retrieved = chunks('a', 'x', 'y')
assert.equal(hitRateAtK(retrieved, ['a', 'b'], 5), 1)
close(recallAtK(retrieved, ['a', 'b'], 5), 0.5)
})
test('hit rate respects the k window', () => {
assert.equal(hitRateAtK(chunks('x', 'y', 'a'), ['a'], 2), 0)
})
// --- precision ----------------------------------------------------------------
test('precision@k is chunk level and deliberately not deduped', () => {
// Two of four injected chunks are noise, regardless of how many documents
// they came from — the small model pays for all four.
close(precisionAtK(chunks('a', 'a', 'x', 'y'), ['a'], 4), 0.5)
})
test('precision divides by what was actually retrieved, not by k', () => {
// Returning 2 good chunks should score 1.0, not 0.4 — declining to pad the
// context with noise is correct behaviour and must not be penalised.
close(precisionAtK(chunks('a', 'a'), ['a'], 5), 1)
})
test('precision is 0 when nothing was retrieved', () => {
close(precisionAtK([], ['a'], 5), 0)
})
// --- reciprocal rank ----------------------------------------------------------
test('reciprocal rank is 1 when the first chunk is relevant', () => {
close(reciprocalRank(chunks('a', 'x'), ['a']), 1)
})
test('reciprocal rank is 1/3 when the first relevant chunk is third', () => {
close(reciprocalRank(chunks('x', 'y', 'a'), ['a']), 1 / 3)
})
test('reciprocal rank is 0 when nothing relevant was retrieved', () => {
close(reciprocalRank(chunks('x', 'y'), ['a']), 0)
})
// --- nDCG ---------------------------------------------------------------------
test('nDCG is 1 when the single relevant document ranks first', () => {
close(ndcgAtK(chunks('a', 'x', 'y'), ['a'], 5), 1)
})
test('nDCG at rank 2 equals 1/log2(3)', () => {
// DCG = 1/log2(2+1) = 1/1.58496 = 0.63093
// IDCG = 1/log2(1+1) = 1
close(ndcgAtK(chunks('x', 'a'), ['a'], 5), 1 / Math.log2(3))
})
test('nDCG normalizes against the known relevant count, not the retrieved set', () => {
// One of three required documents, ranked first.
// DCG = 1/log2(2) = 1
// IDCG = 1/log2(2) + 1/log2(3) + 1/log2(4) = 1 + 0.63093 + 0.5 = 2.13093
// Normalizing against the retrieved set instead would report a perfect 1.0
// for a run that missed two thirds of the answer.
const idcg = 1 + 1 / Math.log2(3) + 1 / Math.log2(4)
close(ndcgAtK(chunks('a', 'x', 'y'), ['a', 'b', 'c'], 5), 1 / idcg)
})
test('nDCG punishes ordering even when recall is unchanged', () => {
// This is the regression nDCG exists to catch: same documents retrieved,
// pushed down the list, recall identical.
const good = chunks('a', 'b', 'x', 'y')
const bad = chunks('x', 'y', 'a', 'b')
close(recallAtK(good, ['a', 'b'], 5), 1)
close(recallAtK(bad, ['a', 'b'], 5), 1)
const nGood = ndcgAtK(good, ['a', 'b'], 5)!
const nBad = ndcgAtK(bad, ['a', 'b'], 5)!
assert.equal(nGood, 1)
assert.ok(nBad < nGood, `expected ${nBad} < ${nGood}`)
})
test('nDCG ideal ranking is capped at k', () => {
// Three relevant docs but k=1: the best achievable is one hit at rank 1.
close(ndcgAtK(chunks('a', 'b', 'c'), ['a', 'b', 'c'], 1), 1)
})
test('nDCG collapses duplicate chunks from the same document', () => {
// Three chunks of 'a' must not be credited as three separate hits.
const idcg = 1 + 1 / Math.log2(3)
close(ndcgAtK(chunks('a', 'a', 'a'), ['a', 'b'], 5), 1 / idcg)
})
test('nDCG is 0 when nothing relevant is retrieved', () => {
close(ndcgAtK(chunks('x', 'y'), ['a'], 5), 0)
})
// --- score distribution ---------------------------------------------------------
test('describeScores reports values that actually occurred', () => {
const d = describeScores([0.1, 0.2, 0.3, 0.4, 0.5])!
assert.equal(d.count, 5)
assert.equal(d.min, 0.1)
assert.equal(d.max, 0.5)
assert.equal(d.median, 0.3)
close(d.mean, 0.3)
// Nearest-rank, so every reported percentile is a real observation.
assert.ok([0.1, 0.2, 0.3, 0.4, 0.5].includes(d.p10))
assert.ok([0.1, 0.2, 0.3, 0.4, 0.5].includes(d.p90))
})
test('describeScores returns null for an empty sample', () => {
assert.equal(describeScores([]), null)
})
test('meanOf ignores nulls and returns null when everything is null', () => {
close(meanOf([1, null, 3]), 2)
assert.equal(meanOf([null, null]), null)
})
// --- aggregation ----------------------------------------------------------------
const mkCase = (over: Partial<RetrievalCase>): RetrievalCase => ({
id: 'c',
tags: [],
retrieved: chunks('a'),
relevantDocIds: ['a'],
expectRefusal: false,
...over,
})
test('aggregate separates answerable cases from refusal cases', () => {
const cases = [
mkCase({ id: 'q1', retrieved: chunks('a'), relevantDocIds: ['a'] }),
mkCase({ id: 'q2', retrieved: [], relevantDocIds: [], expectRefusal: true }),
]
const results = cases.map((c) => scoreCase(c))
const agg = aggregate(cases, results)
assert.equal(agg.cases, 2)
assert.equal(agg.answerable, 1)
// The refusal case contributes null to recall, so the mean is over q1 alone.
close(agg.recall[5], 1)
})
test('empty rate on answerable questions surfaces an over-tight threshold', () => {
const cases = [
mkCase({ id: 'q1', retrieved: [] }),
mkCase({ id: 'q2', retrieved: chunks('a') }),
]
const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
close(agg.emptyRateOnAnswerable, 0.5)
})
test('non-empty rate on refusal questions surfaces an over-loose threshold', () => {
const cases = [
mkCase({ id: 'r1', retrieved: chunks('x'), relevantDocIds: [], expectRefusal: true }),
mkCase({ id: 'r2', retrieved: [], relevantDocIds: [], expectRefusal: true }),
]
const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
close(agg.nonEmptyRateOnRefusal, 0.5)
})
test('score distributions split relevant from irrelevant chunks', () => {
const cases = [mkCase({ retrieved: chunks('a', 'x', 'y'), relevantDocIds: ['a'] })]
const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
assert.equal(agg.relevantScores!.count, 1)
assert.equal(agg.irrelevantScores!.count, 2)
// The relevant chunk ranked first, so it should score above the noise.
assert.ok(agg.relevantScores!.median > agg.irrelevantScores!.median)
})
test('refusal cases contribute only to the irrelevant score population', () => {
const cases = [mkCase({ retrieved: chunks('x', 'y'), relevantDocIds: [], expectRefusal: true })]
const agg = aggregate(cases, cases.map((c) => scoreCase(c)))
assert.equal(agg.relevantScores, null)
assert.equal(agg.irrelevantScores!.count, 2)
})
test('per-tag aggregation slices the same cases without recomputing them wrong', () => {
const cases = [
mkCase({ id: 'q1', tags: ['single-hop'], retrieved: chunks('a'), relevantDocIds: ['a'] }),
mkCase({ id: 'q2', tags: ['multi-hop'], retrieved: chunks('a'), relevantDocIds: ['a', 'b'] }),
]
const byTag = aggregateByTag(cases, cases.map((c) => scoreCase(c)))
close(byTag['single-hop'].recall[5], 1)
close(byTag['multi-hop'].recall[5], 0.5)
assert.equal(byTag['single-hop'].cases, 1)
})

84
admin/tests/unit/eval_source_guard.spec.ts Normal file
View File

@ -0,0 +1,84 @@
/**
* Tests for the eval-corpus leak guard.
*
* The eval corpus shares the `nomad_knowledge_base` Qdrant collection with the
* developer's real documents, isolated by a `collection: __nomad_eval__` payload
* filter. That filter is applied by Qdrant during search and does hold — but a
* harness that depends on a filter must be able to *prove* the filter held, not
* assume it.
*
* `docIdFromSource` is that proof: anything it cannot resolve to a corpus
* document is counted as an unresolved chunk, and a non-zero count fails the
* run. These tests exist because the first version checked only for a `.md`
* extension, which silently accepted a developer's own markdown — and NOMAD
* embeds its own `admin/docs/*.md` on first run, so that was not hypothetical.
*
* npm run test:eval
*/
import * as assert from 'node:assert/strict'
import { test } from 'node:test'
import { join, resolve } from 'node:path'
import { docIdFromSource } from '../../app/utils/eval/corpus_source.js'
const CORPUS = resolve('/srv/nomad/admin/tests/eval/corpus')
const inCorpus = (name: string) => join(CORPUS, name)
const resolveId = (source: unknown) => docIdFromSource(source, CORPUS)
test('resolves a genuine corpus document', () => {
assert.equal(resolveId(inCorpus('water-boiling.md')), 'water-boiling')
})
test('resolves every corpus document to its filename without the extension', () => {
assert.equal(resolveId(inCorpus('regions-elevation-table.md')), 'regions-elevation-table')
assert.equal(resolveId(inCorpus('equipment-tr88-pump.md')), 'equipment-tr88-pump')
})
test("rejects the developer's own markdown, even though it ends in .md", () => {
// The regression this guard exists for. NOMAD embeds admin/docs/*.md into the
// knowledge base on first run, so if the collection filter ever leaked, this
// is the exact shape that would come back. Resolving it to "faq" would have
// been counted as a merely-irrelevant chunk and quietly lowered precision
// instead of failing the run.
assert.equal(resolveId('/srv/nomad/admin/docs/faq.md'), null)
assert.equal(resolveId('/srv/nomad/admin/docs/release-notes.md'), null)
})
test('rejects an uploaded knowledge-base file', () => {
assert.equal(resolveId('/srv/nomad/admin/storage/kb_uploads/notes-abc123.txt'), null)
assert.equal(resolveId('/srv/nomad/admin/storage/kb_uploads/notes-abc123.md'), null)
})
test('rejects a ZIM article source', () => {
assert.equal(resolveId('/srv/nomad/admin/storage/zim/wikipedia_en_100_mini.zim'), null)
})
test('rejects a sibling directory whose path merely starts with the corpus path', () => {
// "…/corpus-backup/x.md" shares a string prefix with "…/corpus". Without the
// trailing separator in the check this would pass.
assert.equal(resolveId('/srv/nomad/admin/tests/eval/corpus-backup/water-boiling.md'), null)
})
test('rejects a path that escapes the corpus via traversal', () => {
assert.equal(resolveId(join(CORPUS, '..', '..', '..', 'docs', 'faq.md')), null)
})
test('rejects a non-markdown file inside the corpus directory', () => {
assert.equal(resolveId(inCorpus('README.txt')), null)
})
test('rejects a missing or non-string source', () => {
assert.equal(resolveId(undefined), null)
assert.equal(resolveId(null), null)
assert.equal(resolveId(42), null)
assert.equal(resolveId(''), null)
})
test('rejects the corpus directory itself', () => {
assert.equal(resolveId(CORPUS), null)
})
test('accepts a nested document, should the corpus ever grow subdirectories', () => {
assert.equal(resolveId(inCorpus('water/boiling.md')), 'boiling')
})

189
admin/tests/unit/rag_pipeline_prompt.spec.ts Normal file
View File

@ -0,0 +1,189 @@
/**
* Characterization tests for the prompt-assembly helpers extracted out of
* OllamaController into RagPipelineService.
*
* These lock in the *current* behaviour, quirks included, so the extraction is
* provably a no-op and so any later deliberate change to context budgeting or
* the num_ctx ladder shows up as a failing test rather than a silent shift in
* answer quality.
*
* Pure functions only — no MySQL, Redis, Qdrant, or Ollama needed:
* npm run test:unit
*/
import * as assert from 'node:assert/strict'
import { test } from 'node:test'
import {
buildContextBlock,
deriveNumCtx,
getContextLimitsForModel,
NUM_CTX_TRIGGER_TOKENS,
PROMPT_CHARS_PER_TOKEN,
trimToContextBudget,
type ContextLimitTier,
} from '../../app/utils/rag_prompt.js'
/** Mirrors RAG_CONTEXT_LIMITS in constants/ollama.ts. Passed in explicitly so
* these tests need no AdonisJS-flavoured imports, per the kb_ratio_lookup
* precedent — and so a change to the shipped tiers shows up here as a
* deliberate edit rather than a silently-passing test. */
const TIERS: ContextLimitTier[] = [
{ maxParams: 3, maxResults: 2, maxTokens: 1000 },
{ maxParams: 8, maxResults: 4, maxTokens: 2500 },
{ maxParams: Infinity, maxResults: 5, maxTokens: 0 },
]
const limits = (model: string) => getContextLimitsForModel(model, TIERS)
const chunk = (text: string, metadata: Record<string, any> = {}) => ({
text,
score: 0.5,
metadata,
})
// --- getContextLimitsForModel ------------------------------------------------
test('context limits: 1-3B models get the tightest budget', () => {
assert.deepEqual(limits('llama3.2:1b'), { maxResults: 2, maxTokens: 1000 })
assert.deepEqual(limits('qwen2.5:3b'), { maxResults: 2, maxTokens: 1000 })
})
test('context limits: fractional sizes parse correctly', () => {
// "1.5b" must read as 1.5, not 1 or 15 — it decides which tier the model lands in.
assert.deepEqual(limits('qwen2.5:1.5b'), { maxResults: 2, maxTokens: 1000 })
})
test('context limits: 4-8B tier', () => {
assert.deepEqual(limits('llama3.1:8b'), { maxResults: 4, maxTokens: 2500 })
})
test('context limits: 13B+ is uncapped', () => {
assert.deepEqual(limits('llama2:70b'), { maxResults: 5, maxTokens: 0 })
})
test('context limits: unparseable model name is assumed to be 8B', () => {
// Documented quirk, not an endorsement: "phi3" has no size token, so it is
// handed the 4-8B budget regardless of what it actually is.
assert.deepEqual(limits('phi3'), { maxResults: 4, maxTokens: 2500 })
})
test('context limits: quantization suffixes do not confuse the size parse', () => {
assert.deepEqual(limits('llama3.1:8b-text-q4_1'), {
maxResults: 4,
maxTokens: 2500,
})
})
// --- trimToContextBudget -----------------------------------------------------
test('trim: caps the number of results', () => {
const docs = [chunk('a'), chunk('b'), chunk('c'), chunk('d')]
const out = trimToContextBudget(docs, { maxResults: 2, maxTokens: 0 })
assert.equal(out.length, 2)
assert.deepEqual(
out.map((d: { text: string }) => d.text),
['a', 'b']
)
})
test('trim: maxTokens of 0 means uncapped', () => {
const docs = [chunk('x'.repeat(50_000)), chunk('y'.repeat(50_000))]
const out = trimToContextBudget(docs, { maxResults: 5, maxTokens: 0 })
assert.equal(out.length, 2)
})
test('trim: the top result survives even when it alone blows the budget', () => {
// This is the guard that stops a small model from getting *no* context at all
// when the single best chunk happens to be enormous.
const huge = chunk('x'.repeat(100_000))
const out = trimToContextBudget([huge, chunk('small')], { maxResults: 5, maxTokens: 1000 })
assert.equal(out.length, 1)
assert.equal(out[0].text.length, 100_000)
})
test('trim: drops later results once the character cap is exceeded', () => {
const capChars = 1000 * PROMPT_CHARS_PER_TOKEN // 3500
const docs = [chunk('a'.repeat(1000)), chunk('b'.repeat(1000)), chunk('c'.repeat(3000))]
const out = trimToContextBudget(docs, { maxResults: 5, maxTokens: 1000 })
// Running totals: 1000 (kept, idx 0), 2000 (<= 3500, kept), 5000 (> 3500, dropped).
assert.deepEqual(
out.map((d: { text: string }) => d.text[0]),
['a', 'b']
)
assert.ok(capChars === 3500)
})
test('trim: the count cap is applied before the token cap', () => {
const docs = [chunk('a'), chunk('b'), chunk('c')]
const out = trimToContextBudget(docs, { maxResults: 1, maxTokens: 1000 })
assert.equal(out.length, 1)
})
// --- buildContextBlock -------------------------------------------------------
test('context block: numbers each chunk from 1', () => {
const out = buildContextBlock([chunk('first'), chunk('second')])
assert.equal(out, '[Context 1]\nfirst\n\n[Context 2]\nsecond')
})
test('context block: labels with full_title when present', () => {
const out = buildContextBlock([chunk('body', { full_title: 'Water - Boiling' })])
assert.equal(out, '[Context 1 — Water - Boiling]\nbody')
})
test('context block: falls back to article_title', () => {
const out = buildContextBlock([chunk('body', { article_title: 'Water' })])
assert.equal(out, '[Context 1 — Water]\nbody')
})
test('context block: full_title wins over article_title', () => {
const out = buildContextBlock([chunk('body', { full_title: 'A - B', article_title: 'A' })])
assert.equal(out, '[Context 1 — A - B]\nbody')
})
test('context block: never leaks the relevance score to the model', () => {
// Deliberate: nomic cosine scores for genuinely relevant passages sit around
// 0.4-0.6, and showing the model "42%" primes it to distrust correct context.
const out = buildContextBlock([chunk('body', { source: 'f.md', semantic_score: 0.42 })])
assert.ok(!out.includes('0.42'))
assert.ok(!out.includes('42'))
})
// --- deriveNumCtx ------------------------------------------------------------
const sys = (chars: number) => ({ role: 'system' as const, content: 'x'.repeat(chars) })
test('numCtx: unset below the trigger, so the backend default applies', () => {
// Ollama's default is a silent 2048. Below the trigger we deliberately say
// nothing — a known risk, captured here so a future fix is a visible change.
assert.equal(deriveNumCtx([sys(100)]), undefined)
})
test('numCtx: exactly at the trigger is still unset (strict greater-than)', () => {
const chars = NUM_CTX_TRIGGER_TOKENS * PROMPT_CHARS_PER_TOKEN // 10500 -> exactly 3000 tokens
assert.equal(deriveNumCtx([sys(chars)]), undefined)
})
test('numCtx: one character past the trigger steps onto the ladder', () => {
const chars = NUM_CTX_TRIGGER_TOKENS * PROMPT_CHARS_PER_TOKEN + 1
assert.equal(deriveNumCtx([sys(chars)]), 8192)
})
test('numCtx: climbs the ladder as the system prompt grows', () => {
// ~7000 tokens of system prompt + 2048 headroom = 9048 -> next rung is 16384.
assert.equal(deriveNumCtx([sys(7000 * PROMPT_CHARS_PER_TOKEN)]), 16384)
})
test('numCtx: saturates at the top of the ladder rather than failing', () => {
assert.equal(deriveNumCtx([sys(1_000_000)]), 65536)
})
test('numCtx: only system messages count toward the budget', () => {
const messages = [sys(100), { role: 'user' as const, content: 'y'.repeat(1_000_000) }]
assert.equal(deriveNumCtx(messages), undefined)
})
test('numCtx: system messages are summed, not measured individually', () => {
const half = (NUM_CTX_TRIGGER_TOKENS * PROMPT_CHARS_PER_TOKEN) / 2 + 10
assert.equal(deriveNumCtx([sys(half), sys(half)]), 8192)
})

70
admin/types/rag.ts
View File

@ -1,3 +1,5 @@
import type { OllamaChatMessage } from './ollama.js'
export type EmbedJobWithProgress = {
jobId: string
fileName: string
@ -42,6 +44,74 @@ export type RerankedRAGResult = Omit<RAGResult, 'keywords'> & {
finalScore: number
}
/** One entry in a recorded retrieval stage: just enough to score a ranking. */
export type StageEntry = { source?: string; score: number }
/**
* The three ranked lists retrieval produces internally, captured so the eval
* harness can score each stage separately and show whether the heuristic
* reranker and the source-diversity penalty are earning their complexity.
*
* `dense` is the raw cosine ordering from Qdrant, `reranked` adds the
* keyword/heading boosts, `diversified` adds the same-document penalty.
*/
export type RetrievalStages = {
dense?: StageEntry[]
reranked?: StageEntry[]
diversified?: StageEntry[]
}
/**
* A chunk as returned by `RagService.searchSimilarDocuments` — the shape the
* chat pipeline consumes and the eval harness scores.
*/
export type RetrievedChunk = {
text: string
score: number
metadata?: Record<string, any>
}
/**
* Knobs on a single pipeline run. Everything is optional: the defaults
* reproduce production chat exactly. The non-default paths exist so the eval
* harness can ablate one stage at a time without a parallel implementation.
*/
export type PipelineOptions = {
topK?: number
scoreThreshold?: number
collection?: string
/** Skip the history-aware rewrite (which is an LLM call, and therefore
* non-deterministic). Retrieval then runs on the raw last user message. */
skipQueryRewrite?: boolean
/** Bypass retrieval entirely and inject these chunks as the context. Used by
* the `oracle` eval mode to isolate generation quality from retrieval. */
oracleContext?: RetrievedChunk[]
/** Ignore the user's NOMAD.md. Off in production; on in evals, where a
* developer's personal instructions would silently skew every result. */
skipNomadMd?: boolean
}
/**
* Everything the pipeline decided on the way to a prompt. The controller uses
* only `messages` and `numCtx`; the eval harness scores the rest. Returning it
* unconditionally keeps one code path for both.
*/
export type PipelineTrace = {
/** null when retrieval was skipped entirely (empty KB, or no user message). */
rewrittenQuery: string | null
/** True when the rewrite LLM call actually ran (it is skipped on turn 1). */
didRewrite: boolean
/** Everything retrieval returned, pre-trim. */
retrieved: RetrievedChunk[]
/** What actually made it into the prompt, post model-size trim. */
injected: RetrievedChunk[]
/** The exact payload handed to Ollama. */
messages: OllamaChatMessage[]
numCtx: number | undefined
contextLimits: { maxResults: number; maxTokens: number }
timings: { rewriteMs: number; retrievalMs: number }
}
export type FileWarning =
| { kind: 'zero_chunks'; fileSizeBytes: number }
| { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number }