diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 079bfa1..078bda7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,7 +95,21 @@ Because NOMAD relies heavily on Docker, we actually recommend against installing 3. **Make your changes.** Follow existing code style and conventions. Test your changes locally against a running NOMAD instance before submitting. -4. **Add release notes** (see [Release Notes](#release-notes) below). +4. **If you touched the AI Assistant or RAG, measure it.** Anything affecting + chunking, embedding, retrieval, reranking, thresholds, the system prompts, or + context assembly should be backed by numbers rather than a spot check — + "it seemed better in the chat window" is how a regression ships. From `admin/`: + + ```bash + node ace eval:corpus --ingest # once; safe, never touches your real knowledge base + node ace eval:retrieval --ablate # seconds, deterministic, no chat model + node ace eval:generation --model= --all-modes + ``` + + Include the before/after numbers in your pull request. See + [`admin/tests/eval/README.md`](admin/tests/eval/README.md) for what the + metrics mean, how the three generation modes separate a code bug from a model + that is simply too small, and the harness's known limitations. 5. **Commit your changes** using [Conventional Commits](#commit-messages). diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index f342538..30c4e40 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -1,7 +1,7 @@ import { ChatService } from '#services/chat_service' import { DockerService } from '#services/docker_service' -import { NomadMdService } from '#services/nomad_md_service' import { OllamaService } from '#services/ollama_service' +import { RagPipelineService } from '#services/rag_pipeline_service' import { RagService } from '#services/rag_service' import Service from '#models/service' import KVStore from '#models/kv_store' @@ -10,10 +10,8 @@ import { chatSchema, getAvailableModelsSchema, unloadChatModelsSchema } from '#v import { assertNotCloudMetadataUrl } from '#validators/common' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' -import { RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js' import { SERVICE_NAMES } from '../../constants/service_names.js' import logger from '@adonisjs/core/services/logger' -type Message = { role: 'system' | 'user' | 'assistant'; content: string } @inject() export default class OllamaController { @@ -21,8 +19,8 @@ export default class OllamaController { private chatService: ChatService, private dockerService: DockerService, private ollamaService: OllamaService, - private ragService: RagService, - private nomadMdService: NomadMdService + private ragPipelineService: RagPipelineService, + private ragService: RagService ) { } async availableModels({ request }: HttpContext) { @@ -62,101 +60,15 @@ export default class OllamaController { } try { - // If there are no system messages in the chat inject system prompts - const hasSystemMessage = reqData.messages.some((msg) => msg.role === 'system') - if (!hasSystemMessage) { - const systemPrompt = { - role: 'system' as const, - content: SYSTEM_PROMPTS.default, - } - logger.debug('[OllamaController] Injecting system prompt') - reqData.messages.unshift(systemPrompt) - } - - // Inject the user-managed NOMAD.md as its own leading system message so the - // user's persistent instructions take precedence, while the default - // formatting prompt and any RAG context below remain intact. A missing or - // blank file yields null and changes nothing. - const nomadPrompt = await this.nomadMdService.getSystemPrompt() - if (nomadPrompt) { - logger.debug('[OllamaController] Injecting NOMAD.md system prompt') - reqData.messages.unshift({ role: 'system' as const, content: nomadPrompt }) - } - - // Query rewriting for better RAG retrieval with manageable context - // Will return user's latest message if no rewriting is needed - const rewrittenQuery = await this.rewriteQueryWithContext(reqData.messages, reqData.model) - - logger.debug(`[OllamaController] Rewritten query for RAG: "${rewrittenQuery}"`) - if (rewrittenQuery) { - const collectionFilter: string | null = request.input('collection', null) - const relevantDocs = await this.ragService.searchSimilarDocuments( - rewrittenQuery, - 5, // Top 5 most relevant chunks - 0.3, // Minimum similarity score of 0.3 - collectionFilter ?? undefined - ) - - logger.debug(`[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${rewrittenQuery}"`) - - // If relevant context is found, inject as a system message with adaptive limits - if (relevantDocs.length > 0) { - // Determine context budget based on model size - const { maxResults, maxTokens } = this.getContextLimitsForModel(reqData.model) - let trimmedDocs = relevantDocs.slice(0, maxResults) - - // Apply token cap if set (estimate ~3.5 chars per token) - // Always include the first (most relevant) result — the cap only gates subsequent results - if (maxTokens > 0) { - const charCap = maxTokens * 3.5 - let totalChars = 0 - trimmedDocs = trimmedDocs.filter((doc, idx) => { - totalChars += doc.text.length - return idx === 0 || totalChars <= charCap - }) - } - - logger.debug( - `[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${reqData.model}, maxResults: ${maxResults}, maxTokens: ${maxTokens || 'unlimited'})` - ) - - // Label each context block with its source title when available (a neutral, - // honest provenance signal) but never the raw relevance score — nomic cosine - // scores for genuinely relevant passages sit ~0.4-0.6, and surfacing e.g. - // "42%" primes the model to distrust correct context. Scores stay in the logs - // above for debugging. - const contextText = trimmedDocs - .map((doc, idx) => { - const title = doc.metadata?.full_title || doc.metadata?.article_title - const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]` - return `${label}\n${doc.text}` - }) - .join('\n\n') - - const systemMessage = { - role: 'system' as const, - content: SYSTEM_PROMPTS.rag_context(contextText), - } - - // Insert system message at the beginning (after any existing system messages) - const firstNonSystemIndex = reqData.messages.findIndex((msg) => msg.role !== 'system') - const insertIndex = firstNonSystemIndex === -1 ? 0 : firstNonSystemIndex - reqData.messages.splice(insertIndex, 0, systemMessage) - } - } - - // If system messages are large (e.g. due to RAG context), request a context window big - // enough to fit them. Ollama respects num_ctx per-request; LM Studio ignores it gracefully. - const systemChars = reqData.messages - .filter((m) => m.role === 'system') - .reduce((sum, m) => sum + m.content.length, 0) - const estimatedSystemTokens = Math.ceil(systemChars / 3.5) - let numCtx: number | undefined - if (estimatedSystemTokens > 3000) { - const needed = estimatedSystemTokens + 2048 // leave room for conversation + response - numCtx = [8192, 16384, 32768, 65536].find((n) => n >= needed) ?? 65536 - logger.debug(`[OllamaController] Large system prompt (~${estimatedSystemTokens} tokens), requesting num_ctx: ${numCtx}`) - } + // Everything from system-prompt assembly through query rewriting, + // retrieval, context trimming and the num_ctx decision lives in + // RagPipelineService so the eval harness exercises this exact code path. + const collectionFilter: string | null = request.input('collection', null) + const trace = await this.ragPipelineService.buildPrompt(reqData.messages, reqData.model, { + collection: collectionFilter ?? undefined, + }) + reqData.messages = trace.messages + const numCtx = trace.numCtx // Check if the model supports "thinking" capability for enhanced response generation. // Thinking is only enabled when the model supports it AND the user wants it: the explicit @@ -420,88 +332,4 @@ export default class OllamaController { return models.map((m, i) => ({ ...m, thinking: thinking[i] })) } - /** - * Determines RAG context limits based on model size extracted from the model name. - * Parses size indicators like "1b", "3b", "8b", "70b" from model names/tags. - */ - private getContextLimitsForModel(modelName: string): { maxResults: number; maxTokens: number } { - // Extract parameter count from model name (e.g., "llama3.2:3b", "qwen2.5:1.5b", "gemma:7b") - const sizeMatch = modelName.match(/(\d+\.?\d*)[bB]/) - const paramBillions = sizeMatch ? parseFloat(sizeMatch[1]) : 8 // default to 8B if unknown - - for (const tier of RAG_CONTEXT_LIMITS) { - if (paramBillions <= tier.maxParams) { - return { maxResults: tier.maxResults, maxTokens: tier.maxTokens } - } - } - - // Fallback: no limits - return { maxResults: 5, maxTokens: 0 } - } - - private async rewriteQueryWithContext( - messages: Message[], - model: string - ): Promise { - const lastUserMessage = [...messages].reverse().find(msg => msg.role === 'user') - - try { - // Skip the entire RAG pipeline if there are no documents to search - const hasDocuments = await this.ragService.hasDocuments() - if (!hasDocuments) { - return null - } - - // Get recent conversation history (last 6 messages for 3 turns) - const recentMessages = messages.slice(-6) - - // Skip rewriting on the very first turn — with only one user message - // there is no prior context to fold in, so the rewrite would just echo - // the message back at the cost of an extra LLM round-trip. From the - // first follow-up onward we need the rewrite so the RAG query carries - // entities and topics from earlier turns ("the bars" → "Hershey's bars - // chocolate poisoning dog"); without it, embeddings match nothing and - // the assistant loses the thread. - const userMessages = recentMessages.filter(msg => msg.role === 'user') - if (userMessages.length < 2) { - return lastUserMessage?.content || null - } - - const conversationContext = recentMessages - .map(msg => { - const role = msg.role === 'user' ? 'User' : 'Assistant' - // Truncate assistant messages to first 200 chars to keep context manageable - const content = msg.role === 'assistant' - ? msg.content.slice(0, 200) + (msg.content.length > 200 ? '...' : '') - : msg.content - return `${role}: "${content}"` - }) - .join('\n') - - const response = await this.ollamaService.chat({ - model, - messages: [ - { - role: 'system', - content: SYSTEM_PROMPTS.query_rewrite, - }, - { - role: 'user', - content: `Conversation:\n${conversationContext}\n\nRewritten Query:`, - }, - ], - }) - - const rewrittenQuery = response.message.content.trim() - logger.info(`[RAG] Query rewritten: "${rewrittenQuery}"`) - return rewrittenQuery - } catch (error) { - logger.error( - `[RAG] Query rewriting failed: ${error instanceof Error ? error.message : error}` - ) - // Fallback to last user message if rewriting fails - return lastUserMessage?.content || null - } - } } - diff --git a/admin/app/services/eval_corpus_service.ts b/admin/app/services/eval_corpus_service.ts new file mode 100644 index 0000000..9776f86 --- /dev/null +++ b/admin/app/services/eval_corpus_service.ts @@ -0,0 +1,172 @@ +import { RagService } from '#services/rag_service' +import { inject } from '@adonisjs/core' +import logger from '@adonisjs/core/services/logger' +import { createHash } from 'node:crypto' +import { readdir, readFile } from 'node:fs/promises' +import { basename, join, resolve } from 'node:path' +import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' +import { EMBEDDING_MODEL_NAME } from '../../constants/ollama.js' +import { + assertGoldensMatchCorpus, + computeCorpusFingerprint, + parseGoldens, + type Golden, +} from '../utils/eval/golden_set.js' +import { EVAL_CORPUS_DIR } from '../utils/eval/corpus_source.js' + +/** Where the golden question set lives, relative to the app root. */ +export const EVAL_GOLDENS_DIR = 'tests/eval/goldens' + +// EVAL_CORPUS_DIR and docIdFromSource live in app/utils/eval/corpus_source.ts: +// they are pure path logic, and keeping them out of this service is what lets +// the leak guard be unit-tested without booting AdonisJS. +export { docIdFromSource, EVAL_CORPUS_DIR } from '../utils/eval/corpus_source.js' + +export type CorpusDocument = { + /** Filename without extension. This is the id goldens refer to. */ + docId: string + /** Absolute path on disk; becomes the Qdrant `source` payload. */ + path: string + text: string +} + +export type IngestSummary = { + fingerprint: string + documents: number + chunks: number + removedBeforeIngest: number + failures: Array<{ docId: string; reason: string }> +} + +/** + * Owns the frozen evaluation corpus: reading it off disk, fingerprinting it, + * and pushing it through NOMAD's real ingest path into the reserved + * `__nomad_eval__` collection tag. + * + * Ingesting through `RagService.embedAndStoreText` rather than writing vectors + * directly is the whole point — chunk size, the token-estimate ratio, the + * search_document prefix, and the embedding model are all in scope of the + * measurement, so a change to any of them shows up as a retrieval score + * movement instead of hiding. + */ +@inject() +export class EvalCorpusService { + constructor(private ragService: RagService) {} + + private corpusPath(): string { + return resolve(join(process.cwd(), EVAL_CORPUS_DIR)) + } + + /** Read every markdown document in the corpus, sorted for determinism. */ + async loadCorpus(): Promise { + const dir = this.corpusPath() + const entries = (await readdir(dir)).filter((f) => f.endsWith('.md')).sort() + if (entries.length === 0) { + throw new Error(`No corpus documents found in ${dir}`) + } + return Promise.all( + entries.map(async (file) => { + const path = join(dir, file) + return { docId: basename(file, '.md'), path, text: await readFile(path, 'utf8') } + }) + ) + } + + /** Load and validate every golden file, cross-checked against the corpus. */ + async loadGoldens(): Promise { + const dir = resolve(join(process.cwd(), EVAL_GOLDENS_DIR)) + const files = (await readdir(dir)).filter((f) => f.endsWith('.jsonl')).sort() + if (files.length === 0) throw new Error(`No golden files found in ${dir}`) + + const goldens: Golden[] = [] + const seen = new Set() + for (const file of files) { + const parsed = parseGoldens(await readFile(join(dir, file), 'utf8'), file) + for (const g of parsed) { + // parseGoldens dedupes within a file; this catches collisions across files. + if (seen.has(g.id)) throw new Error(`Duplicate golden id "${g.id}" in ${file}`) + seen.add(g.id) + goldens.push(g) + } + } + + const corpus = await this.loadCorpus() + assertGoldensMatchCorpus( + goldens, + corpus.map((d) => d.docId) + ) + return goldens + } + + /** + * Hash the corpus together with the ingest parameters that shaped it. + * Reports carry this; two reports with different fingerprints are not + * comparable and `eval:compare` refuses to pretend otherwise. + */ + async fingerprint(): Promise { + const corpus = await this.loadCorpus() + return computeCorpusFingerprint( + { + documents: new Map(corpus.map((d) => [d.docId, d.text])), + chunkTokens: RagService.TARGET_TOKENS_PER_CHUNK, + chunkOverlapTokens: RagService.CHUNK_OVERLAP_TOKENS, + charToTokenRatio: RagService.CHAR_TO_TOKEN_RATIO, + embeddingModel: EMBEDDING_MODEL_NAME, + embeddingDimension: RagService.EMBEDDING_DIMENSION, + }, + (input) => createHash('sha256').update(input).digest('hex') + ) + } + + /** Remove every eval point. Never touches user content. */ + async reset(): Promise { + const removed = await this.ragService.deleteCollectionPoints(KB_EVAL_COLLECTION) + logger.info(`[Eval] Removed ${removed} eval corpus chunks`) + return removed + } + + /** How many eval chunks are currently in the vector store. */ + async count(): Promise { + return this.ragService.countChunksInCollection(KB_EVAL_COLLECTION) + } + + /** + * Wipe and rebuild the eval corpus. + * + * Always a full rebuild: a partial re-ingest would leave the vector store in + * a state no fingerprint describes, and a fingerprint that does not describe + * the store is worse than no fingerprint at all. + */ + async ingest(onProgress?: (docId: string, index: number, total: number) => void): Promise { + const corpus = await this.loadCorpus() + const removedBeforeIngest = await this.reset() + + let chunks = 0 + const failures: IngestSummary['failures'] = [] + + for (const [index, doc] of corpus.entries()) { + onProgress?.(doc.docId, index + 1, corpus.length) + try { + const result = await this.ragService.embedAndStoreText(doc.text, { + source: doc.path, + collection: KB_EVAL_COLLECTION, + }) + if (!result) { + failures.push({ docId: doc.docId, reason: 'embedAndStoreText returned null' }) + continue + } + chunks += result.chunks + } catch (error) { + failures.push({ docId: doc.docId, reason: error instanceof Error ? error.message : String(error) }) + } + } + + return { + fingerprint: await this.fingerprint(), + documents: corpus.length, + chunks, + removedBeforeIngest, + failures, + } + } +} diff --git a/admin/app/services/eval_generation_service.ts b/admin/app/services/eval_generation_service.ts new file mode 100644 index 0000000..2aba609 --- /dev/null +++ b/admin/app/services/eval_generation_service.ts @@ -0,0 +1,350 @@ +import { EvalCorpusService } from '#services/eval_corpus_service' +import { OllamaService } from '#services/ollama_service' +import { RagPipelineService } from '#services/rag_pipeline_service' +import { inject } from '@adonisjs/core' +import logger from '@adonisjs/core/services/logger' +import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' +import type { OllamaChatMessage } from '../../types/ollama.js' +import type { PipelineOptions, RetrievedChunk } from '../../types/rag.js' +import { docIdFromSource } from '../utils/eval/corpus_source.js' +import type { Golden } from '../utils/eval/golden_set.js' +import { + scoreAnswer, + summarizeNumeric, + summarizeRepeats, + type GenerationScores, + type NumericSummary, + type RepeatStats, +} from '../utils/eval/generation_metrics.js' + +/** + * The three ways to run a question, and what each one isolates. + * + * This is the part of the harness that answers "code bug or weak model?". + * Running the same question all three ways turns one ambiguous score into a + * decomposition: + * + * - `oracle` gives the model perfect context by construction. A low score here + * is the model (or the prompt) failing to use good context — retrieval is + * provably not at fault. + * - `e2e` is the real product. `oracle - e2e` is the cost of imperfect retrieval. + * - `noretrieval` is the model's parametric baseline. `e2e - noretrieval` is + * what RAG is actually buying, and on the fictional questions it should be + * nearly everything, since no model can know them. + */ +export type GenerationMode = 'oracle' | 'e2e' | 'noretrieval' + +/** + * Sentinel model name for the extractive reference run. + * + * `--model=mock` needs no Ollama at all: it answers by echoing whatever context + * the pipeline injected, and refuses when nothing was injected. That makes it + * two useful things at once — a way to exercise scoring and reporting with zero + * models installed (so the harness itself is CI-testable), and a genuine + * *ceiling* line: the best a perfectly extractive model could score given this + * retrieval. A real model below the mock line is the bottleneck; a mock line + * that is itself low means retrieval is. + */ +export const MOCK_MODEL = 'mock' + +/** Fixed sampling for eval runs. Never used by production chat. */ +export const EVAL_TEMPERATURE = 0 +export const EVAL_SEED = 42 + +export type GenerationRunOptions = { + mode: GenerationMode + model: string + repeats?: number + topK?: number + scoreThreshold?: number + /** Skip the history-aware rewrite even on multi-turn goldens. */ + skipQueryRewrite?: boolean + onProgress?: (id: string, index: number, total: number) => void +} + +export type GenerationCaseResult = { + id: string + tags: string[] + expectRefusal: boolean + /** One entry per repeat. */ + answers: string[] + scores: GenerationScores[] + correctness: RepeatStats + refusalCorrectness: RepeatStats + leakageFree: RepeatStats + groundedness: NumericSummary | null + retrievedDocIds: string[] + injectedChunks: number + /** Non-null only when something went wrong talking to the model. */ + error?: string +} + +export type GenerationAggregate = { + cases: number + /** Cases whose pass/fail flipped across repeats; excluded from gating. */ + unstable: number + errors: number + correctness: number | null + refusalCorrectness: number | null + leakageRate: number | null + thinkTagLeakRate: number | null + markdownRate: number | null + groundedness: NumericSummary | null + meanAnswerLength: number | null +} + +export type GenerationRunResult = { + params: { + mode: GenerationMode + model: string + repeats: number + temperature: number + seed: number + topK?: number + scoreThreshold?: number + } + overall: GenerationAggregate + byTag: Record + cases: GenerationCaseResult[] + elapsedMs: number +} + +@inject() +export class EvalGenerationService { + constructor( + private ollamaService: OllamaService, + private pipeline: RagPipelineService, + private corpusService: EvalCorpusService + ) {} + + async run(goldens: Golden[], options: GenerationRunOptions): Promise { + const repeats = Math.max(1, options.repeats ?? 3) + const started = Date.now() + const isMock = options.model === MOCK_MODEL + + // Oracle mode needs the corpus text on hand to synthesize perfect context. + const corpusText = options.mode === 'oracle' ? await this.loadCorpusText() : null + + if (!isMock) await this.prepareModel(options.model) + + const cases: GenerationCaseResult[] = [] + for (const [index, golden] of goldens.entries()) { + options.onProgress?.(golden.id, index + 1, goldens.length) + cases.push(await this.runCase(golden, options, repeats, corpusText, isMock)) + } + + return { + params: { + mode: options.mode, + model: options.model, + repeats, + temperature: EVAL_TEMPERATURE, + seed: EVAL_SEED, + topK: options.topK, + scoreThreshold: options.scoreThreshold, + }, + overall: aggregateGeneration(cases), + byTag: aggregateGenerationByTag(cases), + cases, + elapsedMs: Date.now() - started, + } + } + + /** + * Evict other resident models and burn one throwaway generation. + * + * Both borrowed from BenchmarkService, and for the same reasons it added + * them: a cold first run is dramatically slower and behaves differently, and + * leftover models in VRAM change how the one under test is scheduled. Quality + * runs are less timing-sensitive than throughput runs, but a first-token + * timeout or an OOM-driven CPU fallback absolutely does change the answer. + */ + private async prepareModel(model: string): Promise { + try { + await this.ollamaService.unloadAllChatModelsExcept(model) + } catch (error) { + logger.warn(`[Eval] Could not evict resident models: ${errorText(error)}`) + } + try { + await this.ollamaService.chat({ + model, + messages: [{ role: 'user', content: 'Reply with the single word: ready' }], + temperature: EVAL_TEMPERATURE, + seed: EVAL_SEED, + }) + } catch (error) { + // Best-effort: if warm-up fails the scored runs will surface the real + // problem with a better message than we could produce here. + logger.warn(`[Eval] Warm-up generation failed: ${errorText(error)}`) + } + } + + private async loadCorpusText(): Promise> { + const docs = await this.corpusService.loadCorpus() + return new Map(docs.map((d) => [d.docId, { text: d.text, path: d.path }])) + } + + private async runCase( + golden: Golden, + options: GenerationRunOptions, + repeats: number, + corpusText: Map | null, + isMock: boolean + ): Promise { + const messages: OllamaChatMessage[] = [ + ...golden.turns.map((t) => ({ role: t.role, content: t.content })), + { role: 'user' as const, content: golden.query }, + ] + + const pipelineOptions: PipelineOptions = { + collection: KB_EVAL_COLLECTION, + topK: options.topK, + scoreThreshold: options.scoreThreshold, + // The mock run must not touch Ollama at all — that is what makes it + // usable with no models installed. The rewrite is a chat-model call, so + // it is always skipped there (it would 404 and silently fall back, which + // works but quietly makes the "no model needed" claim untrue). + skipQueryRewrite: options.skipQueryRewrite || isMock, + // A developer's personal NOMAD.md would silently skew every score. + skipNomadMd: true, + } + + if (options.mode === 'oracle') { + pipelineOptions.oracleContext = golden.relevantDocIds.map((docId) => { + const doc = corpusText?.get(docId) + return { + text: doc?.text ?? '', + score: 1, + metadata: { source: doc?.path }, + } satisfies RetrievedChunk + }) + } else if (options.mode === 'noretrieval') { + // An empty oracle context short-circuits retrieval without injecting + // anything — the model answers from parametric memory alone. + pipelineOptions.oracleContext = [] + } + + const answers: string[] = [] + const scores: GenerationScores[] = [] + let retrievedDocIds: string[] = [] + let injectedChunks = 0 + let error: string | undefined + + for (let attempt = 0; attempt < repeats; attempt++) { + try { + const trace = await this.pipeline.buildPrompt(messages, options.model, pipelineOptions) + retrievedDocIds = uniqueDocIds(trace.retrieved) + injectedChunks = trace.injected.length + const context = trace.injected.map((c) => c.text).join('\n\n') + + const answer = isMock + ? mockAnswer(context) + : await this.generate(options.model, trace.messages, trace.numCtx) + + answers.push(answer) + scores.push( + scoreAnswer({ + answer, + context, + mustInclude: golden.mustInclude, + mustNotInclude: golden.mustNotInclude, + expectRefusal: golden.expectRefusal, + }) + ) + } catch (err) { + error = errorText(err) + break + } + } + + return { + id: golden.id, + tags: golden.tags, + expectRefusal: golden.expectRefusal, + answers, + scores, + correctness: summarizeRepeats(scores.map((s) => s.correct)), + refusalCorrectness: summarizeRepeats(scores.map((s) => s.refusalCorrect)), + leakageFree: summarizeRepeats(scores.map((s) => s.leakage.length === 0)), + groundedness: summarizeNumeric(scores.map((s) => s.numericGroundedness)), + retrievedDocIds, + injectedChunks, + error, + } + } + + private async generate(model: string, messages: OllamaChatMessage[], numCtx?: number): Promise { + const response = await this.ollamaService.chat({ + model, + messages, + numCtx, + temperature: EVAL_TEMPERATURE, + seed: EVAL_SEED, + }) + return response.message.content.trim() + } +} + +/** + * The extractive reference answer: echo the context, or decline when there is + * none. Deterministic, model-free, and an honest ceiling for the current + * retrieval. + */ +export function mockAnswer(context: string): string { + if (!context.trim()) return "I don't have information about that." + return context +} + +function uniqueDocIds(chunks: RetrievedChunk[]): string[] { + const ids: string[] = [] + const seen = new Set() + for (const chunk of chunks) { + const docId = docIdFromSource(chunk.metadata?.source) + if (docId && !seen.has(docId)) { + seen.add(docId) + ids.push(docId) + } + } + return ids +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +const rate = (values: boolean[]): number | null => + values.length === 0 ? null : values.filter(Boolean).length / values.length + +export function aggregateGeneration(cases: GenerationCaseResult[]): GenerationAggregate { + const scored = cases.filter((c) => c.scores.length > 0) + const allScores = scored.flatMap((c) => c.scores) + + return { + cases: cases.length, + unstable: scored.filter((c) => c.correctness.unstable).length, + errors: cases.filter((c) => c.error).length, + // Mean pass-rate rather than all-or-nothing, so a case that passes 2 of 3 + // is reported as 0.67 instead of being silently rounded either way. + correctness: scored.length === 0 ? null : mean(scored.map((c) => c.correctness.passRate)), + refusalCorrectness: + scored.length === 0 ? null : mean(scored.map((c) => c.refusalCorrectness.passRate)), + leakageRate: rate(allScores.map((s) => s.leakage.length > 0)), + thinkTagLeakRate: rate(allScores.map((s) => s.thinkTagLeak)), + markdownRate: rate(allScores.map((s) => s.markdownFormatted)), + groundedness: summarizeNumeric(allScores.map((s) => s.numericGroundedness)), + meanAnswerLength: allScores.length === 0 ? null : mean(allScores.map((s) => s.length)), + } +} + +export function aggregateGenerationByTag( + cases: GenerationCaseResult[] +): Record { + const tags = new Set(cases.flatMap((c) => c.tags)) + const out: Record = {} + for (const tag of [...tags].sort()) { + out[tag] = aggregateGeneration(cases.filter((c) => c.tags.includes(tag))) + } + return out +} + +const mean = (values: number[]) => values.reduce((a, b) => a + b, 0) / values.length diff --git a/admin/app/services/eval_report_service.ts b/admin/app/services/eval_report_service.ts new file mode 100644 index 0000000..337a961 --- /dev/null +++ b/admin/app/services/eval_report_service.ts @@ -0,0 +1,154 @@ +import { inject } from '@adonisjs/core' +import { execFile } from 'node:child_process' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { promisify } from 'node:util' +import type { GenerationRunResult } from './eval_generation_service.js' +import type { RetrievalRunResult } from './eval_retrieval_service.js' +import { flattenByK, type EvalReport, type ReportMeta } from '../utils/eval/report.js' + +const run = promisify(execFile) + +export const EVAL_REPORTS_DIR = 'tests/eval/reports' +export const EVAL_BASELINES_DIR = 'tests/eval/baselines' + +/** + * Turns run results into durable, comparable report files. + * + * Two formats, on purpose: JSON is what `eval:compare` diffs, and Markdown is + * what a human actually reads when a number moved and they need to know which + * question broke and what the model said. + */ +@inject() +export class EvalReportService { + async buildMeta( + kind: 'retrieval' | 'generation', + corpusFingerprint: string, + params: Record + ): Promise { + const [git, nomadVersion, platform] = await Promise.all([ + readGitState(), + readNomadVersion(), + readPlatform(), + ]) + return { + kind, + createdAt: new Date().toISOString(), + corpusFingerprint, + gitSha: git.sha, + gitBranch: git.branch, + gitDirty: git.dirty, + nomadVersion, + platform, + params, + } + } + + fromRetrieval(meta: ReportMeta, result: RetrievalRunResult): EvalReport { + const flatten = (agg: RetrievalRunResult['overall']) => ({ + ...flattenByK('recall', agg.recall), + ...flattenByK('hitRate', agg.hitRate), + ...flattenByK('precision', agg.precision), + ...flattenByK('ndcg', agg.ndcg), + mrr: agg.mrr, + emptyRateOnAnswerable: agg.emptyRateOnAnswerable, + nonEmptyRateOnRefusal: agg.nonEmptyRateOnRefusal, + }) + return { + meta, + metrics: flatten(result.overall), + byTag: Object.fromEntries(Object.entries(result.byTag).map(([tag, agg]) => [tag, flatten(agg)])), + cases: result.cases, + } + } + + fromGeneration(meta: ReportMeta, result: GenerationRunResult): EvalReport { + const flatten = (agg: GenerationRunResult['overall']) => ({ + correctness: agg.correctness, + refusalCorrectness: agg.refusalCorrectness, + leakageRate: agg.leakageRate, + thinkTagLeakRate: agg.thinkTagLeakRate, + markdownRate: agg.markdownRate, + groundedness: agg.groundedness?.mean ?? null, + }) + return { + meta, + metrics: flatten(result.overall), + byTag: Object.fromEntries(Object.entries(result.byTag).map(([tag, agg]) => [tag, flatten(agg)])), + // Answers are kept in full. When a score moves, the only question anyone + // actually asks is "what did it say?", and a truncated answer cannot + // answer it. + cases: result.cases, + } + } + + /** Write `.json` and `.md`; returns the JSON path. */ + async write(report: EvalReport, slug: string, markdown: string): Promise { + const dir = resolve(join(process.cwd(), EVAL_REPORTS_DIR)) + await mkdir(dir, { recursive: true }) + const jsonPath = join(dir, `${slug}.json`) + await writeFile(jsonPath, JSON.stringify(report, null, 2)) + await writeFile(join(dir, `${slug}.md`), markdown) + return jsonPath + } + + async read(path: string): Promise { + const parsed = JSON.parse(await readFile(resolve(path), 'utf8')) + if (!parsed?.meta?.kind || !parsed?.metrics) { + throw new Error(`${path} is not an eval report`) + } + return parsed as EvalReport + } + + /** + * Baselines are filed under their corpus fingerprint. That is not cosmetic: + * it makes it structurally impossible to overwrite the baseline for one + * corpus with a run against another. + */ + baselinePath(report: EvalReport, name: string): string { + return resolve( + join(process.cwd(), EVAL_BASELINES_DIR, report.meta.corpusFingerprint, `${name}.json`) + ) + } + + async promoteToBaseline(report: EvalReport, name: string): Promise { + const path = this.baselinePath(report, name) + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, JSON.stringify(report, null, 2)) + return path + } +} + +async function readGitState() { + const git = async (args: string[]) => (await run('git', args)).stdout.trim() + try { + const [sha, branch, status] = await Promise.all([ + git(['rev-parse', 'HEAD']), + git(['rev-parse', '--abbrev-ref', 'HEAD']), + git(['status', '--porcelain']), + ]) + // A dirty tree matters: it means the report does not correspond to any + // commit, so a later "this regressed at sha X" claim would be unfounded. + return { sha, branch, dirty: status.length > 0 } + } catch { + return { sha: null, branch: null, dirty: false } + } +} + +async function readNomadVersion(): Promise { + try { + const pkg = JSON.parse(await readFile(resolve(join(process.cwd(), '..', 'package.json')), 'utf8')) + return pkg.version ?? null + } catch { + return null + } +} + +async function readPlatform() { + const os = await import('node:os') + return { + cpuArchitecture: os.arch(), + osName: `${os.type()} ${os.release()}`, + nodeVersion: process.version, + } +} diff --git a/admin/app/services/eval_retrieval_service.ts b/admin/app/services/eval_retrieval_service.ts new file mode 100644 index 0000000..e3a8685 --- /dev/null +++ b/admin/app/services/eval_retrieval_service.ts @@ -0,0 +1,153 @@ +import { EvalCorpusService } from '#services/eval_corpus_service' +import { RagService } from '#services/rag_service' +import { inject } from '@adonisjs/core' +import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' +import { RAG_DEFAULT_SCORE_THRESHOLD, RAG_DEFAULT_TOP_K } from '../../constants/ollama.js' +import type { RetrievalStages } from '../../types/rag.js' +import { docIdFromSource } from '../utils/eval/corpus_source.js' +import type { Golden } from '../utils/eval/golden_set.js' +import { + aggregate, + aggregateByTag, + DEFAULT_K_VALUES, + scoreCase, + type RetrievalAggregate, + type RetrievalCase, + type RetrievalCaseResult, + type ScoredChunk, +} from '../utils/eval/retrieval_metrics.js' + +export type RetrievalRunOptions = { + topK?: number + scoreThreshold?: number + kValues?: number[] + /** Score the raw dense / reranked / diversified orderings separately. */ + ablate?: boolean +} + +export type StageAblation = { + dense: RetrievalAggregate + reranked: RetrievalAggregate + diversified: RetrievalAggregate +} + +export type RetrievalRunResult = { + params: { topK: number; scoreThreshold: number; kValues: number[] } + overall: RetrievalAggregate + byTag: Record + cases: RetrievalCaseResult[] + ablation: StageAblation | null + /** + * Chunks that came back without a resolvable eval doc id. Non-zero means the + * collection filter leaked and the run is measuring the wrong corpus. + */ + unresolvedChunks: number +} + +/** + * Runs the golden question set through NOMAD's real retrieval path and scores + * the result. + * + * Deliberately does not touch the chat model. The only model call is the + * embedding of each query, whose output is stable, so two runs over the same + * corpus produce identical numbers on any hardware. That is what makes this the + * fast inner loop and the only tier worth gating CI on: a movement here is a + * code change, full stop. + * + * Multi-turn goldens are scored on their raw final message. Resolving the + * coreference would need the chat model, which would make this tier + * non-deterministic — so the multi-turn bucket here reports the honest floor, + * and the rewrite's contribution is measured in the generation tier instead. + */ +@inject() +export class EvalRetrievalService { + constructor( + private ragService: RagService, + private corpusService: EvalCorpusService + ) {} + + async run(goldens: Golden[], options: RetrievalRunOptions = {}): Promise { + const topK = options.topK ?? RAG_DEFAULT_TOP_K + const scoreThreshold = options.scoreThreshold ?? RAG_DEFAULT_SCORE_THRESHOLD + const kValues = options.kValues ?? DEFAULT_K_VALUES + + const cases: RetrievalCase[] = [] + const denseCases: RetrievalCase[] = [] + const rerankedCases: RetrievalCase[] = [] + const diversifiedCases: RetrievalCase[] = [] + let unresolvedChunks = 0 + + for (const golden of goldens) { + const stages: RetrievalStages = {} + const docs = await this.ragService.searchSimilarDocuments( + golden.query, + topK, + scoreThreshold, + KB_EVAL_COLLECTION, + options.ablate ? stages : undefined + ) + + const retrieved: ScoredChunk[] = docs.map((d) => { + const docId = docIdFromSource(d.metadata?.source) + if (!docId) unresolvedChunks++ + return { docId, score: d.score, semanticScore: d.metadata?.semantic_score } + }) + + cases.push(toCase(golden, retrieved)) + + if (options.ablate) { + denseCases.push(toCase(golden, stageToChunks(stages.dense))) + rerankedCases.push(toCase(golden, stageToChunks(stages.reranked))) + diversifiedCases.push(toCase(golden, stageToChunks(stages.diversified))) + } + } + + const results = cases.map((c) => scoreCase(c, kValues)) + + const aggregateStage = (stageCases: RetrievalCase[]) => + aggregate(stageCases, stageCases.map((c) => scoreCase(c, kValues)), kValues) + + return { + params: { topK, scoreThreshold, kValues }, + overall: aggregate(cases, results, kValues), + byTag: aggregateByTag(cases, results, kValues), + cases: results, + ablation: options.ablate + ? { + dense: aggregateStage(denseCases), + reranked: aggregateStage(rerankedCases), + diversified: aggregateStage(diversifiedCases), + } + : null, + unresolvedChunks, + } + } + + /** Confirm the corpus is actually ingested before reporting a score of zero. */ + async assertCorpusReady(): Promise { + const chunks = await this.corpusService.count() + if (chunks === 0) { + throw new Error( + 'The eval corpus is not ingested — every score would be zero. Run: node ace eval:corpus --ingest' + ) + } + return chunks + } +} + +function toCase(golden: Golden, retrieved: ScoredChunk[]): RetrievalCase { + return { + id: golden.id, + tags: golden.tags, + retrieved, + relevantDocIds: golden.relevantDocIds, + expectRefusal: golden.expectRefusal, + } +} + +function stageToChunks(stage: Array<{ source?: string; score: number }> | undefined): ScoredChunk[] { + return (stage ?? []).map((entry) => ({ + docId: docIdFromSource(entry.source), + score: entry.score, + })) +} diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index d030bd7..27841a3 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -48,6 +48,12 @@ type ChatInput = { thinkingCapable?: boolean stream?: boolean numCtx?: number + // Sampling controls. Left unset for normal chat, which inherits the backend's + // defaults — the historical behaviour. The eval harness sets temperature 0 and + // a fixed seed so repeated runs are as close to comparable as llama.cpp allows + // (batching and GPU non-determinism still move outputs, hence --repeats). + temperature?: number + seed?: number // Aborts the upstream request when the client disconnects, so an abandoned generation // doesn't keep decoding server-side and block Ollama's single parallel slot (#1065). signal?: AbortSignal @@ -350,6 +356,12 @@ export class OllamaService { if (chatRequest.numCtx) { params.num_ctx = chatRequest.numCtx } + if (chatRequest.temperature !== undefined) { + params.temperature = chatRequest.temperature + } + if (chatRequest.seed !== undefined) { + params.seed = chatRequest.seed + } const response = await this.openai.chat.completions.create(params, { signal: chatRequest.signal }) const choice = response.choices[0] @@ -392,6 +404,12 @@ export class OllamaService { if (chatRequest.numCtx) { params.num_ctx = chatRequest.numCtx } + if (chatRequest.temperature !== undefined) { + params.temperature = chatRequest.temperature + } + if (chatRequest.seed !== undefined) { + params.seed = chatRequest.seed + } const stream = (await this.openai.chat.completions.create(params, { signal: chatRequest.signal, diff --git a/admin/app/services/rag_pipeline_service.ts b/admin/app/services/rag_pipeline_service.ts new file mode 100644 index 0000000..464794b --- /dev/null +++ b/admin/app/services/rag_pipeline_service.ts @@ -0,0 +1,222 @@ +import { NomadMdService } from '#services/nomad_md_service' +import { OllamaService } from '#services/ollama_service' +import { RagService } from '#services/rag_service' +import { inject } from '@adonisjs/core' +import logger from '@adonisjs/core/services/logger' +import { + RAG_CONTEXT_LIMITS, + RAG_DEFAULT_SCORE_THRESHOLD, + RAG_DEFAULT_TOP_K, + SYSTEM_PROMPTS, +} from '../../constants/ollama.js' +import type { OllamaChatMessage } from '../../types/ollama.js' +import type { PipelineOptions, PipelineTrace, RetrievedChunk } from '../../types/rag.js' +import { + buildContextBlock, + deriveNumCtx, + getContextLimitsForModel, + trimToContextBudget, +} from '../utils/rag_prompt.js' + +/** + * Everything that happens between "a user sent a message" and "a payload goes + * to Ollama": system-prompt assembly, history-aware query rewriting, retrieval, + * model-size-aware context trimming, and the num_ctx decision. + * + * This used to live inline in OllamaController.chat. It was moved here so there + * is exactly one implementation of the prompt-building pipeline — the chat + * endpoint and the eval harness both call `buildPrompt`, so a measurement of + * the harness is a measurement of production, not of a copy that drifts. + * + * The behaviour is a verbatim port. Every quirk preserved below is marked; the + * quirks are worth fixing but each one changes output, and the point of the + * harness is to stop changing output without measuring it. + */ +@inject() +export class RagPipelineService { + constructor( + private ollamaService: OllamaService, + private ragService: RagService, + private nomadMdService: NomadMdService + ) {} + + /** + * Build the exact message array to send to Ollama, plus a trace of every + * decision made along the way. + * + * The caller passes the conversation as received; this never mutates it. + */ + async buildPrompt( + messages: OllamaChatMessage[], + model: string, + opts: PipelineOptions = {} + ): Promise { + const working: OllamaChatMessage[] = [...messages] + + // Default formatting prompt, only when the caller supplied no system message. + const hasSystemMessage = working.some((msg) => msg.role === 'system') + if (!hasSystemMessage) { + logger.debug('[RagPipeline] Injecting system prompt') + working.unshift({ role: 'system', content: SYSTEM_PROMPTS.default }) + } + + // The user-managed NOMAD.md goes in front of the formatting prompt so the + // user's persistent instructions take precedence. Skipped in evals, where a + // developer's personal NOMAD.md would silently skew every score. + if (!opts.skipNomadMd) { + const nomadPrompt = await this.nomadMdService.getSystemPrompt() + if (nomadPrompt) { + logger.debug('[RagPipeline] Injecting NOMAD.md system prompt') + working.unshift({ role: 'system', content: nomadPrompt }) + } + } + + const trace: PipelineTrace = { + rewrittenQuery: null, + didRewrite: false, + retrieved: [], + injected: [], + messages: working, + numCtx: undefined, + contextLimits: { maxResults: RAG_DEFAULT_TOP_K, maxTokens: 0 }, + timings: { rewriteMs: 0, retrievalMs: 0 }, + } + + // --- Retrieval ------------------------------------------------------- + // oracleContext bypasses retrieval entirely (eval `oracle` mode). + let relevantDocs: RetrievedChunk[] = [] + if (opts.oracleContext) { + relevantDocs = opts.oracleContext + trace.retrieved = relevantDocs + } else { + const rewriteStart = Date.now() + const { query, didRewrite } = await this.resolveRetrievalQuery(working, model, opts) + trace.timings.rewriteMs = Date.now() - rewriteStart + trace.rewrittenQuery = query + trace.didRewrite = didRewrite + + if (query) { + const retrievalStart = Date.now() + relevantDocs = await this.ragService.searchSimilarDocuments( + query, + opts.topK ?? RAG_DEFAULT_TOP_K, + opts.scoreThreshold ?? RAG_DEFAULT_SCORE_THRESHOLD, + opts.collection + ) + trace.timings.retrievalMs = Date.now() - retrievalStart + trace.retrieved = relevantDocs + logger.debug( + `[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${query}"` + ) + } + } + + // --- Context trimming + injection ------------------------------------- + if (relevantDocs.length > 0) { + const limits = getContextLimitsForModel(model, RAG_CONTEXT_LIMITS) + trace.contextLimits = limits + const trimmedDocs = trimToContextBudget(relevantDocs, limits) + trace.injected = trimmedDocs + + logger.debug( + `[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${model}, maxResults: ${limits.maxResults}, maxTokens: ${limits.maxTokens || 'unlimited'})` + ) + + const systemMessage: OllamaChatMessage = { + role: 'system', + content: SYSTEM_PROMPTS.rag_context(buildContextBlock(trimmedDocs)), + } + + // After any existing system messages, before the first non-system message. + const firstNonSystemIndex = working.findIndex((msg) => msg.role !== 'system') + const insertIndex = firstNonSystemIndex === -1 ? 0 : firstNonSystemIndex + working.splice(insertIndex, 0, systemMessage) + } + + trace.numCtx = deriveNumCtx(working) + if (trace.numCtx) { + logger.debug(`[RagPipeline] Large system prompt, requesting num_ctx: ${trace.numCtx}`) + } + + return trace + } + + /** + * Decide what string to hand to retrieval. + * + * Returns null when the RAG pipeline should be skipped entirely — an empty + * knowledge base, or a conversation with no user message at all. + */ + private async resolveRetrievalQuery( + messages: OllamaChatMessage[], + model: string, + opts: PipelineOptions + ): Promise<{ query: string | null; didRewrite: boolean }> { + const lastUserMessage = [...messages].reverse().find((msg) => msg.role === 'user') + + try { + // Skip the entire RAG pipeline if there are no documents to search. + const hasDocuments = await this.ragService.hasDocuments() + if (!hasDocuments) { + return { query: null, didRewrite: false } + } + + if (opts.skipQueryRewrite) { + return { query: lastUserMessage?.content ?? null, didRewrite: false } + } + + // Last 6 messages ≈ 3 turns. + // + // PRESERVED QUIRK: this slice is taken *after* system messages have been + // unshifted, so on short conversations the system prompts land inside the + // window and get labelled "Assistant" in the transcript below. Faithful + // to the original; a candidate fix once the harness can measure it. + const recentMessages = messages.slice(-6) + + // Skip rewriting on the very first turn — with only one user message there + // is no prior context to fold in, so the rewrite would just echo the + // message back at the cost of an extra LLM round-trip. From the first + // follow-up onward the rewrite carries entities from earlier turns + // ("the bars" -> "Hershey's bars chocolate poisoning dog"); without it, + // embeddings match nothing and the assistant loses the thread. + const userMessages = recentMessages.filter((msg) => msg.role === 'user') + if (userMessages.length < 2) { + return { query: lastUserMessage?.content ?? null, didRewrite: false } + } + + const conversationContext = recentMessages + .map((msg) => { + const role = msg.role === 'user' ? 'User' : 'Assistant' + // Truncate assistant messages to keep the rewrite prompt manageable. + const content = + msg.role === 'assistant' + ? msg.content.slice(0, 200) + (msg.content.length > 200 ? '...' : '') + : msg.content + return `${role}: "${content}"` + }) + .join('\n') + + const response = await this.ollamaService.chat({ + model, + messages: [ + { role: 'system', content: SYSTEM_PROMPTS.query_rewrite }, + { + role: 'user', + content: `Conversation:\n${conversationContext}\n\nRewritten Query:`, + }, + ], + }) + + const rewrittenQuery = response.message.content.trim() + logger.info(`[RAG] Query rewritten: "${rewrittenQuery}"`) + return { query: rewrittenQuery, didRewrite: true } + } catch (error) { + logger.error( + `[RAG] Query rewriting failed: ${error instanceof Error ? error.message : error}` + ) + // Fall back to the last user message rather than losing retrieval entirely. + return { query: lastUserMessage?.content ?? null, didRewrite: false } + } + } + +} diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 13f0ec7..858b58a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -22,7 +22,18 @@ import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision import { decideContentReindex, type ReindexOutcome } from '../utils/content_reindex_decision.js' import KbRatioRegistry from '#models/kb_ratio_registry' import { decideWarnings } from '../utils/kb_warning_decision.js' -import type { FileWarning, FileWarningsResult, StoredFileInfo } from '../../types/rag.js' +import type { FileWarning, FileWarningsResult, RetrievalStages, StoredFileInfo } from '../../types/rag.js' +import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js' + +/** + * Qdrant filter that hides the developer eval corpus from every read path that + * enumerates the *user's* knowledge base. The fixtures share this collection + * (NOMAD collections are a payload tag), so without this they would show up as + * broken files in the KB UI and skew the ingest-health warnings. + */ +const EXCLUDE_EVAL_FILTER = { + must_not: [{ key: 'collection', match: { value: KB_EVAL_COLLECTION } }], +} import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' @@ -59,6 +70,10 @@ export class RagService { public static MODEL_CONTEXT_LENGTH = 2048 // nomic-embed-text has 2K token context public static MAX_SAFE_TOKENS = 1600 // Leave buffer for prefix and tokenization variance public static TARGET_TOKENS_PER_CHUNK = 1500 // Target 1500 tokens per chunk for embedding + // Overlap between adjacent chunks. Was an inline literal at the chunker call + // site; named here because it shapes what lands in the vector store and so + // must be part of the eval corpus fingerprint. + public static CHUNK_OVERLAP_TOKENS = 150 public static PREFIX_TOKEN_BUDGET = 10 // Reserve ~10 tokens for prefixes public static CHAR_TO_TOKEN_RATIO = 2 // Conservative chars-per-token estimate; technical docs // (numbers, symbols, abbreviations) tokenize denser @@ -336,7 +351,7 @@ export class RagService { // We need to convert our embedding model's token counts to character counts // since nomic-embed-text tokenizer uses ~3 chars per token const targetCharsPerChunk = Math.floor(RagService.TARGET_TOKENS_PER_CHUNK * RagService.CHAR_TO_TOKEN_RATIO) - const overlapChars = Math.floor(150 * RagService.CHAR_TO_TOKEN_RATIO) + const overlapChars = Math.floor(RagService.CHUNK_OVERLAP_TOKENS * RagService.CHAR_TO_TOKEN_RATIO) const chunker = await TokenChunker.create({ chunkSize: targetCharsPerChunk, @@ -840,7 +855,15 @@ export class RagService { query: string, limit: number = 5, scoreThreshold: number = 0.3, // Lower default threshold - was 0.7, now 0.3 - collection?: string + collection?: string, + /** + * Optional sink for the intermediate ranked lists. When supplied, the raw + * dense order, the post-rerank order, and the post-diversity order are all + * written here. Purely observational — nothing about the returned result + * changes — and it is what lets the eval harness answer "is the reranking + * heuristic actually helping?" without a second implementation of search. + */ + stagesOut?: RetrievalStages ): Promise }>> { try { logger.debug(`[RAG] Starting similarity search for query: "${query}"`) @@ -947,6 +970,21 @@ export class RagService { // Apply source diversity penalty to avoid all results from the same document const diverseResults = this.applySourceDiversity(rerankedResults) + // Record the three ranked lists for the eval harness's stage ablation. + // Sliced to `limit` so each stage is compared on the window that would + // actually have been injected, not on the wider rerank candidate pool. + if (stagesOut) { + const summarize = (rows: Array<{ text: string; score: number; source?: string }>) => + rows.slice(0, limit).map((r) => ({ source: r.source, score: r.score })) + stagesOut.dense = summarize(resultsWithMetadata) + stagesOut.reranked = rerankedResults + .slice(0, limit) + .map((r) => ({ source: r.source, score: r.finalScore })) + stagesOut.diversified = diverseResults + .slice(0, limit) + .map((r) => ({ source: r.source, score: r.finalScore })) + } + // Return top N results with enhanced metadata return diverseResults.slice(0, limit).map((result) => ({ text: result.text, @@ -955,6 +993,10 @@ export class RagService { chunk_index: result.chunk_index, created_at: result.created_at, semantic_score: result.score, + // The originating file/ZIM path. Stored on every point but previously + // dropped here, which made it impossible to map a retrieved chunk back + // to its document — needed for citations and for recall@k scoring. + source: result.source, // Enhanced ZIM metadata (likely be undefined for non-ZIM content) article_title: result.article_title, section_title: result.section_title, @@ -1149,6 +1191,10 @@ export class RagService { key: 'source', limit: RagService.FACET_SOURCE_LIMIT, exact: true, + // Keep the developer eval corpus out of the user's Stored Files list. + // It shares this Qdrant collection but is not the user's content, and + // its fixture paths would render as broken/missing files in the KB UI. + filter: EXCLUDE_EVAL_FILTER, }) for (const hit of facetResult.hits) { if (typeof hit.value === 'string') sources.add(hit.value) @@ -1221,7 +1267,10 @@ export class RagService { }) const collections = new Set() for (const hit of facetResult.hits) { - if (typeof hit.value === 'string') collections.add(hit.value) + // The reserved eval tag is internal; never offer it in a subject picker. + if (typeof hit.value === 'string' && hit.value !== KB_EVAL_COLLECTION) { + collections.add(hit.value) + } } return Array.from(collections).sort() } @@ -1317,6 +1366,43 @@ export class RagService { } } + /** + * Count the points carrying a given `collection` tag. + * + * Distinct from the facet-based counts above, which deliberately exclude + * internal collections. This answers "how many chunks are in exactly this + * collection", which is what the eval harness needs to confirm an ingest + * landed and what a future per-collection UI would want. + */ + public async countChunksInCollection(collection: string): Promise { + await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION) + const result = await this.qdrant!.count(RagService.CONTENT_COLLECTION_NAME, { + filter: { must: [{ key: 'collection', match: { value: collection } }] }, + exact: true, + }) + return result.count + } + + /** + * Hard-delete every point carrying a given `collection` tag. + * + * Note this is NOT what `deleteKnowledgeCollection` does — that one clears the + * tag and leaves the user's documents in place, because deleting a user's + * content because they renamed a folder would be indefensible. This one + * genuinely removes the points, and exists for the eval corpus, which is + * disposable by construction. Returns the number of points removed. + */ + public async deleteCollectionPoints(collection: string): Promise { + await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION) + const before = await this.countChunksInCollection(collection) + if (before === 0) return 0 + await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, { + wait: true, + filter: { must: [{ key: 'collection', match: { value: collection } }] }, + }) + return before + } + /** * Resolve a stored-file `source` to an absolute disk path, but only if the * path lives under the uploads directory. Mirrors the docs_service traversal @@ -1434,6 +1520,9 @@ export class RagService { key: 'source', limit: RagService.FACET_SOURCE_LIMIT, exact: true, + // Same exclusion as getStoredFiles: eval fixtures are not user files, + // and counting them here would raise bogus zero_chunks warnings. + filter: EXCLUDE_EVAL_FILTER, }) for (const hit of facetResult.hits) { if (typeof hit.value === 'string') chunksBySource.set(hit.value, hit.count) diff --git a/admin/app/utils/eval/corpus_source.ts b/admin/app/utils/eval/corpus_source.ts new file mode 100644 index 0000000..cc794d6 --- /dev/null +++ b/admin/app/utils/eval/corpus_source.ts @@ -0,0 +1,38 @@ +import { basename, join, resolve, sep } from 'node:path' + +/** Where the frozen corpus lives, relative to the app root. */ +export const EVAL_CORPUS_DIR = 'tests/eval/corpus' + +/** + * Map a retrieved chunk's `source` payload back to its corpus document id, or + * null if the chunk did not come from the eval corpus. + * + * The eval corpus shares the `nomad_knowledge_base` Qdrant collection with the + * developer's real documents — NOMAD "collections" are a payload tag, not + * separate Qdrant collections — and is isolated by a + * `collection: __nomad_eval__` filter that Qdrant applies during search. That + * filter does hold, but a harness whose every number depends on a filter should + * be able to *prove* the filter held rather than assume it. This function is + * that proof: the caller counts every unresolved chunk and fails the run if the + * count is non-zero. + * + * The check is on the resolved **path**, not the file extension. An earlier + * version accepted any `*.md`, which was worse than useless here: NOMAD embeds + * its own `admin/docs/*.md` into the knowledge base on first run, so a leaked + * `faq.md` would have resolved to the plausible-looking document id "faq", been + * counted as a merely-irrelevant chunk, and quietly depressed precision with no + * indication that anything had gone wrong. + * + * Mirrors the resolve-then-prefix-check guard in `RagService.resolveUploadPath`. + * `corpusDir` is injectable so this stays testable without a real corpus on disk. + */ +export function docIdFromSource(source: unknown, corpusDir?: string): string | null { + if (typeof source !== 'string') return null + const dir = resolve(corpusDir ?? join(process.cwd(), EVAL_CORPUS_DIR)) + const abs = resolve(source) + // The trailing separator matters: without it, a sibling directory such as + // "…/corpus-backup" shares a string prefix with "…/corpus" and would pass. + if (!abs.startsWith(dir + sep)) return null + if (!abs.endsWith('.md')) return null + return basename(abs, '.md') +} diff --git a/admin/app/utils/eval/generation_metrics.ts b/admin/app/utils/eval/generation_metrics.ts new file mode 100644 index 0000000..67500b4 --- /dev/null +++ b/admin/app/utils/eval/generation_metrics.ts @@ -0,0 +1,254 @@ +/** + * Deterministic scorers for generated answers. + * + * Every metric here is computable offline with no judge model, which is what + * makes the generation tier gateable. An LLM judge can be layered on later for + * the things regex genuinely cannot see (nuanced faithfulness, completeness), + * but nothing in this file needs one — and a number you can compute without a + * model is a number that cannot drift because the judge changed. + * + * These are intentionally *conservative*. Each one is a proxy, and each proxy's + * blind spot is documented where it lives. A proxy you understand the limits of + * beats a score you cannot explain. + */ + +/** Answer-level scoring of one generated response against one golden. */ +export type GenerationScores = { + /** Every mustInclude pattern matched and no mustNotInclude pattern did. */ + correct: boolean + matchedRequired: string[] + missedRequired: string[] + /** Forbidden patterns that appeared. Non-empty means a specific wrong claim. */ + hitForbidden: string[] + /** The model declined to answer. */ + refused: boolean + /** + * Refusal behaviour was right: declined an out-of-corpus question, or + * answered an answerable one. + */ + refusalCorrect: boolean + /** The answer narrated its own retrieval ("according to Context 1"). */ + leakage: string[] + /** Reasoning tags survived into the user-visible answer. */ + thinkTagLeak: boolean + /** + * Fraction of the answer's numeric claims that also appear in the injected + * context. null when the answer makes no numeric claims. + */ + numericGroundedness: number | null + /** Numbers asserted by the answer that the context does not support. */ + ungroundedNumbers: string[] + /** The answer used at least some markdown structure. */ + markdownFormatted: boolean + /** Characters. Useful for spotting a model that answers by writing an essay. */ + length: number +} + +/** + * Phrases that mean "I am not answering this". + * + * Tuned against NOMAD's actual rag_context prompt, which instructs the model to + * answer from general knowledge rather than hedge — so a hedge here is either a + * correct refusal on an out-of-corpus question or a prompt regression on an + * answerable one. Which of those it is, is exactly what `refusalCorrect` says. + */ +const REFUSAL_PATTERNS: RegExp[] = [ + /\bi (?:don'?t|do not) (?:know|have)\b/i, + /\bi(?:'m| am) (?:not able|unable) to\b/i, + /\bi (?:couldn'?t|could not|can'?t|cannot) find\b/i, + /\b(?:no|not enough) (?:information|details|data)\b/i, + /\b(?:does|do) not (?:contain|include|mention|specify|provide|cover)\b/i, + /\b(?:isn'?t|is not|aren'?t|are not) (?:mentioned|specified|covered|available|provided)\b/i, + /\bnot (?:mentioned|specified|covered|stated|documented)\b/i, + /\bunable to (?:answer|determine|find)\b/i, + /\bthere (?:is|'s) no (?:information|mention|record)\b/i, + // "I'd" has no space before the contraction, so this cannot reuse the + // `i (?:would|'d)` shape used above. + /\bi(?:'d| would) need more\b/i, +] + +/** + * Phrases that narrate the retrieval machinery. + * + * The rag_context prompt explicitly forbids these ("Never narrate your + * retrieval or reasoning process"), so any hit is a measurable prompt + * regression. This is the cheapest, sharpest signal in the whole harness: pure + * regex, zero ambiguity, and it catches a bad prompt edit on the first run. + */ +const LEAKAGE_PATTERNS: RegExp[] = [ + /\b(?:according to|based on|per|from) the (?:provided |retrieved |given |supplied )?context\b/i, + /\bcontext \d+\b/i, + /\bthe knowledge ?base\b/i, + /\bthe (?:provided|retrieved|supplied|given) (?:documents?|passages?|excerpts?|text)\b/i, + /\bthe context (?:does not|doesn'?t|is|was|seems|appears)\b/i, + /\b(?:in|from) the (?:documents?|passages?) (?:provided|above|below)\b/i, + // "I wasn't able to find specific context regarding X, but here's a general + // answer" is the verbatim symptom that started NOMAD's RAG work. It is both a + // refusal and a leak, and it is the single most important string this + // detector has to catch — a prompt change that brings it back must fail + // loudly on the very next run. + /\b(?:no|any|specific|relevant) context\b/i, + /\bsearch results?\b/i, +] + +/** Reasoning-model tags that must never reach the user. */ +const THINK_TAG = /<\/?(?:think|thought|thinking|reasoning)\b[^>]*>/i + +export function detectRefusal(answer: string): boolean { + return REFUSAL_PATTERNS.some((re) => re.test(answer)) +} + +export function detectLeakage(answer: string): string[] { + return LEAKAGE_PATTERNS.filter((re) => re.test(answer)).map((re) => re.source) +} + +export function hasThinkTagLeak(answer: string): boolean { + return THINK_TAG.test(answer) +} + +/** + * Loose markdown check: a header, a list, emphasis, a table, or a code fence. + * + * Deliberately loose. SYSTEM_PROMPTS.default asks for markdown "for + * readability", not for a specific structure, so requiring headers would fail + * perfectly good one-sentence answers. This only catches a model that has + * stopped formatting entirely. + */ +export function isMarkdownFormatted(answer: string): boolean { + return /(^|\n)\s{0,3}#{1,6}\s|(^|\n)\s*[-*+]\s|(^|\n)\s*\d+\.\s|\*\*[^*]+\*\*|`[^`]+`|(^|\n)\s*\|/.test( + answer + ) +} + +/** + * Numeric tokens the answer asserts, normalized for comparison. + * + * Numbers are the highest-value fabrication signal in this domain: a wrong + * boiling time, bleach dose, or canner pressure is a wrong answer with real + * consequences, and it is exactly the kind of specific the rag_context prompt + * forbids inventing. + * + * Thousands separators are stripped so "2,000" and "2000" compare equal. + */ +export function extractNumbers(text: string): string[] { + const matches = text.match(/\d[\d,]*(?:\.\d+)?/g) ?? [] + const normalized = matches.map((m) => m.replace(/,/g, '').replace(/\.0+$/, '')) + return [...new Set(normalized)] +} + +/** + * Fraction of the answer's numeric claims that the injected context supports. + * + * Returns null when the answer contains no numbers — a qualitative answer is + * not ungrounded, it is just not measurable this way, and folding it in as 1.0 + * would quietly inflate the score. + * + * **Known limitation, stated plainly:** this only sees numbers. An answer that + * fabricates a procedure or a proper noun scores a perfect 1.0 here. It is a + * fabrication *detector*, not a faithfulness *guarantee* — the LLM-judge tier + * exists for the rest. Small integers (0-10) are excluded because they appear + * incidentally in almost any prose ("3 layers", "step 2") and would swamp the + * signal with false grounding. + */ +export function numericGroundedness( + answer: string, + context: string +): { score: number | null; ungrounded: string[] } { + const contextNumbers = new Set(extractNumbers(context)) + const claimed = extractNumbers(answer).filter((n) => { + const value = Number.parseFloat(n) + return !Number.isNaN(value) && value > 10 + }) + if (claimed.length === 0) return { score: null, ungrounded: [] } + const ungrounded = claimed.filter((n) => !contextNumbers.has(n)) + return { score: (claimed.length - ungrounded.length) / claimed.length, ungrounded } +} + +/** Which of a golden's patterns matched, using case-insensitive regex semantics. */ +export function matchPatterns(answer: string, patterns: string[]): { matched: string[]; missed: string[] } { + const matched: string[] = [] + const missed: string[] = [] + for (const pattern of patterns) { + if (new RegExp(pattern, 'i').test(answer)) matched.push(pattern) + else missed.push(pattern) + } + return { matched, missed } +} + +export type ScoreAnswerInput = { + answer: string + /** The context text actually injected into the prompt; '' when none was. */ + context: string + mustInclude: string[] + mustNotInclude: string[] + expectRefusal: boolean +} + +export function scoreAnswer(input: ScoreAnswerInput): GenerationScores { + const { answer, context, mustInclude, mustNotInclude, expectRefusal } = input + + const required = matchPatterns(answer, mustInclude) + const forbidden = matchPatterns(answer, mustNotInclude) + const refused = detectRefusal(answer) + const grounding = numericGroundedness(answer, context) + + return { + // An out-of-corpus question has no mustInclude patterns, so `correct` there + // is carried entirely by refusalCorrect below rather than by assertions. + correct: required.missed.length === 0 && forbidden.matched.length === 0, + matchedRequired: required.matched, + missedRequired: required.missed, + hitForbidden: forbidden.matched, + refused, + refusalCorrect: expectRefusal ? refused : !refused, + leakage: detectLeakage(answer), + thinkTagLeak: hasThinkTagLeak(answer), + numericGroundedness: grounding.score, + ungroundedNumbers: grounding.ungrounded, + markdownFormatted: isMarkdownFormatted(answer), + length: answer.length, + } +} + +// --- aggregation over repeats ------------------------------------------------- + +export type RepeatStats = { + /** How many of N repeats passed. */ + passes: number + repeats: number + passRate: number + /** + * True when the outcome was neither always-pass nor always-fail. Unstable + * cases are excluded from gating — treating a coin flip as a regression is + * how a harness loses the team's trust in one afternoon. + */ + unstable: boolean +} + +export function summarizeRepeats(outcomes: boolean[]): RepeatStats { + const repeats = outcomes.length + const passes = outcomes.filter(Boolean).length + return { + passes, + repeats, + passRate: repeats === 0 ? 0 : passes / repeats, + unstable: repeats > 1 && passes > 0 && passes < repeats, + } +} + +export type NumericSummary = { mean: number; stddev: number; n: number } + +/** + * Mean and population standard deviation, ignoring nulls. + * + * The stddev is not decoration: with temperature 0 it should be near zero, and + * a non-trivial value is the harness telling you the run is noisier than the + * difference you are about to interpret. + */ +export function summarizeNumeric(values: Array): NumericSummary | null { + const defined = values.filter((v): v is number => v !== null && Number.isFinite(v)) + if (defined.length === 0) return null + const mean = defined.reduce((a, b) => a + b, 0) / defined.length + const variance = defined.reduce((acc, v) => acc + (v - mean) ** 2, 0) / defined.length + return { mean, stddev: Math.sqrt(variance), n: defined.length } +} diff --git a/admin/app/utils/eval/golden_set.ts b/admin/app/utils/eval/golden_set.ts new file mode 100644 index 0000000..8b6b9ba --- /dev/null +++ b/admin/app/utils/eval/golden_set.ts @@ -0,0 +1,190 @@ +/** + * Loading and validating the golden question set, plus the corpus fingerprint. + * + * Pure and dependency-free so it runs under bare node in unit tests. Callers + * supply file contents; nothing here touches the filesystem except through the + * explicit `readFile`-shaped arguments the service passes in. + */ + +/** A prior conversation turn, for multi-turn / coreference cases. */ +export type GoldenTurn = { role: 'user' | 'assistant'; content: string } + +export type Golden = { + /** Stable identifier. Used as the join key across reports and baselines. */ + id: string + /** The user's question, as they would actually type it. */ + query: string + /** Conversation history preceding `query`. Empty for single-turn cases. */ + turns: GoldenTurn[] + /** + * Documents that genuinely answer the question, by corpus doc id (the + * markdown filename without its extension). Empty for out-of-corpus cases. + */ + relevantDocIds: string[] + /** + * Patterns the answer must contain. Each entry is a case-insensitive + * **regular expression** — plain text is a valid regex, and alternation lets + * one entry accept "3 minutes" or "three minutes" without inflating the list. + */ + mustInclude: string[] + /** Patterns the answer must NOT contain. Same regex semantics. */ + mustNotInclude: string[] + /** + * True when the corpus genuinely cannot answer the question and the correct + * behaviour is to decline rather than invent. Scored as refusal-correctness. + */ + expectRefusal: boolean + /** Free-form buckets for per-slice reporting (single-hop, acronym, ...). */ + tags: string[] +} + +export class GoldenSetError extends Error {} + +/** + * Parse a JSONL golden file, validating hard enough that a typo fails loudly at + * load rather than silently scoring zero for the rest of the project's life. + */ +export function parseGoldens(jsonl: string, sourceName = 'goldens'): Golden[] { + const goldens: Golden[] = [] + const seen = new Set() + + jsonl.split('\n').forEach((rawLine, idx) => { + const line = rawLine.trim() + if (!line || line.startsWith('//')) return + + const where = `${sourceName}:${idx + 1}` + let parsed: any + try { + parsed = JSON.parse(line) + } catch (err) { + throw new GoldenSetError(`${where}: not valid JSON — ${(err as Error).message}`) + } + + const req = (field: string) => { + if (parsed[field] === undefined) throw new GoldenSetError(`${where}: missing "${field}"`) + return parsed[field] + } + + const id = req('id') + if (typeof id !== 'string' || !id) throw new GoldenSetError(`${where}: "id" must be a non-empty string`) + if (seen.has(id)) throw new GoldenSetError(`${where}: duplicate id "${id}"`) + seen.add(id) + + const query = req('query') + if (typeof query !== 'string' || !query.trim()) { + throw new GoldenSetError(`${where}: "query" must be a non-empty string`) + } + + const strArray = (field: string): string[] => { + const v = parsed[field] ?? [] + if (!Array.isArray(v) || v.some((x) => typeof x !== 'string')) { + throw new GoldenSetError(`${where}: "${field}" must be an array of strings`) + } + return v + } + + const mustInclude = strArray('mustInclude') + const mustNotInclude = strArray('mustNotInclude') + // Compile every pattern now. A bad regex that only blows up on the one run + // where it finally matches is far worse than one that fails at load. + for (const pattern of [...mustInclude, ...mustNotInclude]) { + try { + new RegExp(pattern, 'i') + } catch (err) { + throw new GoldenSetError(`${where}: invalid regex ${JSON.stringify(pattern)} — ${(err as Error).message}`) + } + } + + const turns = (parsed.turns ?? []) as GoldenTurn[] + if (!Array.isArray(turns) || turns.some((t) => t?.role !== 'user' && t?.role !== 'assistant')) { + throw new GoldenSetError(`${where}: "turns" must be an array of {role: user|assistant, content}`) + } + + const expectRefusal = Boolean(parsed.expectRefusal) + const relevantDocIds = strArray('relevantDocIds') + if (expectRefusal && relevantDocIds.length > 0) { + throw new GoldenSetError( + `${where}: "${id}" expects a refusal but also lists relevant documents — one of those is wrong` + ) + } + if (!expectRefusal && relevantDocIds.length === 0) { + throw new GoldenSetError( + `${where}: "${id}" lists no relevant documents and does not expect a refusal — it can never be scored` + ) + } + + goldens.push({ + id, + query, + turns, + relevantDocIds, + mustInclude, + mustNotInclude, + expectRefusal, + tags: strArray('tags'), + }) + }) + + if (goldens.length === 0) throw new GoldenSetError(`${sourceName}: no goldens found`) + return goldens +} + +/** + * Every doc id a golden refers to must exist in the corpus, or recall is being + * computed against a target that can never be hit. + */ +export function assertGoldensMatchCorpus(goldens: Golden[], corpusDocIds: Iterable): void { + const corpus = new Set(corpusDocIds) + const missing = new Set() + for (const g of goldens) { + for (const docId of g.relevantDocIds) if (!corpus.has(docId)) missing.add(docId) + } + if (missing.size > 0) { + throw new GoldenSetError( + `goldens reference documents that are not in the corpus: ${[...missing].sort().join(', ')}` + ) + } +} + +/** + * Inputs to the corpus fingerprint. Anything that changes what ends up in the + * vector store belongs here — if it changes, prior reports are not comparable. + */ +export type FingerprintInputs = { + /** docId -> raw file contents. */ + documents: Map + chunkTokens: number + chunkOverlapTokens: number + charToTokenRatio: number + embeddingModel: string + embeddingDimension: number +} + +/** + * A stable hash over the corpus and the ingest parameters that shaped it. + * + * Reports carry this. Comparing two reports with different fingerprints is + * meaningless, and the compare command refuses to do it — which is the whole + * point: it makes "I changed the chunk size and the score moved" impossible to + * confuse with "I changed the prompt and the score moved". + * + * `hash` is injected so this stays pure and testable; the service passes a + * node:crypto sha256. + */ +export function computeCorpusFingerprint( + inputs: FingerprintInputs, + hash: (input: string) => string +): string { + const docIds = [...inputs.documents.keys()].sort() + const parts: string[] = [ + `chunkTokens=${inputs.chunkTokens}`, + `chunkOverlapTokens=${inputs.chunkOverlapTokens}`, + `charToTokenRatio=${inputs.charToTokenRatio}`, + `embeddingModel=${inputs.embeddingModel}`, + `embeddingDimension=${inputs.embeddingDimension}`, + ] + for (const id of docIds) { + parts.push(`doc=${id}\n${inputs.documents.get(id)}`) + } + return hash(parts.join('\n---\n')).slice(0, 16) +} diff --git a/admin/app/utils/eval/quiet.ts b/admin/app/utils/eval/quiet.ts new file mode 100644 index 0000000..a62bb3c --- /dev/null +++ b/admin/app/utils/eval/quiet.ts @@ -0,0 +1,22 @@ +import logger from '@adonisjs/core/services/logger' + +/** + * Turn down the application logger for the duration of an eval run. + * + * RagService logs every keyword extraction, every batch, and the text of every + * chunk at debug level. That is the right default for diagnosing an ingest, and + * completely wrong for a tool whose entire output is a score table — the + * numbers scroll off the top of the terminal before you can read them. + * + * Returns a restore function so the level is put back even if the caller throws. + * Pass `debug: true` (from `--debug`) to leave the logger alone when you are + * actually trying to see the pipeline internals. + */ +export function quietLogging(debug = false): () => void { + if (debug) return () => {} + const previous = logger.level + logger.level = 'warn' + return () => { + logger.level = previous + } +} diff --git a/admin/app/utils/eval/report.ts b/admin/app/utils/eval/report.ts new file mode 100644 index 0000000..4095316 --- /dev/null +++ b/admin/app/utils/eval/report.ts @@ -0,0 +1,254 @@ +/** + * Report shape, baseline diffing, and the gate decision. + * + * Pure functions over plain objects — the service layer does the file I/O. The + * comparison logic in particular has to be testable without a filesystem, + * because it is the piece that decides whether a pull request is blocked. + */ + +export type ReportKind = 'retrieval' | 'generation' + +/** + * Provenance. Every field here exists to answer "is this report comparable to + * that one?", and `corpusFingerprint` is the one that can veto the comparison + * outright. + */ +export type ReportMeta = { + kind: ReportKind + createdAt: string + corpusFingerprint: string + gitSha: string | null + gitBranch: string | null + gitDirty: boolean + nomadVersion: string | null + /** Host platform. Recorded for context; never used to justify a score. */ + platform: { + cpuArchitecture: string | null + osName: string | null + nodeVersion: string + } + params: Record +} + +/** + * A report is metadata plus a flat bag of named metrics plus the per-case + * detail. Flattening the metrics is deliberate: it means the diff logic does + * not need to know anything about retrieval versus generation, so adding a + * metric later requires no change to the comparison or the gate. + */ +export type EvalReport = { + meta: ReportMeta + /** metric name -> value. null means "not measurable in this run". */ + metrics: Record + /** tag -> metric name -> value. */ + byTag: Record> + /** Per-question detail, for the "what did it actually say" question. */ + cases: unknown[] +} + +/** + * Whether a metric going up is good. + * + * Without this the diff cannot tell an improvement from a regression, and a + * leakage rate falling to zero would be reported as a failure. + */ +export type MetricDirection = 'higher-is-better' | 'lower-is-better' + +export const METRIC_DIRECTIONS: Record = { + // retrieval + 'recall@1': 'higher-is-better', + 'recall@3': 'higher-is-better', + 'recall@5': 'higher-is-better', + 'recall@10': 'higher-is-better', + 'hitRate@1': 'higher-is-better', + 'hitRate@3': 'higher-is-better', + 'hitRate@5': 'higher-is-better', + 'hitRate@10': 'higher-is-better', + 'precision@1': 'higher-is-better', + 'precision@3': 'higher-is-better', + 'precision@5': 'higher-is-better', + 'precision@10': 'higher-is-better', + 'ndcg@1': 'higher-is-better', + 'ndcg@3': 'higher-is-better', + 'ndcg@5': 'higher-is-better', + 'ndcg@10': 'higher-is-better', + mrr: 'higher-is-better', + emptyRateOnAnswerable: 'lower-is-better', + nonEmptyRateOnRefusal: 'lower-is-better', + // generation + correctness: 'higher-is-better', + refusalCorrectness: 'higher-is-better', + leakageRate: 'lower-is-better', + thinkTagLeakRate: 'lower-is-better', + markdownRate: 'higher-is-better', + groundedness: 'higher-is-better', +} + +export type MetricDelta = { + metric: string + baseline: number | null + current: number | null + delta: number | null + direction: MetricDirection + /** Signed so that positive always means "better", whatever the direction. */ + improvement: number | null + regressed: boolean + /** Present in one report but not the other. */ + onlyIn?: 'baseline' | 'current' +} + +export type ComparisonResult = { + comparable: boolean + /** Why the comparison was refused, when it was. */ + incomparableReason?: string + tolerance: number + deltas: MetricDelta[] + regressions: MetricDelta[] + improvements: MetricDelta[] +} + +/** + * Default tolerance band. + * + * Not zero, on purpose. Even at temperature 0 the generation tier moves a + * little between runs, and a gate that fires on 0.001 gets switched off within + * a week. 0.02 is roughly "one question in fifty" on a 99-question set — small + * enough to catch a real regression, large enough to ignore a coin flip. + */ +export const DEFAULT_TOLERANCE = 0.02 + +/** + * Slack for floating-point representation error at the tolerance boundary. + * + * Without it, `0.9 - 0.02` evaluates to a delta of -0.020000000000000018, which + * is "greater than the tolerance" by 1.8e-17 and would block a pull request. A + * gate that fires on the last bit of a double is a gate people learn to ignore. + */ +const BOUNDARY_EPSILON = 1e-9 + +export function compareReports( + baseline: EvalReport, + current: EvalReport, + tolerance: number = DEFAULT_TOLERANCE +): ComparisonResult { + if (baseline.meta.kind !== current.meta.kind) { + return incomparable( + `cannot compare a ${baseline.meta.kind} report against a ${current.meta.kind} report`, + tolerance + ) + } + + // The hard veto. Different corpora, chunk sizes, or embedding models mean the + // two numbers were produced by different experiments, and diffing them would + // manufacture a regression (or hide one) out of nothing. + if (baseline.meta.corpusFingerprint !== current.meta.corpusFingerprint) { + return incomparable( + `corpus fingerprint changed (${baseline.meta.corpusFingerprint} -> ${current.meta.corpusFingerprint}). ` + + 'The corpus, chunk size, or embedding model differs, so these runs are not measuring the same thing. Re-baseline instead.', + tolerance + ) + } + + const names = [...new Set([...Object.keys(baseline.metrics), ...Object.keys(current.metrics)])].sort() + const deltas: MetricDelta[] = names.map((metric) => { + const b = baseline.metrics[metric] ?? null + const c = current.metrics[metric] ?? null + const direction = METRIC_DIRECTIONS[metric] ?? 'higher-is-better' + const onlyIn = + !(metric in baseline.metrics) ? ('current' as const) + : !(metric in current.metrics) ? ('baseline' as const) + : undefined + + if (b === null || c === null) { + return { metric, baseline: b, current: c, delta: null, direction, improvement: null, regressed: false, onlyIn } + } + + const delta = c - b + const improvement = direction === 'higher-is-better' ? delta : -delta + return { + metric, + baseline: b, + current: c, + delta, + direction, + improvement, + regressed: improvement < -(tolerance + BOUNDARY_EPSILON), + onlyIn, + } + }) + + return { + comparable: true, + tolerance, + deltas, + regressions: deltas.filter((d) => d.regressed), + improvements: deltas.filter( + (d) => d.improvement !== null && d.improvement > tolerance + BOUNDARY_EPSILON + ), + } +} + +function incomparable(reason: string, tolerance: number): ComparisonResult { + return { + comparable: false, + incomparableReason: reason, + tolerance, + deltas: [], + regressions: [], + improvements: [], + } +} + +/** Flatten a `Record` into `name@k` keys. */ +export function flattenByK( + prefix: string, + values: Record +): Record { + return Object.fromEntries(Object.entries(values).map(([k, v]) => [`${prefix}@${k}`, v])) +} + +const pct = (v: number | null) => (v === null ? 'n/a' : v.toFixed(4)) + +/** Human-readable summary, worst regressions first. */ +export function renderComparisonMarkdown( + baselinePath: string, + currentPath: string, + result: ComparisonResult +): string { + const lines: string[] = ['# Eval comparison', ''] + lines.push(`- baseline: \`${baselinePath}\``) + lines.push(`- current: \`${currentPath}\``) + lines.push(`- tolerance: ${result.tolerance}`) + lines.push('') + + if (!result.comparable) { + lines.push('## Not comparable', '', result.incomparableReason ?? 'unknown reason', '') + return lines.join('\n') + } + + lines.push( + result.regressions.length > 0 + ? `## ${result.regressions.length} regression(s)` + : '## No regressions' + ) + lines.push('') + lines.push('| metric | baseline | current | delta | verdict |') + lines.push('|---|---:|---:|---:|---|') + + // Worst first — the thing a developer needs is at the top, not sorted + // alphabetically halfway down a wall of unchanged rows. + const ordered = [...result.deltas].sort((a, b) => (a.improvement ?? 0) - (b.improvement ?? 0)) + for (const d of ordered) { + const verdict = + d.onlyIn === 'current' ? 'new' + : d.onlyIn === 'baseline' ? 'removed' + : d.delta === null ? '—' + : d.regressed ? '**REGRESSED**' + : (d.improvement ?? 0) > result.tolerance ? 'improved' + : 'within tolerance' + const arrow = d.delta === null ? 'n/a' : `${d.delta >= 0 ? '+' : ''}${d.delta.toFixed(4)}` + lines.push(`| ${d.metric} | ${pct(d.baseline)} | ${pct(d.current)} | ${arrow} | ${verdict} |`) + } + lines.push('') + return lines.join('\n') +} diff --git a/admin/app/utils/eval/retrieval_metrics.ts b/admin/app/utils/eval/retrieval_metrics.ts new file mode 100644 index 0000000..32a5717 --- /dev/null +++ b/admin/app/utils/eval/retrieval_metrics.ts @@ -0,0 +1,323 @@ +/** + * Information-retrieval metrics for the RAG eval harness. + * + * Pure functions over ranked results — no I/O, no models, no clock. Everything + * here is deterministic, which is the point: a change in these numbers is + * unambiguously a change in the code, never in the weather. + * + * ## Two levels of measurement, and why both + * + * Retrieval returns *chunks*, but a golden answer lives in a *document*. The + * two questions we care about are different, so they are measured differently: + * + * - **Document level** (recall, MRR, nDCG): "did the answer's document make it + * into the context at all, and how near the top?" Chunks are collapsed to + * their document, keeping each document's best rank. + * - **Chunk level** (precision): "how much of what we injected is noise?" This + * is deliberately *not* deduped — five chunks from one irrelevant document + * cost five slots of a small model's context and should be counted five + * times. + * + * Reporting only one of these hides a real failure mode in the other. + */ + +/** One retrieved chunk, reduced to what scoring needs. */ +export type ScoredChunk = { + /** Corpus document this chunk came from; null if it could not be resolved. */ + docId: string | null + /** The pipeline's final (post-rerank) score. */ + score: number + /** The raw pre-rerank cosine score, when available. */ + semanticScore?: number +} + +export type RetrievalCase = { + id: string + tags: string[] + /** Ranked chunks, best first, exactly as the pipeline returned them. */ + retrieved: ScoredChunk[] + /** Documents that genuinely answer the question. */ + relevantDocIds: string[] + /** True when the right behaviour is to retrieve nothing useful. */ + expectRefusal: boolean +} + +/** + * Collapse a ranked chunk list to a ranked document list, keeping each + * document's best (earliest) position. Chunks with no resolvable document are + * dropped — they still occupy a rank in the chunk-level metrics, but they + * cannot be credited to any document. + */ +export function toDocumentRanking(retrieved: ScoredChunk[]): string[] { + const seen = new Set() + const ranking: string[] = [] + for (const chunk of retrieved) { + if (!chunk.docId || seen.has(chunk.docId)) continue + seen.add(chunk.docId) + ranking.push(chunk.docId) + } + return ranking +} + +/** + * Fraction of the relevant documents that appear within the top `k` chunks. + * + * Returns null when there are no relevant documents (an out-of-corpus case) — + * recall is undefined there, and returning 0 would drag the mean down for + * questions that are *supposed* to retrieve nothing. + */ +export function recallAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null { + if (relevantDocIds.length === 0) return null + const relevant = new Set(relevantDocIds) + const found = new Set() + for (const chunk of retrieved.slice(0, k)) { + if (chunk.docId && relevant.has(chunk.docId)) found.add(chunk.docId) + } + return found.size / relevant.size +} + +/** + * 1 if *any* relevant document appears in the top `k`, else 0. + * + * Distinct from recall on multi-hop questions: finding one of two required + * documents is a hit but only 0.5 recall. Reporting both is what separates + * "found something" from "found enough to answer". + */ +export function hitRateAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null { + if (relevantDocIds.length === 0) return null + const relevant = new Set(relevantDocIds) + return retrieved.slice(0, k).some((c) => c.docId && relevant.has(c.docId)) ? 1 : 0 +} + +/** + * Fraction of the top `k` *chunks* that come from a relevant document. + * + * Not deduped, on purpose — this measures context pollution, and a small model + * drowning in four irrelevant chunks does not care that they share a source. + * Denominator is min(k, retrieved.length) so a pipeline returning 2 good chunks + * is not punished for the 3 it correctly declined to return. + */ +export function precisionAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null { + if (relevantDocIds.length === 0) return null + const window = retrieved.slice(0, k) + if (window.length === 0) return 0 + const relevant = new Set(relevantDocIds) + const hits = window.filter((c) => c.docId && relevant.has(c.docId)).length + return hits / window.length +} + +/** + * Reciprocal of the rank of the first relevant chunk (1-indexed); 0 if none. + * Averaged over cases this is MRR. + */ +export function reciprocalRank(retrieved: ScoredChunk[], relevantDocIds: string[]): number | null { + if (relevantDocIds.length === 0) return null + const relevant = new Set(relevantDocIds) + const idx = retrieved.findIndex((c) => c.docId && relevant.has(c.docId)) + return idx === -1 ? 0 : 1 / (idx + 1) +} + +/** + * Normalized discounted cumulative gain at `k`, over the **document** ranking, + * with binary gain. + * + * DCG@k = Σ_{i=1..k} rel_i / log2(i + 1) + * IDCG@k = Σ_{i=1..min(R, k)} 1 / log2(i + 1) where R = |relevant| + * nDCG@k = DCG@k / IDCG@k + * + * The ideal ranking is defined against the *known* number of relevant documents + * rather than against however many happened to be retrieved. That distinction + * matters: normalizing against the retrieved set would score a run that found + * one of three required documents, and ranked it first, as a perfect 1.0. + * + * This is the metric that catches "right documents, wrong order" — a reranking + * regression that leaves recall untouched while pushing the answer to position + * five, where a 1B model with a 2-result budget will never see it. + */ +export function ndcgAtK(retrieved: ScoredChunk[], relevantDocIds: string[], k: number): number | null { + if (relevantDocIds.length === 0) return null + const relevant = new Set(relevantDocIds) + const docRanking = toDocumentRanking(retrieved).slice(0, k) + + let dcg = 0 + docRanking.forEach((docId, i) => { + if (relevant.has(docId)) dcg += 1 / Math.log2(i + 2) // i is 0-indexed, so rank = i + 1 + }) + + let idcg = 0 + for (let i = 0; i < Math.min(relevant.size, k); i++) { + idcg += 1 / Math.log2(i + 2) + } + + return idcg === 0 ? 0 : dcg / idcg +} + +/** + * Score separation: the gap between what relevant and irrelevant chunks score. + * + * This is how the similarity threshold gets calibrated with evidence instead of + * intuition. If the relevant p10 sits below the irrelevant p90, no threshold + * can cleanly separate them and the honest conclusion is that the *retriever* + * needs work, not the cutoff. + */ +export type ScoreDistribution = { + count: number + min: number + p10: number + median: number + p90: number + max: number + mean: number +} + +export function describeScores(scores: number[]): ScoreDistribution | null { + if (scores.length === 0) return null + const sorted = [...scores].sort((a, b) => a - b) + const pct = (p: number) => { + // Nearest-rank percentile: no interpolation, so a reported value is always + // a value that actually occurred. + const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)) + return sorted[idx] + } + return { + count: sorted.length, + min: sorted[0], + p10: pct(10), + median: pct(50), + p90: pct(90), + max: sorted[sorted.length - 1], + mean: sorted.reduce((a, b) => a + b, 0) / sorted.length, + } +} + +/** Mean over the entries that are actually defined; null if none are. */ +export function meanOf(values: Array): number | null { + const defined = values.filter((v): v is number => v !== null) + if (defined.length === 0) return null + return defined.reduce((a, b) => a + b, 0) / defined.length +} + +export type RetrievalCaseResult = { + id: string + tags: string[] + retrievedCount: number + relevantDocIds: string[] + retrievedDocIds: string[] + expectRefusal: boolean + recall: Record + hitRate: Record + precision: Record + ndcg: Record + reciprocalRank: number | null + /** True when this case retrieved nothing at all. */ + empty: boolean +} + +export type RetrievalAggregate = { + cases: number + /** Cases with at least one relevant document (i.e. excluding refusal cases). */ + answerable: number + recall: Record + hitRate: Record + precision: Record + ndcg: Record + mrr: number | null + /** + * Fraction of *answerable* questions that retrieved nothing. A non-zero value + * means the score threshold is filtering out real answers. + */ + emptyRateOnAnswerable: number | null + /** + * Fraction of *out-of-corpus* questions that retrieved something anyway. This + * is the other half of the threshold trade-off: context handed to the model + * for a question the corpus cannot answer is exactly what produces a + * confident, wrong reply. + */ + nonEmptyRateOnRefusal: number | null + relevantScores: ScoreDistribution | null + irrelevantScores: ScoreDistribution | null +} + +export const DEFAULT_K_VALUES = [1, 3, 5, 10] + +export function scoreCase(c: RetrievalCase, kValues: number[] = DEFAULT_K_VALUES): RetrievalCaseResult { + const byK = (fn: (k: number) => T): Record => + Object.fromEntries(kValues.map((k) => [k, fn(k)])) + + return { + id: c.id, + tags: c.tags, + retrievedCount: c.retrieved.length, + relevantDocIds: c.relevantDocIds, + retrievedDocIds: toDocumentRanking(c.retrieved), + expectRefusal: c.expectRefusal, + recall: byK((k) => recallAtK(c.retrieved, c.relevantDocIds, k)), + hitRate: byK((k) => hitRateAtK(c.retrieved, c.relevantDocIds, k)), + precision: byK((k) => precisionAtK(c.retrieved, c.relevantDocIds, k)), + ndcg: byK((k) => ndcgAtK(c.retrieved, c.relevantDocIds, k)), + reciprocalRank: reciprocalRank(c.retrieved, c.relevantDocIds), + empty: c.retrieved.length === 0, + } +} + +export function aggregate( + cases: RetrievalCase[], + results: RetrievalCaseResult[], + kValues: number[] = DEFAULT_K_VALUES +): RetrievalAggregate { + const answerable = results.filter((r) => !r.expectRefusal) + const refusals = results.filter((r) => r.expectRefusal) + + const byK = (pick: (r: RetrievalCaseResult, k: number) => number | null): Record => + Object.fromEntries(kValues.map((k) => [k, meanOf(results.map((r) => pick(r, k)))])) + + // Split every retrieved chunk's score by whether its document was relevant. + // Refusal cases contribute only to the irrelevant side — by definition + // nothing they retrieve is relevant, and that is precisely the population a + // threshold needs to exclude. + const relevantScores: number[] = [] + const irrelevantScores: number[] = [] + for (const c of cases) { + const relevant = new Set(c.relevantDocIds) + for (const chunk of c.retrieved) { + const bucket = chunk.docId && relevant.has(chunk.docId) ? relevantScores : irrelevantScores + bucket.push(chunk.semanticScore ?? chunk.score) + } + } + + return { + cases: results.length, + answerable: answerable.length, + recall: byK((r, k) => r.recall[k]), + hitRate: byK((r, k) => r.hitRate[k]), + precision: byK((r, k) => r.precision[k]), + ndcg: byK((r, k) => r.ndcg[k]), + mrr: meanOf(results.map((r) => r.reciprocalRank)), + emptyRateOnAnswerable: + answerable.length === 0 ? null : answerable.filter((r) => r.empty).length / answerable.length, + nonEmptyRateOnRefusal: + refusals.length === 0 ? null : refusals.filter((r) => !r.empty).length / refusals.length, + relevantScores: describeScores(relevantScores), + irrelevantScores: describeScores(irrelevantScores), + } +} + +/** Aggregate restricted to cases carrying a given tag, for per-slice reporting. */ +export function aggregateByTag( + cases: RetrievalCase[], + results: RetrievalCaseResult[], + kValues: number[] = DEFAULT_K_VALUES +): Record { + const tags = new Set(results.flatMap((r) => r.tags)) + const byId = new Map(cases.map((c) => [c.id, c])) + const out: Record = {} + for (const tag of [...tags].sort()) { + const tagged = results.filter((r) => r.tags.includes(tag)) + out[tag] = aggregate( + tagged.map((r) => byId.get(r.id)!).filter(Boolean), + tagged, + kValues + ) + } + return out +} diff --git a/admin/app/utils/rag_prompt.ts b/admin/app/utils/rag_prompt.ts new file mode 100644 index 0000000..ca2b61f --- /dev/null +++ b/admin/app/utils/rag_prompt.ts @@ -0,0 +1,123 @@ +/** + * Pure prompt-budgeting helpers for the chat RAG pipeline. + * + * These live here rather than in RagPipelineService so they can be exercised + * under bare `node --experimental-strip-types` with no MySQL, Redis, Qdrant, or + * Ollama — the same reason `kb_ratio_lookup.ts` is shaped this way. The service + * supplies the config; everything below is a function of its arguments. + */ + +/** + * Chars-per-token estimate used when budgeting the *prompt* (context trimming + * and the num_ctx ladder). + * + * NOTE: RagService.CHAR_TO_TOKEN_RATIO is 2, not 3.5 — the two halves of the + * system disagree about what a token costs, and RagService's own doc-comment + * says 3. That divergence is real and known; reconciling it changes chunk size + * and therefore retrieval results, so it is deliberately left alone here and + * tracked as its own measured change rather than folded into a refactor. + */ +export const PROMPT_CHARS_PER_TOKEN = 3.5 + +/** + * num_ctx is only requested once the system prompt is large enough to risk + * overflowing Ollama's silent 2048 default. Below the trigger we send nothing + * and inherit the server default. + */ +export const NUM_CTX_TRIGGER_TOKENS = 3000 +export const NUM_CTX_RESPONSE_HEADROOM = 2048 +export const NUM_CTX_LADDER = [8192, 16384, 32768, 65536] + +export type ContextLimits = { maxResults: number; maxTokens: number } +export type ContextLimitTier = { maxParams: number; maxResults: number; maxTokens: number } + +/** The minimum shape these helpers need; the real chunks carry more. */ +export type BudgetableChunk = { text: string; metadata?: Record } +export type BudgetableMessage = { role: 'system' | 'user' | 'assistant'; content: string } + +/** + * Determine RAG context limits from the model size encoded in its name. + * Parses size indicators like "1b", "3b", "8b", "70b". + * + * PRESERVED QUIRK: an unparseable model name is treated as 8B. That is a guess, + * and for a name like "phi3" or a custom tag it can hand a small model far more + * context than it can actually use. Faithful to the pre-extraction behaviour. + */ +export function getContextLimitsForModel( + modelName: string, + tiers: readonly ContextLimitTier[] +): ContextLimits { + // e.g. "llama3.2:3b", "qwen2.5:1.5b", "gemma:7b" + const sizeMatch = modelName.match(/(\d+\.?\d*)[bB]/) + const paramBillions = sizeMatch ? Number.parseFloat(sizeMatch[1]) : 8 // default to 8B if unknown + + for (const tier of tiers) { + if (paramBillions <= tier.maxParams) { + return { maxResults: tier.maxResults, maxTokens: tier.maxTokens } + } + } + + return { maxResults: 5, maxTokens: 0 } +} + +/** + * Apply the model-size context budget: cap the result count, then cap total + * characters. + * + * The first (most relevant) result is always kept — the token cap only gates + * subsequent results, so a single oversized chunk never starves the model of + * context entirely. + */ +export function trimToContextBudget( + docs: T[], + limits: ContextLimits +): T[] { + const byCount = docs.slice(0, limits.maxResults) + if (limits.maxTokens <= 0) return byCount + + const charCap = limits.maxTokens * PROMPT_CHARS_PER_TOKEN + let totalChars = 0 + return byCount.filter((doc, idx) => { + totalChars += doc.text.length + return idx === 0 || totalChars <= charCap + }) +} + +/** + * Render retrieved chunks into the block the rag_context prompt wraps. + * + * Each block is labelled with its source title when one is available — a + * neutral, honest provenance signal — but never with the raw relevance score. + * nomic cosine scores for genuinely relevant passages sit around 0.4-0.6, and + * surfacing e.g. "42%" primes the model to distrust correct context. Scores + * stay in the logs and in the eval report. + */ +export function buildContextBlock(docs: BudgetableChunk[]): string { + return docs + .map((doc, idx) => { + const title = doc.metadata?.full_title || doc.metadata?.article_title + const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]` + return `${label}\n${doc.text}` + }) + .join('\n\n') +} + +/** + * Request a context window big enough to hold the system messages, but only + * once they are large enough to be at risk. + * + * Ollama respects num_ctx per request; LM Studio ignores it gracefully. Below + * the trigger we send nothing and inherit the server default — which for Ollama + * is a silent 2048. + */ +export function deriveNumCtx(messages: BudgetableMessage[]): number | undefined { + const systemChars = messages + .filter((m) => m.role === 'system') + .reduce((sum, m) => sum + m.content.length, 0) + const estimatedSystemTokens = Math.ceil(systemChars / PROMPT_CHARS_PER_TOKEN) + + if (estimatedSystemTokens <= NUM_CTX_TRIGGER_TOKENS) return undefined + + const needed = estimatedSystemTokens + NUM_CTX_RESPONSE_HEADROOM + return NUM_CTX_LADDER.find((n) => n >= needed) ?? NUM_CTX_LADDER[NUM_CTX_LADDER.length - 1] +} diff --git a/admin/commands/eval/compare.ts b/admin/commands/eval/compare.ts new file mode 100644 index 0000000..1ae4e34 --- /dev/null +++ b/admin/commands/eval/compare.ts @@ -0,0 +1,127 @@ +import { BaseCommand, args, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' + +/** + * Diff an eval report against a baseline and fail on regression. + * + * This is the "did my change help?" command, and the one CI would call. + * + * node ace eval:compare tests/eval/baselines//retrieval.json tests/eval/reports/latest.json + * node ace eval:compare base.json current.json --tolerance=0.05 + * node ace eval:compare --promote=retrieval tests/eval/reports/latest.json + */ +export default class EvalCompare extends BaseCommand { + static commandName = 'eval:compare' + static description = 'Diff an eval report against a baseline; exits non-zero on regression' + + @args.string({ description: 'Baseline report JSON (or the report to promote, with --promote)' }) + declare baseline: string + + @args.string({ description: 'Current report JSON', required: false }) + declare current: string + + @flags.string({ description: 'Regression tolerance per metric (default: 0.02)' }) + declare tolerance: string + + @flags.string({ + description: 'Promote the given report to a committed baseline under this name, then exit', + }) + declare promote: string + + @flags.boolean({ description: 'Print the full metric table, not just the regressions' }) + declare verbose: boolean + + /** + * Deliberately does not boot the application. + * + * Comparing two report files is pure I/O — it needs no MySQL, no Redis, no + * Qdrant, and no Ollama. Keeping it that way is what lets CI run the gate on + * a committed baseline without standing up the whole stack, and it avoids the + * app's noisy Redis teardown on exit. + */ + static options: CommandOptions = { + startApp: false, + } + + async run() { + const { EvalReportService } = await import('#services/eval_report_service') + const { compareReports, DEFAULT_TOLERANCE } = await import('../../app/utils/eval/report.js') + const service = new EvalReportService() + + try { + if (this.promote) { + const report = await service.read(this.baseline) + const path = await service.promoteToBaseline(report, this.promote) + this.logger.success(`Promoted to baseline: ${path}`) + this.logger.info(`Corpus fingerprint: ${report.meta.corpusFingerprint}`) + this.logger.info('Commit this file so the whole team gates against the same numbers.') + return + } + + if (!this.current) { + this.logger.error('Two report paths are required (or use --promote= with one).') + this.exitCode = 1 + return + } + + const [baseline, current] = await Promise.all([ + service.read(this.baseline), + service.read(this.current), + ]) + const tolerance = this.tolerance ? Number.parseFloat(this.tolerance) : DEFAULT_TOLERANCE + const result = compareReports(baseline, current, tolerance) + + if (!result.comparable) { + // Refusing is the correct outcome, not a failure of the tool — diffing + // runs from different corpora would manufacture a regression. + this.logger.error('Reports are not comparable:') + this.logger.error(` ${result.incomparableReason}`) + this.exitCode = 1 + return + } + + this.logger.info( + `baseline ${short(baseline.meta.gitSha)} (${baseline.meta.createdAt}) -> current ${short(current.meta.gitSha)} (${current.meta.createdAt})` + ) + this.logger.info(`corpus ${current.meta.corpusFingerprint} · tolerance ${tolerance}`) + if (current.meta.gitDirty) { + this.logger.warning('Current report was produced from a dirty working tree.') + } + this.logger.info('') + + const rows = this.verbose + ? [...result.deltas].sort((a, b) => (a.improvement ?? 0) - (b.improvement ?? 0)) + : [...result.regressions, ...result.improvements] + + if (rows.length === 0) { + this.logger.success('No metric moved outside the tolerance band.') + } else { + for (const d of rows) { + const line = ` ${d.metric.padEnd(24)} ${fmt(d.baseline)} -> ${fmt(d.current)} ${signed(d.delta)}` + if (d.regressed) this.logger.error(`${line} REGRESSED`) + else if ((d.improvement ?? 0) > tolerance) this.logger.success(`${line} improved`) + else this.logger.info(line) + } + } + + this.logger.info('') + if (result.regressions.length > 0) { + this.logger.error( + `${result.regressions.length} metric(s) regressed beyond the ${tolerance} tolerance.` + ) + this.exitCode = 1 + } else { + this.logger.success( + `No regressions. ${result.improvements.length} metric(s) improved beyond tolerance.` + ) + } + } catch (error) { + this.logger.error(error instanceof Error ? error.message : String(error)) + this.exitCode = 1 + } + } +} + +const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(4).padStart(6)) +const signed = (v: number | null) => (v === null ? ' ' : `${v >= 0 ? '+' : ''}${v.toFixed(4)}`) +const short = (sha: string | null) => (sha ? sha.slice(0, 7) : 'unknown') diff --git a/admin/commands/eval/corpus.ts b/admin/commands/eval/corpus.ts new file mode 100644 index 0000000..1f12d19 --- /dev/null +++ b/admin/commands/eval/corpus.ts @@ -0,0 +1,155 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' + +/** + * Manage the frozen evaluation corpus. + * + * # Validate the corpus and goldens without touching the vector store + * node ace eval:corpus --check + * + * # Show what is currently ingested + * node ace eval:corpus --status + * + * # Wipe and rebuild (the normal path; safe — never touches user content) + * node ace eval:corpus --ingest + * + * # Remove the eval corpus entirely + * node ace eval:corpus --reset + */ +export default class EvalCorpus extends BaseCommand { + static commandName = 'eval:corpus' + static description = 'Manage the frozen RAG evaluation corpus (ingest, reset, status, check)' + + @flags.boolean({ description: 'Wipe and re-ingest the corpus into the reserved eval collection' }) + declare ingest: boolean + + @flags.boolean({ description: 'Remove every eval corpus chunk from the vector store' }) + declare reset: boolean + + @flags.boolean({ description: 'Show the fingerprint and current chunk count' }) + declare status: boolean + + @flags.boolean({ + description: 'Validate the corpus and goldens only — no Qdrant, no Ollama, no writes', + }) + declare check: boolean + + @flags.boolean({ description: 'Leave application debug logging on (very noisy)' }) + declare debug: boolean + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { EvalCorpusService } = await import('#services/eval_corpus_service') + const { quietLogging } = await import('../../app/utils/eval/quiet.js') + const service = await this.app.container.make(EvalCorpusService) + const restoreLogging = quietLogging(this.debug) + try { + await this.dispatch(service) + } finally { + restoreLogging() + } + } + + private async dispatch(service: any) { + + // Default to --status so a bare invocation is always safe. + if (!this.ingest && !this.reset && !this.check) this.status = true + + if (this.check) { + return this.runCheck(service) + } + + if (this.reset) { + const removed = await service.reset() + this.logger.success(`Removed ${removed} eval corpus chunks`) + return + } + + if (this.ingest) { + return this.runIngest(service) + } + + const [fingerprint, chunks, goldens] = await Promise.all([ + service.fingerprint(), + service.count(), + service.loadGoldens(), + ]) + this.logger.info(`Fingerprint: ${fingerprint}`) + this.logger.info(`Chunks in KB: ${chunks}`) + this.logger.info(`Goldens: ${goldens.length}`) + if (chunks === 0) { + this.logger.warning('Corpus is not ingested. Run: node ace eval:corpus --ingest') + } + } + + /** + * Validation-only path. Deliberately avoids the container-resolved services' + * network dependencies so a contributor can check their golden edits without + * a running stack. + */ + private async runCheck(service: any) { + try { + const corpus = await service.loadCorpus() + const goldens = await service.loadGoldens() + const fingerprint = await service.fingerprint() + + const tagCounts = new Map() + for (const g of goldens) { + for (const tag of g.tags) tagCounts.set(tag, (tagCounts.get(tag) ?? 0) + 1) + } + + // A document no golden asks about is dead weight in the corpus: it costs + // ingest time and adds distractor noise nobody chose deliberately. + const referenced = new Set(goldens.flatMap((g: any) => g.relevantDocIds)) + const unreferenced = corpus + .map((d: any) => d.docId) + .filter((id: string) => !referenced.has(id)) + + this.logger.success(`Corpus OK: ${corpus.length} documents, ${goldens.length} goldens`) + this.logger.info(`Fingerprint: ${fingerprint}`) + this.logger.info( + `Tags: ${[...tagCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([t, n]) => `${t}=${n}`) + .join(' ')}` + ) + const refusals = goldens.filter((g: any) => g.expectRefusal).length + this.logger.info(`Refusal cases: ${refusals} / ${goldens.length}`) + if (unreferenced.length > 0) { + this.logger.warning(`Documents no golden references: ${unreferenced.join(', ')}`) + } + } catch (error) { + this.logger.error(error instanceof Error ? error.message : String(error)) + this.exitCode = 1 + } + } + + private async runIngest(service: any) { + this.logger.info('Rebuilding the eval corpus (user content is never touched)...') + // Validate before destroying anything — a typo in a golden should not cost + // you the ingest you were about to run. + await service.loadGoldens() + + const summary = await service.ingest((docId: string, index: number, total: number) => { + this.logger.info(` [${index}/${total}] ${docId}`) + }) + + this.logger.info('') + if (summary.removedBeforeIngest > 0) { + this.logger.info(`Removed ${summary.removedBeforeIngest} chunks from the previous ingest`) + } + this.logger.success(`Ingested ${summary.documents} documents into ${summary.chunks} chunks`) + this.logger.info(`Fingerprint: ${summary.fingerprint}`) + + if (summary.failures.length > 0) { + this.logger.error(`${summary.failures.length} document(s) failed to ingest:`) + for (const f of summary.failures) this.logger.error(` ${f.docId}: ${f.reason}`) + // A partial corpus produces scores that look real and are not. + this.logger.error('The corpus is incomplete — results from this state are not comparable.') + this.exitCode = 1 + } + } +} diff --git a/admin/commands/eval/generation.ts b/admin/commands/eval/generation.ts new file mode 100644 index 0000000..2de609d --- /dev/null +++ b/admin/commands/eval/generation.ts @@ -0,0 +1,238 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import type { + GenerationAggregate, + GenerationMode, +} from '../../app/services/eval_generation_service.js' + +const MODES: GenerationMode[] = ['oracle', 'e2e', 'noretrieval'] + +/** + * Score generated answers against the golden set. + * + * The `--mode` flag is the whole point of this command: + * + * oracle perfect context injected by construction -> isolates the MODEL + * e2e real retrieval -> the real product + * noretrieval no context at all -> parametric baseline + * + * Run all three on the same model and the ambiguous complaint "the AI gave a + * bad answer" decomposes into a number that can be acted on. + * + * node ace eval:generation --model=llama3.2:latest --mode=oracle + * node ace eval:generation --model=llama3:8b --all-modes + * node ace eval:generation --model=mock # no Ollama needed + */ +export default class EvalGeneration extends BaseCommand { + static commandName = 'eval:generation' + static description = 'Score generated answers, with oracle/e2e/noretrieval ablation' + + @flags.string({ description: 'Ollama model to evaluate, or "mock" for the extractive ceiling' }) + declare model: string + + @flags.string({ description: `One of: ${MODES.join(', ')} (default: e2e)` }) + declare mode: string + + @flags.boolean({ description: 'Run all three modes and print the decomposition' }) + declare allModes: boolean + + @flags.string({ description: 'Repeats per question, for stability (default: 3)' }) + declare repeats: string + + @flags.string({ description: 'Only run goldens carrying this tag' }) + declare tag: string + + @flags.string({ description: 'Limit to the first N goldens (for a quick smoke run)' }) + declare limit: string + + @flags.boolean({ description: 'Print every failing question with the answer the model gave' }) + declare verbose: boolean + + @flags.boolean({ description: 'Leave application debug logging on (very noisy)' }) + declare debug: boolean + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { EvalCorpusService } = await import('#services/eval_corpus_service') + const { EvalGenerationService, MOCK_MODEL } = await import('#services/eval_generation_service') + const { quietLogging } = await import('../../app/utils/eval/quiet.js') + + const corpusService = await this.app.container.make(EvalCorpusService) + const generationService = await this.app.container.make(EvalGenerationService) + const restoreLogging = quietLogging(this.debug) + + try { + if (!this.model) { + this.logger.error('--model is required (use --model=mock to run without Ollama)') + this.exitCode = 1 + return + } + + const modes: GenerationMode[] = this.allModes + ? MODES + : [(this.mode as GenerationMode) || 'e2e'] + for (const mode of modes) { + if (!MODES.includes(mode)) { + this.logger.error(`Unknown mode "${mode}". Expected one of: ${MODES.join(', ')}`) + this.exitCode = 1 + return + } + } + + let goldens = await corpusService.loadGoldens() + if (this.tag) goldens = goldens.filter((g) => g.tags.includes(this.tag)) + if (this.limit) goldens = goldens.slice(0, Number.parseInt(this.limit, 10)) + if (goldens.length === 0) { + this.logger.error('No goldens matched the given filters') + this.exitCode = 1 + return + } + + // e2e and oracle both need the corpus present; noretrieval does not, but + // requiring it uniformly keeps the three modes comparable. + const chunks = await corpusService.count() + if (chunks === 0 && modes.some((m) => m !== 'noretrieval')) { + this.logger.error('The eval corpus is not ingested. Run: node ace eval:corpus --ingest') + this.exitCode = 1 + return + } + + const fingerprint = await corpusService.fingerprint() + const repeats = this.repeats ? Number.parseInt(this.repeats, 10) : 3 + this.logger.info( + `Corpus ${fingerprint} · ${goldens.length} goldens · model=${this.model} · repeats=${repeats}` + ) + if (this.model === MOCK_MODEL) { + this.logger.info( + 'Mock model: answers are the injected context verbatim — this is the extractive ceiling for the current retrieval, not a real model.' + ) + } + this.logger.info('') + + const summaries: Array<{ mode: GenerationMode; agg: GenerationAggregate }> = [] + + for (const mode of modes) { + const result = await generationService.run(goldens, { + mode, + model: this.model, + repeats, + onProgress: (id, index, total) => { + if (this.verbose) this.logger.info(` [${index}/${total}] ${mode}: ${id}`) + }, + }) + summaries.push({ mode, agg: result.overall }) + + this.printAggregate(mode, result.overall, result.elapsedMs) + + if (result.overall.errors > 0) { + this.logger.error( + `${result.overall.errors} question(s) errored talking to the model — the scores above are incomplete.` + ) + const firstError = result.cases.find((c) => c.error) + if (firstError) this.logger.error(` first error (${firstError.id}): ${firstError.error}`) + this.exitCode = 1 + } + + if (this.verbose) this.printFailures(result.cases) + } + + if (summaries.length > 1) this.printDecomposition(summaries) + } catch (error) { + this.logger.error(error instanceof Error ? error.message : String(error)) + this.exitCode = 1 + } finally { + restoreLogging() + } + } + + private printAggregate(mode: GenerationMode, agg: GenerationAggregate, elapsedMs: number) { + this.logger.info(`=== ${mode.toUpperCase()} (${(elapsedMs / 1000).toFixed(1)}s) ===`) + this.logger.info(` correctness ${fmt(agg.correctness)}`) + this.logger.info(` refusal correct ${fmt(agg.refusalCorrectness)}`) + this.logger.info(` leakage rate ${fmt(agg.leakageRate)} (lower is better)`) + this.logger.info(` think-tag leak ${fmt(agg.thinkTagLeakRate)} (should be 0)`) + this.logger.info(` markdown formatted ${fmt(agg.markdownRate)}`) + this.logger.info( + ` numeric grounding ${agg.groundedness ? `${agg.groundedness.mean.toFixed(3)} (n=${agg.groundedness.n})` : ' n/a'}` + ) + this.logger.info( + ` mean answer length ${agg.meanAnswerLength === null ? 'n/a' : Math.round(agg.meanAnswerLength)} chars` + ) + if (agg.unstable > 0) { + this.logger.warning( + ` ${agg.unstable} question(s) flipped between repeats — excluded from gating, do not read them as a regression.` + ) + } + this.logger.info('') + } + + /** + * The reason this command exists: turn three scores into an attribution. + */ + private printDecomposition(summaries: Array<{ mode: GenerationMode; agg: GenerationAggregate }>) { + const get = (mode: GenerationMode) => summaries.find((s) => s.mode === mode)?.agg.correctness ?? null + const oracle = get('oracle') + const e2e = get('e2e') + const none = get('noretrieval') + + this.logger.info('=== Decomposition ===') + if (oracle !== null && e2e !== null) { + const cost = oracle - e2e + this.logger.info(` retrieval cost ${cost.toFixed(3)} (oracle ${oracle.toFixed(3)} - e2e ${e2e.toFixed(3)})`) + this.logger.info(` model ceiling ${oracle.toFixed(3)} (what this model manages with perfect context)`) + } + if (e2e !== null && none !== null) { + this.logger.info(` RAG contribution ${(e2e - none).toFixed(3)} (e2e ${e2e.toFixed(3)} - noretrieval ${none.toFixed(3)})`) + } + + if (oracle !== null && e2e !== null) { + // The actual triage rule, stated so nobody has to re-derive it. + const cost = oracle - e2e + if (oracle < 0.6) { + this.logger.warning( + ' Oracle is low: this model struggles even with perfect context. Retrieval work will not fix it — try a larger model.' + ) + } else if (cost > 0.15) { + this.logger.warning( + ' Large retrieval cost: the model can use good context but is not being given it. Work on retrieval.' + ) + } else { + this.logger.success(' Model and retrieval are both holding up on this corpus.') + } + } + if (none !== null && e2e !== null && e2e <= none) { + this.logger.warning( + ' RAG is not adding anything over the bare model on this set — check that retrieval is actually reaching the prompt.' + ) + } + } + + private printFailures(cases: Array) { + const failures = cases.filter((c) => c.correctness.passRate < 1 || c.refusalCorrectness.passRate < 1) + if (failures.length === 0) return + this.logger.info('--- failures ---') + for (const f of failures) { + this.logger.info(` ${f.id} correct=${f.correctness.passes}/${f.correctness.repeats} refusal=${f.refusalCorrectness.passes}/${f.refusalCorrectness.repeats}`) + const score = f.scores[0] + if (score?.missedRequired?.length) { + this.logger.info(` missed: ${score.missedRequired.join(' | ')}`) + } + if (score?.hitForbidden?.length) { + this.logger.info(` said forbidden: ${score.hitForbidden.join(' | ')}`) + } + if (score?.leakage?.length) { + this.logger.info(` narrated retrieval`) + } + if (f.answers[0]) { + const preview = f.answers[0].replace(/\s+/g, ' ').slice(0, 220) + this.logger.info(` said: ${preview}${f.answers[0].length > 220 ? '…' : ''}`) + } + } + this.logger.info('') + } +} + +const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3)) diff --git a/admin/commands/eval/matrix.ts b/admin/commands/eval/matrix.ts new file mode 100644 index 0000000..1522257 --- /dev/null +++ b/admin/commands/eval/matrix.ts @@ -0,0 +1,178 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import type { GenerationAggregate } from '../../app/services/eval_generation_service.js' + +type Row = { + model: string + oracle: GenerationAggregate | null + e2e: GenerationAggregate | null + error?: string +} + +/** + * Build the model capability table. + * + * This is the artifact that answers a GitHub issue. When a user reports "the + * AI gave me a bad answer", look up their model in this table: + * + * - scoring at or near its row -> the model is at its ceiling. The honest + * answer is "run a larger model", not a bug. + * - scoring well below its row -> something is wrong with their config or + * our code, and it is worth investigating. + * + * Without it, every quality concern is unfalsifiable. + * + * node ace eval:matrix --models=llama3.2:latest,llama3:8b --limit=30 + * node ace eval:matrix --models=... --promote # commit as the reference table + */ +export default class EvalMatrix extends BaseCommand { + static commandName = 'eval:matrix' + static description = 'Score several models side by side to build the capability reference table' + + @flags.string({ description: 'Comma-separated Ollama model names' }) + declare models: string + + @flags.string({ description: 'Repeats per question (default: 1 — the matrix is already long)' }) + declare repeats: string + + @flags.string({ description: 'Limit to the first N goldens' }) + declare limit: string + + @flags.string({ description: 'Only run goldens carrying this tag' }) + declare tag: string + + @flags.boolean({ description: 'Write the table to tests/eval/baselines//matrix.json' }) + declare promote: boolean + + @flags.boolean({ description: 'Leave application debug logging on (very noisy)' }) + declare debug: boolean + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { EvalCorpusService } = await import('#services/eval_corpus_service') + const { EvalGenerationService } = await import('#services/eval_generation_service') + const { quietLogging } = await import('../../app/utils/eval/quiet.js') + const { mkdir, writeFile } = await import('node:fs/promises') + const { dirname, join, resolve } = await import('node:path') + + const corpusService = await this.app.container.make(EvalCorpusService) + const generationService = await this.app.container.make(EvalGenerationService) + const restoreLogging = quietLogging(this.debug) + + try { + if (!this.models) { + this.logger.error('--models is required, e.g. --models=llama3.2:latest,llama3:8b') + this.exitCode = 1 + return + } + const models = this.models.split(',').map((m) => m.trim()).filter(Boolean) + + let goldens = await corpusService.loadGoldens() + if (this.tag) goldens = goldens.filter((g) => g.tags.includes(this.tag)) + if (this.limit) goldens = goldens.slice(0, Number.parseInt(this.limit, 10)) + + const fingerprint = await corpusService.fingerprint() + const repeats = this.repeats ? Number.parseInt(this.repeats, 10) : 1 + this.logger.info( + `Corpus ${fingerprint} · ${goldens.length} goldens · ${models.length} models · repeats=${repeats}` + ) + this.logger.info('Running oracle and e2e for each model. This takes a while.') + this.logger.info('') + + const rows: Row[] = [] + for (const model of models) { + this.logger.info(`--- ${model} ---`) + const row: Row = { model, oracle: null, e2e: null } + try { + // Only oracle and e2e: those two are what the triage rule needs, and + // adding noretrieval would half again the runtime of an already long + // command for a number that does not change the verdict. + for (const mode of ['oracle', 'e2e'] as const) { + const started = Date.now() + const result = await generationService.run(goldens, { mode, model, repeats }) + row[mode] = result.overall + this.logger.info( + ` ${mode.padEnd(6)} correctness=${fmt(result.overall.correctness)} refusal=${fmt(result.overall.refusalCorrectness)} (${((Date.now() - started) / 1000).toFixed(0)}s)` + ) + if (result.overall.errors > 0) { + this.logger.warning(` ${result.overall.errors} question(s) errored in ${mode}`) + } + } + } catch (error) { + row.error = error instanceof Error ? error.message : String(error) + this.logger.error(` failed: ${row.error}`) + } + rows.push(row) + } + + this.logger.info('') + this.printTable(rows) + + if (this.promote) { + const path = resolve(join(process.cwd(), 'tests/eval/baselines', fingerprint, 'matrix.json')) + await mkdir(dirname(path), { recursive: true }) + await writeFile( + path, + JSON.stringify( + { + corpusFingerprint: fingerprint, + createdAt: new Date().toISOString(), + goldens: goldens.length, + repeats, + tag: this.tag ?? null, + rows, + }, + null, + 2 + ) + ) + this.logger.success(`Capability table written: ${path}`) + this.logger.info('Commit it — this is the reference a support triage compares against.') + } + } catch (error) { + this.logger.error(error instanceof Error ? error.message : String(error)) + this.exitCode = 1 + } finally { + restoreLogging() + } + } + + private printTable(rows: Row[]) { + this.logger.info('=== Capability matrix ===') + this.logger.info('') + const width = Math.max(20, ...rows.map((r) => r.model.length + 2)) + this.logger.info( + `${'model'.padEnd(width)}${'oracle'.padStart(8)}${'e2e'.padStart(8)}${'ret.cost'.padStart(10)}${'refusal'.padStart(9)}${'leakage'.padStart(9)}${'ground'.padStart(8)}` + ) + for (const row of rows) { + if (row.error) { + this.logger.info(`${row.model.padEnd(width)} ERROR: ${row.error}`) + continue + } + const cost = + row.oracle?.correctness !== null && row.oracle?.correctness !== undefined && + row.e2e?.correctness !== null && row.e2e?.correctness !== undefined + ? row.oracle.correctness - row.e2e.correctness + : null + this.logger.info( + row.model.padEnd(width) + + fmt(row.oracle?.correctness ?? null).padStart(8) + + fmt(row.e2e?.correctness ?? null).padStart(8) + + (cost === null ? ' n/a' : `${cost >= 0 ? '+' : ''}${cost.toFixed(3)}`).padStart(10) + + fmt(row.e2e?.refusalCorrectness ?? null).padStart(9) + + fmt(row.e2e?.leakageRate ?? null).padStart(9) + + fmt(row.e2e?.groundedness?.mean ?? null).padStart(8) + ) + } + this.logger.info('') + this.logger.info('oracle = correctness with perfect context (the model\'s ceiling)') + this.logger.info('e2e = correctness with real retrieval (what a user gets)') + this.logger.info('ret.cost = oracle - e2e (what imperfect retrieval costs this model)') + this.logger.info('leakage = rate of narrating retrieval; lower is better') + } +} + +const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3)) diff --git a/admin/commands/eval/retrieval.ts b/admin/commands/eval/retrieval.ts new file mode 100644 index 0000000..6ca6fb5 --- /dev/null +++ b/admin/commands/eval/retrieval.ts @@ -0,0 +1,276 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import type { RetrievalAggregate } from '../../app/utils/eval/retrieval_metrics.js' + +/** + * Score NOMAD's retrieval against the frozen golden set. + * + * No chat model is involved, so this is deterministic and hardware-independent: + * a movement in these numbers is a code change, not a slow machine or an unlucky + * sample. It is the fast inner loop for anything touching chunking, embedding, + * thresholds, or reranking. + * + * node ace eval:retrieval + * node ace eval:retrieval --ablate # is the reranker helping? + * node ace eval:retrieval --threshold=0.5 # sweep the cutoff + * node ace eval:retrieval --tag=multi-hop # one slice only + */ +export default class EvalRetrieval extends BaseCommand { + static commandName = 'eval:retrieval' + static description = 'Score RAG retrieval against the golden set (deterministic, no chat model)' + + @flags.string({ description: 'Chunks to retrieve per query (default: the production value)' }) + declare topK: string + + @flags.string({ description: 'Minimum similarity score (default: the production value)' }) + declare threshold: string + + @flags.boolean({ description: 'Also score the raw dense, reranked, and diversified orderings' }) + declare ablate: boolean + + @flags.string({ description: 'Only run goldens carrying this tag' }) + declare tag: string + + @flags.boolean({ description: 'Print each failing question and what it retrieved' }) + declare verbose: boolean + + @flags.boolean({ description: 'Leave application debug logging on (very noisy)' }) + declare debug: boolean + + @flags.boolean({ description: 'Write a JSON + Markdown report to tests/eval/reports/' }) + declare report: boolean + + static options: CommandOptions = { + startApp: true, + } + + async run() { + const { EvalCorpusService } = await import('#services/eval_corpus_service') + const { EvalRetrievalService } = await import('#services/eval_retrieval_service') + const { EvalReportService } = await import('#services/eval_report_service') + const { quietLogging } = await import('../../app/utils/eval/quiet.js') + + const corpusService = await this.app.container.make(EvalCorpusService) + const retrievalService = await this.app.container.make(EvalRetrievalService) + const reportService = await this.app.container.make(EvalReportService) + const restoreLogging = quietLogging(this.debug) + + try { + const chunks = await retrievalService.assertCorpusReady() + const fingerprint = await corpusService.fingerprint() + let goldens = await corpusService.loadGoldens() + + if (this.tag) { + goldens = goldens.filter((g) => g.tags.includes(this.tag)) + if (goldens.length === 0) { + this.logger.error(`No goldens carry the tag "${this.tag}"`) + this.exitCode = 1 + return + } + } + + this.logger.info(`Corpus ${fingerprint} · ${chunks} chunks · ${goldens.length} goldens`) + this.logger.info('') + + const started = Date.now() + const result = await retrievalService.run(goldens, { + topK: this.topK ? Number.parseInt(this.topK, 10) : undefined, + scoreThreshold: this.threshold ? Number.parseFloat(this.threshold) : undefined, + ablate: this.ablate, + }) + const elapsed = ((Date.now() - started) / 1000).toFixed(1) + + this.logger.info( + `Params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} (${elapsed}s)` + ) + this.logger.info('') + this.printAggregate('OVERALL', result.overall, result.params.kValues) + + if (result.unresolvedChunks > 0) { + // Every retrieved chunk should belong to the eval corpus. Anything else + // means the collection filter leaked and the numbers above describe a + // corpus nobody chose. + this.logger.error( + `${result.unresolvedChunks} retrieved chunk(s) did not belong to the eval corpus — the collection filter leaked.` + ) + this.exitCode = 1 + } + + this.printThresholdGuidance(result.overall) + + if (result.ablation) { + this.logger.info('') + this.logger.info('=== Stage ablation (does each heuristic earn its place?) ===') + const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0] + const row = (name: string, agg: RetrievalAggregate) => + this.logger.info( + ` ${name.padEnd(14)} recall@${k}=${fmt(agg.recall[k])} ndcg@${k}=${fmt(agg.ndcg[k])} mrr=${fmt(agg.mrr)} prec@${k}=${fmt(agg.precision[k])}` + ) + row('dense only', result.ablation.dense) + row('+ rerank', result.ablation.reranked) + row('+ diversity', result.ablation.diversified) + this.explainAblation(result.ablation, k) + } + + this.logger.info('') + this.logger.info('=== By tag ===') + for (const [tag, agg] of Object.entries(result.byTag).sort()) { + const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0] + this.logger.info( + ` ${tag.padEnd(20)} n=${String(agg.cases).padStart(3)} recall@${k}=${fmt(agg.recall[k])} ndcg@${k}=${fmt(agg.ndcg[k])}` + ) + } + + const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0] + const misses = result.cases.filter((c) => !c.expectRefusal && (c.recall[k] ?? 1) < 1) + this.logger.info('') + this.logger.info(`${misses.length} of ${result.overall.answerable} answerable questions missed at k=${k}`) + + if (this.verbose && misses.length > 0) { + this.logger.info('') + this.logger.info('=== Misses ===') + for (const miss of misses) { + this.logger.info(` ${miss.id}`) + this.logger.info(` wanted: ${miss.relevantDocIds.join(', ') || '(none)'}`) + this.logger.info(` retrieved: ${miss.retrievedDocIds.join(', ') || '(nothing)'}`) + } + } else if (misses.length > 0) { + this.logger.info('Re-run with --verbose to see which questions and what they retrieved.') + } + + if (this.report) { + const meta = await reportService.buildMeta('retrieval', fingerprint, { + ...result.params, + tag: this.tag ?? null, + }) + const doc = reportService.fromRetrieval(meta, result) + const slug = `retrieval-${meta.createdAt.replace(/[:.]/g, '-')}` + const path = await reportService.write(doc, slug, renderRetrievalMarkdown(doc, result, misses)) + this.logger.info('') + this.logger.success(`Report written: ${path}`) + } + } catch (error) { + this.logger.error(error instanceof Error ? error.message : String(error)) + this.exitCode = 1 + } finally { + restoreLogging() + } + } + + private printAggregate(label: string, agg: RetrievalAggregate, kValues: number[]) { + this.logger.info(`=== ${label} (${agg.answerable} answerable of ${agg.cases}) ===`) + const row = (name: string, values: Record) => + this.logger.info( + ` ${name.padEnd(10)} ${kValues.map((k) => `@${k}=${fmt(values[k])}`).join(' ')}` + ) + row('recall', agg.recall) + row('hit rate', agg.hitRate) + row('precision', agg.precision) + row('ndcg', agg.ndcg) + this.logger.info(` mrr ${fmt(agg.mrr)}`) + } + + /** + * Turn the score distributions into an actual recommendation. The raw + * percentiles are the evidence; this is the reading of them, which is what + * the threshold constants have never had. + */ + private printThresholdGuidance(agg: RetrievalAggregate) { + const rel = agg.relevantScores + const irr = agg.irrelevantScores + this.logger.info('') + this.logger.info('=== Score separation (how to calibrate the threshold) ===') + if (rel) { + this.logger.info( + ` relevant chunks n=${rel.count} min=${rel.min.toFixed(3)} p10=${rel.p10.toFixed(3)} median=${rel.median.toFixed(3)} p90=${rel.p90.toFixed(3)}` + ) + } + if (irr) { + this.logger.info( + ` irrelevant chunks n=${irr.count} min=${irr.min.toFixed(3)} p10=${irr.p10.toFixed(3)} median=${irr.median.toFixed(3)} p90=${irr.p90.toFixed(3)}` + ) + } + if (rel && irr) { + if (rel.p10 > irr.p90) { + this.logger.success( + ` Clean separation: a threshold between ${irr.p90.toFixed(3)} and ${rel.p10.toFixed(3)} splits them.` + ) + } else { + this.logger.warning( + ` Overlapping: relevant p10 (${rel.p10.toFixed(3)}) sits below irrelevant p90 (${irr.p90.toFixed(3)}).` + ) + this.logger.warning( + ' No cutoff separates these cleanly — the retriever needs work, not the threshold.' + ) + } + } + if (agg.emptyRateOnAnswerable !== null && agg.emptyRateOnAnswerable > 0) { + this.logger.warning( + ` ${pct(agg.emptyRateOnAnswerable)} of answerable questions retrieved nothing — threshold may be too high.` + ) + } + if (agg.nonEmptyRateOnRefusal !== null && agg.nonEmptyRateOnRefusal > 0) { + this.logger.warning( + ` ${pct(agg.nonEmptyRateOnRefusal)} of out-of-corpus questions retrieved something anyway — that context is what produces confident wrong answers.` + ) + } + } + + private explainAblation(ablation: { dense: RetrievalAggregate; reranked: RetrievalAggregate; diversified: RetrievalAggregate }, k: number) { + const verdict = (name: string, before: number | null, after: number | null) => { + if (before === null || after === null) return + const delta = after - before + if (Math.abs(delta) < 1e-6) { + this.logger.warning(` ${name} changed nothing at k=${k} — it is complexity with no measured benefit.`) + } else if (delta < 0) { + this.logger.warning(` ${name} made ndcg@${k} worse by ${Math.abs(delta).toFixed(4)}.`) + } else { + this.logger.success(` ${name} improved ndcg@${k} by ${delta.toFixed(4)}.`) + } + } + verdict('Reranking', ablation.dense.ndcg[k], ablation.reranked.ndcg[k]) + verdict('Source diversity', ablation.reranked.ndcg[k], ablation.diversified.ndcg[k]) + } +} + +const fmt = (v: number | null) => (v === null ? ' n/a' : v.toFixed(3)) +const pct = (v: number) => `${(v * 100).toFixed(0)}%` + +/** + * The human-readable half of a report. Leads with the misses, because when a + * number moves the next question is always "which questions?" and a table of + * aggregates cannot answer it. + */ +function renderRetrievalMarkdown(doc: any, result: any, misses: any[]): string { + const lines: string[] = ['# Retrieval eval', ''] + lines.push(`- corpus: \`${doc.meta.corpusFingerprint}\``) + lines.push(`- commit: \`${doc.meta.gitSha ?? 'unknown'}\`${doc.meta.gitDirty ? ' (dirty tree)' : ''}`) + lines.push(`- when: ${doc.meta.createdAt}`) + lines.push(`- params: topK=${result.params.topK} threshold=${result.params.scoreThreshold}`) + lines.push('') + lines.push('## Metrics', '') + lines.push('| metric | value |', '|---|---:|') + for (const [name, value] of Object.entries(doc.metrics)) { + lines.push(`| ${name} | ${value === null ? 'n/a' : (value as number).toFixed(4)} |`) + } + lines.push('') + lines.push(`## Misses (${misses.length})`, '') + if (misses.length === 0) { + lines.push('None.') + } else { + for (const m of misses) { + lines.push(`### \`${m.id}\``) + lines.push(`- wanted: ${m.relevantDocIds.join(', ') || '_none_'}`) + lines.push(`- retrieved: ${m.retrievedDocIds.join(', ') || '_nothing_'}`) + lines.push('') + } + } + lines.push('## By tag', '') + lines.push('| tag | recall@5 | ndcg@5 |', '|---|---:|---:|') + for (const [tag, metrics] of Object.entries(doc.byTag).sort()) { + const m = metrics as Record + const cell = (v: number | null | undefined) => (v === null || v === undefined ? 'n/a' : v.toFixed(4)) + lines.push(`| ${tag} | ${cell(m['recall@5'])} | ${cell(m['ndcg@5'])} |`) + } + return lines.join('\n') + '\n' +} diff --git a/admin/constants/kb_collections.ts b/admin/constants/kb_collections.ts index a0b3a1e..2b0b46f 100644 --- a/admin/constants/kb_collections.ts +++ b/admin/constants/kb_collections.ts @@ -21,6 +21,21 @@ export const KB_COLLECTIONS = [ export type KbCollection = (typeof KB_COLLECTIONS)[number] +/** + * Reserved collection tag for the developer evaluation corpus (`ace eval:*`). + * + * The eval fixtures live in the same Qdrant collection as everything else — + * NOMAD "collections" are a payload tag, not separate Qdrant collections — so + * this tag is what keeps them out of the user's Knowledge Base UI and out of + * ordinary chat retrieval. Every read path that enumerates user content filters + * it out, and `sanitizeCollectionName` refuses to mint it, so a user cannot + * create a colliding tag by accident. + * + * The leading/trailing underscores are deliberate: they make the tag obviously + * internal if it ever does surface in a log or a raw Qdrant query. + */ +export const KB_EVAL_COLLECTION = '__nomad_eval__' + /** Hard cap on a user-created tag's length, enforced client- and server-side. */ export const KB_COLLECTION_NAME_MAX_LENGTH = 40 @@ -36,5 +51,9 @@ export function sanitizeCollectionName(raw: string | null | undefined): string | if (!raw) return null const trimmed = raw.trim().toLowerCase() if (!trimmed) return null + // The eval tag is reserved. Treating a collision as "uncategorized" rather + // than throwing keeps this a pure normalizer, and the user's documents stay + // visible in the KB instead of vanishing into a hidden internal collection. + if (trimmed === KB_EVAL_COLLECTION) return null return trimmed.slice(0, KB_COLLECTION_NAME_MAX_LENGTH) } diff --git a/admin/constants/ollama.ts b/admin/constants/ollama.ts index 6431f63..d004a50 100644 --- a/admin/constants/ollama.ts +++ b/admin/constants/ollama.ts @@ -76,6 +76,15 @@ export const RAG_CONTEXT_LIMITS: { maxParams: number; maxResults: number; maxTok { maxParams: Infinity, maxResults: 5, maxTokens: 0 }, // 13B+ (no cap) ] +/** + * Retrieval defaults for the chat pipeline. These were previously inline + * literals at the `searchSimilarDocuments` call site in OllamaController, which + * made them impossible to sweep or record in a report. They are named here so + * the pipeline and the eval harness read the same numbers. + */ +export const RAG_DEFAULT_TOP_K = 5 +export const RAG_DEFAULT_SCORE_THRESHOLD = 0.3 + export const SYSTEM_PROMPTS = { default: ` Format all responses using markdown for better readability. Vanilla markdown or GitHub-flavored markdown is preferred. diff --git a/admin/package.json b/admin/package.json index 333eaea..3be46cb 100644 --- a/admin/package.json +++ b/admin/package.json @@ -10,6 +10,8 @@ "build": "node ace build", "dev": "node ace serve --hmr", "test": "node ace test", + "test:unit": "node --import ts-node-maintained/register/esm --test \"tests/unit/**/*.spec.ts\"", + "test:eval": "node --import ts-node-maintained/register/esm --test \"tests/unit/eval_*.spec.ts\" \"tests/unit/rag_pipeline_*.spec.ts\"", "lint": "eslint .", "format": "prettier --write .", "gen:curated-data": "node --experimental-strip-types scripts/generate_curated_data.ts && prettier --write app/data/conditions.ts app/data/natural_remedies.ts app/data/home_remedies.ts", diff --git a/admin/tests/eval/README.md b/admin/tests/eval/README.md new file mode 100644 index 0000000..1503181 --- /dev/null +++ b/admin/tests/eval/README.md @@ -0,0 +1,352 @@ +# NOMAD AI Quality Harness + +A reproducible way to measure whether NOMAD's RAG pipeline is getting better or +worse, and to tell a **code regression** apart from **a small model on modest +hardware being asked too much**. + +This is a developer tool. There is no UI, nothing is user-facing, and none of it +runs in production. + +> **Throughput is out of scope.** tokens/sec, time-to-first-token, and the NOMAD +> Score belong to `node ace benchmark:run`. Nothing here is comparable across +> machines and nothing here should ever be, because every developer and every +> user has different hardware. This measures *quality* only. + +--- + +## The one-minute version + +```bash +node ace eval:corpus --ingest # once, and after any corpus edit +node ace eval:retrieval --ablate # seconds, no chat model, deterministic +node ace eval:generation --model= --all-modes # minutes; answers "code or model?" +``` + +--- + +## Why the three modes matter + +`eval:generation --all-modes` runs every question three ways. That is what turns +"the AI gave a bad answer" into something you can act on: + +| Mode | Context the model gets | What a low score means | +|---|---|---| +| `oracle` | The golden's own documents, injected verbatim | **The model.** Retrieval was perfect by construction, so this is the model's ceiling with this prompt. | +| `e2e` | Whatever real retrieval found | The actual product experience. | +| `noretrieval` | Nothing | The model's parametric baseline — what it knows without NOMAD. | + +Read the decomposition it prints: + +- **`oracle` is low** → the model cannot use good context. No amount of retrieval + work will fix it. The honest answer to the user is "run a larger model." +- **`oracle - e2e` is large** → the model *can* use good context but is not being + given it. This is a retrieval bug, and it is ours. +- **`e2e - noretrieval` is near zero** → RAG is not contributing. Check that + retrieval is actually reaching the prompt. + +There is a fourth reference line, `--model=mock`, which needs no Ollama at all. +It answers by echoing the injected context, so it is the **extractive ceiling**: +the best a perfect model could do given the current retrieval. A real model +below the mock line is the bottleneck; a mock line that is itself low means +retrieval is. + +--- + +## Commands + +### `eval:corpus` + +```bash +node ace eval:corpus --check # validate corpus + goldens, no services needed +node ace eval:corpus --status # fingerprint and current chunk count +node ace eval:corpus --ingest # wipe and rebuild (always a full rebuild) +node ace eval:corpus --reset # remove every eval chunk +``` + +Ingest goes through NOMAD's **real** `RagService.embedAndStoreText`, so chunk +size, the token-estimate ratio, the `search_document:` prefix, and the embedding +model are all inside the measurement. Change any of them and the score moves — +which is the point. + +### Isolation: how the eval corpus stays out of your knowledge base + +**The eval corpus is not a separate Qdrant collection.** It lives in the same +`nomad_knowledge_base` collection as your real documents, tagged with the +reserved payload value `collection: __nomad_eval__`. That is not a shortcut — +NOMAD "collections" *are* payload tags rather than separate Qdrant collections, +so using the tag means the harness exercises the same filter path production +chat uses, with no production code changed to accommodate it. + +Three separate mechanisms keep the two apart: + +**1. Writes are scoped.** Ingest only ever adds points carrying the eval tag. +`--reset` deletes by that filter and nothing else. Your documents are never +written, re-tagged, or deleted. (On the machine this was built on: 306 points +before ingest, 335 after — exactly the 29 eval chunks, nothing else moved.) + +**2. Reads are filtered server-side.** Every eval query passes +`must: [{ key: 'collection', match: { value: '__nomad_eval__' } }]`, and Qdrant +applies it during search against a keyword payload index, so your documents +never compete for a result slot. Verified directly against a knowledge base +containing unrelated content — the same query returns a real user document at +similarity **0.83** unfiltered, and only eval documents at ~0.50 filtered: + +``` +unfiltered: 0.831 evolution_of_steam_locomotive.txt +filtered: 0.522 water-river-song.md [eval] +``` + +**3. A leak would fail the run, loudly.** Every retrieved chunk's `source` is +resolved back to a corpus document by path. Anything outside +`tests/eval/corpus/` is counted as an unresolved chunk, and a non-zero count +prints an error and exits 1 rather than quietly reporting a score. The check is +on the resolved path, not the file extension — NOMAD embeds its own +`admin/docs/*.md` into the knowledge base on first run, so an extension check +would have accepted a leaked `faq.md` as the plausible document id "faq". +See `tests/unit/eval_source_guard.spec.ts`. + +**The one thing the tag does not isolate** is the *physical* collection: your +documents and the eval fixtures share an HNSW index. That has no effect on +correctness (the filter is applied during search) and results were verified +byte-identical across runs, but if you want true physical separation the change +is to thread a collection name through `_ensureCollection`, +`embedAndStoreText`, and `searchSimilarDocuments`. That was deliberately not +done, because it means touching three production methods for a test-only +benefit that the payload filter already delivers. + +### `eval:retrieval` + +```bash +node ace eval:retrieval +node ace eval:retrieval --ablate # is the reranker earning its complexity? +node ace eval:retrieval --threshold=0.5 # sweep the cutoff +node ace eval:retrieval --tag=multi-hop +node ace eval:retrieval --verbose # show every miss and what it retrieved +node ace eval:retrieval --report # write JSON + Markdown to reports/ +``` + +Embedding is the only model call, and its output is stable, so **this tier is +deterministic and hardware-independent**. Two runs produce byte-identical +numbers. A movement here is a code change, full stop — which makes it the only +tier worth gating CI on. + +Multi-turn goldens are scored on their raw final message, because resolving the +coreference needs the chat model and would make the tier non-deterministic. That +bucket therefore reports the honest floor; the rewrite's contribution shows up in +the generation tier instead. + +### `eval:generation` + +```bash +node ace eval:generation --model=mock # no Ollama required +node ace eval:generation --model=llama3.2:latest --all-modes +node ace eval:generation --model=llama3:8b --repeats=5 --verbose +node ace eval:generation --model=... --tag=out-of-corpus +``` + +Runs at `temperature: 0` with a fixed seed. That reduces variance but does not +eliminate it — llama.cpp batching and GPU scheduling still move outputs — so +`--repeats` defaults to 3 and any question whose pass/fail flips across repeats +is reported as **unstable** and excluded from gating. Do not read an unstable +question as a regression. + +Before scoring, the harness evicts other resident models and burns one throwaway +generation, borrowed from `BenchmarkService` for the same reason it added them: +a cold first run behaves differently. + +### `eval:matrix` + +```bash +node ace eval:matrix --models=qwen2.5:0.5b,llama3.2:latest,llama3:8b --limit=25 +node ace eval:matrix --models=... --promote +``` + +Produces the **capability table** — the artifact that answers a support ticket. +When a user reports a bad answer, look up their model: + +- scoring at or near its row → the model is at its ceiling, not a bug +- scoring well below its row → their config or our code, worth investigating + +### `eval:compare` + +```bash +node ace eval:retrieval --report +node ace eval:compare --promote=retrieval # set the baseline +node ace eval:compare tests/eval/baselines//retrieval.json +``` + +Exits non-zero when any metric regresses beyond the tolerance (default `0.02`). +It boots no services, so CI can run it against a committed baseline without +standing up MySQL, Redis, Qdrant, or Ollama. + +**It refuses to compare reports with different corpus fingerprints.** That is +correct behaviour, not a limitation: if the corpus, chunk size, or embedding +model changed, the two runs measured different things and diffing them would +manufacture a regression. Re-baseline instead. + +--- + +## The corpus and the goldens + +- `corpus/*.md` — 28 short documents across NOMAD's real domains. +- `goldens/*.jsonl` — 99 questions, one JSON object per line. + +The corpus is built with deliberate traps, not just easy questions: + +| Tag | What it tests | +|---|---| +| `distractor` | `water-river-song.md` is a poem that shares vocabulary with real water questions and answers none of them. This is exactly the "poetic, tangential passage" failure `SYSTEM_PROMPTS.rag_context` rule 1 defends against. | +| `near-miss` | Water-bath vs pressure canning: two documents that look alike and give opposite advice. | +| `out-of-corpus` | Questions the corpus genuinely cannot answer, including adversarial ones about topics the corpus *partly* covers (the TR-88's warranty). The right answer is to decline. | +| `fictional` | The Thornfield protocol and the TR-88 pump do not exist. No model can know them, so a correct answer proves retrieval worked rather than that the model memorised the internet. | +| `acronym` / `acronym-control` | The same question asked with an acronym and spelled out, to measure what `preprocessQuery`'s 28-entry glossary expansion actually buys. | +| `chunk-boundary` | Facts buried late in the one long document, which is the only one that splits into multiple chunks. | +| `multi-hop` | Answers requiring two documents (elevation table + boiling times). | +| `multi-turn` | A pronoun in the second turn, the only thing that exercises `rewriteQueryWithContext`. | + +### Golden format + +```jsonc +{ + "id": "water-boil-altitude-01", + "query": "How long do I need to boil water at high altitude?", + "turns": [], // prior messages for multi-turn cases + "relevantDocIds": ["water-boiling"], // corpus filenames without .md + "mustInclude": ["\\b(3|three) minutes?"], // case-insensitive REGEX + "mustNotInclude": ["distill"], + "expectRefusal": false, // true for out-of-corpus + "tags": ["single-hop", "numeric"] +} +``` + +`mustInclude` and `mustNotInclude` entries are **regular expressions**, so one +entry can accept "3 minutes" or "three minutes" without inflating the list. +Every pattern is compiled at load, so a bad regex fails immediately rather than +on the one run where it finally matters. + +Validation is strict on purpose. A golden that lists a document not in the +corpus, or that expects a refusal while also naming relevant documents, is +rejected at load — those mistakes are otherwise invisible and just quietly lower +the score forever. + +### Editing the corpus + +Any edit changes the fingerprint, which invalidates every existing baseline. +That is deliberate. After editing: + +```bash +node ace eval:corpus --check # validate first +node ace eval:corpus --ingest # rebuild +node ace eval:retrieval --report +node ace eval:compare --promote=retrieval +``` + +Note the safety facts in the corpus are real. If you add documents, keep any +health, water, or food-safety content accurate — invent only clearly-fictional +non-safety things (place names, equipment model numbers) when you need something +unguessable. + +--- + +## What the metrics mean + +**Retrieval** — measured at two levels, because they answer different questions. +Document level (recall, hit rate, MRR, nDCG) collapses chunks to their source +document: *did the answer's document reach the context?* Chunk level (precision) +does not dedupe: *how much of what we injected is noise?* — five chunks from one +irrelevant document cost a small model five slots. + +`nDCG` normalizes against the *known* number of relevant documents, not against +whatever was retrieved. It is the metric that catches "right documents, wrong +order", a reranking regression that leaves recall untouched while pushing the +answer to position five where a 1B model's 2-result budget will never see it. +The implementation is cross-checked against `pytrec_eval` (TREC's reference +implementation) — see `tests/unit/eval_retrieval_metrics.spec.ts`. + +**Generation** — all deterministic, no judge model required: + +| Metric | What it catches | +|---|---| +| `correctness` | The `mustInclude` / `mustNotInclude` assertions. | +| `refusalCorrectness` | Declining out-of-corpus questions *and* not hedging on answerable ones. This is the "Sorry, I wasn't able to find specific context" symptom, measured. | +| `leakageRate` | Narrating retrieval ("according to Context 1", "the knowledge base"), which `rag_context` rule 4 forbids. Pure regex, zero ambiguity, catches a bad prompt edit on the first run. | +| `groundedness` | Fraction of the answer's numeric claims that appear in the injected context. | +| `thinkTagLeakRate` | Reasoning tags reaching the user. Should always be 0. | + +**Groundedness only sees numbers, and only numbers above 10.** An answer that +fabricates a procedure or a proper noun scores a perfect 1.0. It is a +fabrication *detector*, not a faithfulness guarantee. Numbers are the right +first target for this domain — a wrong bleach dose or canner pressure is a wrong +answer with consequences — but do not read a high score as "the answer is +faithful". Small integers are excluded because "3 layers" and "step 2" appear in +any prose and would swamp the signal. + +--- + +## Known limitations + +Read these before trusting a number. + +1. **The corpus is small, so retrieval recall has little headroom.** 28 + documents produce 29 chunks; retrieving the top 5 means retrieving 17% of the + entire corpus on every query. Real NOMAD knowledge bases hold millions of + chunks. `recall@5` therefore sits near 0.99 and cannot detect a modest + retrieval regression. The metrics that *do* have headroom on this corpus are + `recall@1`, `precision@k`, `nonEmptyRateOnRefusal`, and the score + distributions. To make recall discriminating, add substantially more + distractor documents, or ingest `install/wikipedia_en_100_mini_*.zim` under + the eval tag as a harder tier. + +2. **`oracle` is not guaranteed to beat `e2e`.** Oracle injects the golden's + whole documents; e2e injects up to `maxResults` retrieved chunks, which can + include a genuinely helpful extra document. Treat small inversions as noise + unless they survive `--repeats=5`. + +3. **Temperature 0 is not determinism.** The generation tier still moves between + runs. Always report `--repeats` ≥ 3 before concluding anything, and ignore + questions the harness flags as unstable. + +4. **No LLM judge yet.** Faithfulness beyond numeric grounding, completeness, + and answer relevance are not measured. `autoevals` (MIT, TypeScript, talks to + Ollama through the same `/v1` endpoint `OllamaService` already uses) is the + intended addition, reported in a separate section and never mixed into the + deterministic scores — a weak local judge grading a weak local model is not + something to gate on. + +--- + +## Layout + +``` +tests/eval/ + corpus/ 28 markdown fixtures — the frozen knowledge base + goldens/ 99 questions as JSONL + baselines/ /*.json — COMMITTED; the gate compares against these + reports/ run artifacts — gitignored +``` + +Baselines are filed under their corpus fingerprint so it is structurally +impossible to overwrite one corpus's baseline with a run against another. + +Implementation: + +- `app/utils/eval/*` — pure functions (metrics, golden parsing, report diffing). + No I/O, no models, no services. 153 tests, run with: + + ```bash + npm run test:eval # this harness only — should always be green + npm run test:unit # every tests/unit spec + ``` + + `test:unit` currently reports **6 pre-existing failures** unrelated to this + harness: `drug_interactions`, `drug_ingest_status`, and `drug_labels` are + written against `@japa/runner` rather than `node:test`, and + `app_auto_update`, `content_auto_update`, and `content_auto_update_backoff` + import services that need a booted application. Both groups belong in the Japa + suite (`node ace test`, which needs MySQL and Redis). They fail identically + before and after any change here — use `test:eval` when you want a signal you + can trust. +- `app/services/eval_*_service.ts` — orchestration; these need Qdrant and Ollama. +- `commands/eval/*` — the CLI. +- `app/services/rag_pipeline_service.ts` — the prompt pipeline, shared with the + chat endpoint. The harness measures production code, not a copy of it. diff --git a/admin/tests/eval/baselines/51e642964facf251/retrieval.json b/admin/tests/eval/baselines/51e642964facf251/retrieval.json new file mode 100644 index 0000000..7a95274 --- /dev/null +++ b/admin/tests/eval/baselines/51e642964facf251/retrieval.json @@ -0,0 +1,4732 @@ +{ + "meta": { + "kind": "retrieval", + "createdAt": "2026-08-12T19:11:55.528Z", + "corpusFingerprint": "51e642964facf251", + "gitSha": "0bd1c6f4f9888d577fe232de06ac144bb8337131", + "gitBranch": "dev", + "gitDirty": true, + "nomadVersion": "1.34.0", + "platform": { + "cpuArchitecture": "x64", + "osName": "Linux 7.0.0-28-generic", + "nodeVersion": "v24.15.0" + }, + "params": { + "topK": 5, + "scoreThreshold": 0.3, + "kValues": [ + 1, + 3, + 5, + 10 + ], + "tag": null + } + }, + "metrics": { + "recall@1": 0.9308510638297872, + "recall@3": 0.9875886524822695, + "recall@5": 0.9911347517730497, + "recall@10": 0.9911347517730497, + "hitRate@1": 0.9680851063829787, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 0.9680851063829787, + "precision@3": 0.3652482269503546, + "precision@5": 0.22127659574468053, + "precision@10": 0.22127659574468053, + "ndcg@1": 0.9680851063829787, + "ndcg@3": 0.979689461702366, + "ndcg@5": 0.9818395398566271, + "ndcg@10": 0.9818395398566271, + "mrr": 0.9840425531914894, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": 1 + }, + "byTag": { + "acronym": { + "recall@1": 1, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.3333333333333333, + "precision@5": 0.19999999999999998, + "precision@10": 0.19999999999999998, + "ndcg@1": 1, + "ndcg@3": 1, + "ndcg@5": 1, + "ndcg@10": 1, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "acronym-control": { + "recall@1": 1, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.3333333333333333, + "precision@5": 0.2, + "precision@10": 0.2, + "ndcg@1": 1, + "ndcg@3": 1, + "ndcg@5": 1, + "ndcg@10": 1, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "adversarial": { + "recall@1": null, + "recall@3": null, + "recall@5": null, + "recall@10": null, + "hitRate@1": null, + "hitRate@3": null, + "hitRate@5": null, + "hitRate@10": null, + "precision@1": null, + "precision@3": null, + "precision@5": null, + "precision@10": null, + "ndcg@1": null, + "ndcg@3": null, + "ndcg@5": null, + "ndcg@10": null, + "mrr": null, + "emptyRateOnAnswerable": null, + "nonEmptyRateOnRefusal": 1 + }, + "chunk-boundary": { + "recall@1": 1, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.5833333333333333, + "precision@5": 0.35, + "precision@10": 0.35, + "ndcg@1": 1, + "ndcg@3": 1, + "ndcg@5": 1, + "ndcg@10": 1, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "distractor": { + "recall@1": 0, + "recall@3": 0.3333333333333333, + "recall@5": 0.6666666666666666, + "recall@10": 0.6666666666666666, + "hitRate@1": 0, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 0, + "precision@3": 0.3333333333333333, + "precision@5": 0.4, + "precision@10": 0.4, + "ndcg@1": 0, + "ndcg@3": 0.2960819109658652, + "ndcg@5": 0.49818925746641285, + "ndcg@10": 0.49818925746641285, + "mrr": 0.5, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "distractor-control": { + "recall@1": 1, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.3333333333333333, + "precision@5": 0.2, + "precision@10": 0.2, + "ndcg@1": 1, + "ndcg@3": 1, + "ndcg@5": 1, + "ndcg@10": 1, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "fictional": { + "recall@1": 1, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.3333333333333333, + "precision@5": 0.19999999999999998, + "precision@10": 0.19999999999999998, + "ndcg@1": 1, + "ndcg@3": 1, + "ndcg@5": 1, + "ndcg@10": 1, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "multi-hop": { + "recall@1": 0.5, + "recall@3": 0.9285714285714286, + "recall@5": 0.9285714285714286, + "recall@10": 0.9285714285714286, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.619047619047619, + "precision@5": 0.37142857142857144, + "precision@10": 0.37142857142857144, + "ndcg@1": 1, + "ndcg@3": 0.9332668545590923, + "ndcg@5": 0.9332668545590923, + "ndcg@10": 0.9332668545590923, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "multi-turn": { + "recall@1": 0.375, + "recall@3": 0.875, + "recall@5": 0.875, + "recall@10": 0.875, + "hitRate@1": 0.5, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 0.5, + "precision@3": 0.3333333333333333, + "precision@5": 0.2, + "precision@10": 0.2, + "ndcg@1": 0.5, + "ndcg@3": 0.7187516749770934, + "ndcg@5": 0.7187516749770934, + "ndcg@10": 0.7187516749770934, + "mrr": 0.75, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "near-miss": { + "recall@1": 0.5, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.6666666666666666, + "precision@5": 0.4, + "precision@10": 0.4, + "ndcg@1": 1, + "ndcg@3": 0.9197207891481876, + "ndcg@5": 0.9197207891481876, + "ndcg@10": 0.9197207891481876, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "numeric": { + "recall@1": 0.9607843137254902, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.36601307189542487, + "precision@5": 0.21960784313725482, + "precision@10": 0.21960784313725482, + "ndcg@1": 1, + "ndcg@3": 1, + "ndcg@5": 1, + "ndcg@10": 1, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + }, + "out-of-corpus": { + "recall@1": null, + "recall@3": null, + "recall@5": null, + "recall@10": null, + "hitRate@1": null, + "hitRate@3": null, + "hitRate@5": null, + "hitRate@10": null, + "precision@1": null, + "precision@3": null, + "precision@5": null, + "precision@10": null, + "ndcg@1": null, + "ndcg@3": null, + "ndcg@5": null, + "ndcg@10": null, + "mrr": null, + "emptyRateOnAnswerable": null, + "nonEmptyRateOnRefusal": 1 + }, + "single-hop": { + "recall@1": 1, + "recall@3": 1, + "recall@5": 1, + "recall@10": 1, + "hitRate@1": 1, + "hitRate@3": 1, + "hitRate@5": 1, + "hitRate@10": 1, + "precision@1": 1, + "precision@3": 0.3333333333333334, + "precision@5": 0.1999999999999999, + "precision@10": 0.1999999999999999, + "ndcg@1": 1, + "ndcg@3": 1, + "ndcg@5": 1, + "ndcg@10": 1, + "mrr": 1, + "emptyRateOnAnswerable": 0, + "nonEmptyRateOnRefusal": null + } + }, + "cases": [ + { + "id": "water-boil-time-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-boiling" + ], + "retrievedDocIds": [ + "water-boiling", + "food-canning-waterbath", + "water-chemical-treatment", + "water-storage", + "food-canning-pressure" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-boil-altitude-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-boiling" + ], + "retrievedDocIds": [ + "water-boiling", + "food-canning-waterbath", + "food-canning-pressure", + "water-chemical-treatment", + "regions-elevation-table" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-boil-altitude-threshold-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-boiling" + ], + "retrievedDocIds": [ + "water-boiling", + "food-canning-pressure", + "food-canning-waterbath", + "regions-elevation-table", + "water-chemical-treatment" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-boil-taste-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-boiling" + ], + "retrievedDocIds": [ + "water-boiling", + "food-canning-waterbath", + "water-filtration", + "water-chemical-treatment", + "food-canning-pressure" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-bleach-dose-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-chemical-treatment" + ], + "retrievedDocIds": [ + "water-chemical-treatment", + "water-storage", + "food-canning-waterbath", + "water-boiling", + "water-filtration" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-bleach-cloudy-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-chemical-treatment" + ], + "retrievedDocIds": [ + "water-chemical-treatment", + "water-boiling", + "food-canning-waterbath", + "water-storage", + "sanitation-waste" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-bleach-wait-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-chemical-treatment" + ], + "retrievedDocIds": [ + "water-chemical-treatment", + "water-boiling", + "water-storage", + "food-canning-waterbath", + "sanitation-waste" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-cldioxide-crypto-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-chemical-treatment" + ], + "retrievedDocIds": [ + "water-chemical-treatment", + "water-boiling", + "food-mylar-oxygen", + "food-canning-pressure", + "water-storage" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-filter-pore-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-filtration" + ], + "retrievedDocIds": [ + "water-filtration", + "food-mylar-oxygen", + "water-chemical-treatment", + "food-canning-pressure", + "water-boiling" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-filter-virus-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-filtration" + ], + "retrievedDocIds": [ + "water-filtration", + "water-boiling", + "water-chemical-treatment", + "sanitation-waste", + "water-storage" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-filter-freeze-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-filtration" + ], + "retrievedDocIds": [ + "water-filtration", + "shelter-cold-layering", + "water-storage", + "sanitation-waste", + "water-boiling" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-storage-amount-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-storage" + ], + "retrievedDocIds": [ + "water-storage", + "water-chemical-treatment", + "food-canning-waterbath", + "sanitation-waste", + "power-battery-chemistry" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-storage-rotation-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-storage" + ], + "retrievedDocIds": [ + "water-storage", + "equipment-tr88-pump", + "water-boiling", + "food-canning-waterbath", + "food-storage-rotation" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "water-storage-containers-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-storage" + ], + "retrievedDocIds": [ + "water-storage", + "sanitation-waste", + "food-canning-waterbath", + "food-storage-rotation", + "water-chemical-treatment" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-pressure-temp-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-canning-pressure" + ], + "retrievedDocIds": [ + "food-canning-pressure", + "food-canning-waterbath", + "water-boiling", + "food-dehydrating", + "firstaid-hypothermia" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-pressure-psi-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-canning-pressure" + ], + "retrievedDocIds": [ + "food-canning-pressure", + "food-canning-waterbath", + "regions-elevation-table", + "radio-antenna-basics", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-greenbeans-method-01", + "tags": [ + "near-miss", + "multi-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-canning-pressure", + "food-canning-waterbath" + ], + "retrievedDocIds": [ + "food-canning-waterbath", + "water-boiling", + "food-canning-pressure", + "water-chemical-treatment", + "water-storage" + ], + "expectRefusal": false, + "recall": { + "1": 0.5, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 0.9197207891481876, + "5": 0.9197207891481876, + "10": 0.9197207891481876 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-waterbath-ph-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-canning-waterbath" + ], + "retrievedDocIds": [ + "food-canning-waterbath", + "food-canning-pressure", + "water-boiling", + "water-chemical-treatment", + "water-storage" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-tomato-acid-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-canning-waterbath" + ], + "retrievedDocIds": [ + "food-canning-waterbath", + "food-canning-pressure", + "water-storage", + "water-chemical-treatment", + "food-dehydrating" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-dehydrate-veg-temp-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-dehydrating" + ], + "retrievedDocIds": [ + "food-dehydrating", + "food-canning-pressure", + "food-canning-waterbath", + "water-boiling", + "shelter-cold-layering" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-jerky-temp-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-dehydrating" + ], + "retrievedDocIds": [ + "food-dehydrating", + "food-canning-pressure", + "firstaid-hypothermia", + "shelter-cold-layering", + "food-canning-waterbath" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-conditioning-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-dehydrating" + ], + "retrievedDocIds": [ + "food-dehydrating", + "food-storage-rotation", + "food-canning-pressure", + "food-canning-waterbath", + "water-boiling" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-fifo-01", + "tags": [ + "acronym" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-storage-rotation" + ], + "retrievedDocIds": [ + "food-storage-rotation", + "protocol-thornfield-cache", + "gear-bug-out-bag", + "food-canning-pressure", + "water-storage" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-fifo-spelled-01", + "tags": [ + "acronym-control" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-storage-rotation" + ], + "retrievedDocIds": [ + "food-storage-rotation", + "water-storage", + "food-canning-pressure", + "food-canning-waterbath", + "gear-bug-out-bag" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-brownrice-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-storage-rotation" + ], + "retrievedDocIds": [ + "food-storage-rotation", + "water-storage", + "food-mylar-oxygen", + "food-dehydrating", + "protocol-thornfield-cache" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-wheat-shelflife-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-storage-rotation" + ], + "retrievedDocIds": [ + "food-storage-rotation", + "water-storage", + "food-dehydrating", + "food-mylar-oxygen", + "food-canning-pressure" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-oxyabsorber-5gal-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-mylar-oxygen" + ], + "retrievedDocIds": [ + "food-mylar-oxygen", + "water-storage", + "food-canning-pressure", + "gear-bug-out-bag", + "food-storage-rotation" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-oxyabsorber-1gal-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-mylar-oxygen" + ], + "retrievedDocIds": [ + "food-mylar-oxygen", + "water-storage", + "food-canning-pressure", + "gear-bug-out-bag", + "food-storage-rotation" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-mylar-thickness-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-mylar-oxygen" + ], + "retrievedDocIds": [ + "food-mylar-oxygen", + "food-storage-rotation", + "water-storage", + "food-canning-pressure", + "gear-bug-out-bag" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "food-oxyabsorber-sugar-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-mylar-oxygen" + ], + "retrievedDocIds": [ + "food-mylar-oxygen", + "water-chemical-treatment", + "food-storage-rotation", + "water-storage", + "food-canning-waterbath" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-tq-placement-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-bleeding-control" + ], + "retrievedDocIds": [ + "firstaid-bleeding-control", + "firstaid-ifak-contents", + "sanitation-waste", + "firstaid-burns", + "regions-elevation-table" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-tq-time-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-bleeding-control" + ], + "retrievedDocIds": [ + "firstaid-bleeding-control", + "firstaid-ifak-contents", + "sanitation-waste", + "shelter-cold-layering", + "navigation-map-compass" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-woundpack-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-bleeding-control" + ], + "retrievedDocIds": [ + "firstaid-bleeding-control", + "food-canning-pressure", + "firstaid-burns", + "firstaid-ifak-contents", + "food-mylar-oxygen" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-ifak-acronym-01", + "tags": [ + "acronym" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-ifak-contents" + ], + "retrievedDocIds": [ + "firstaid-ifak-contents", + "gear-bug-out-bag", + "power-emp-protection", + "food-mylar-oxygen", + "gear-edc" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-ifak-spelled-01", + "tags": [ + "acronym-control" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-ifak-contents" + ], + "retrievedDocIds": [ + "firstaid-ifak-contents", + "gear-bug-out-bag", + "firstaid-bleeding-control", + "gear-edc", + "food-mylar-oxygen" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-ifak-carry-01", + "tags": [ + "acronym" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-ifak-contents" + ], + "retrievedDocIds": [ + "firstaid-ifak-contents", + "gear-bug-out-bag", + "gear-edc", + "firstaid-bleeding-control", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-hypothermia-temp-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-hypothermia" + ], + "retrievedDocIds": [ + "firstaid-hypothermia", + "shelter-cold-layering", + "food-dehydrating", + "food-canning-pressure", + "water-boiling" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-hypothermia-shivering-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-hypothermia" + ], + "retrievedDocIds": [ + "firstaid-hypothermia", + "shelter-cold-layering", + "power-emp-protection", + "navigation-map-compass", + "radio-antenna-basics" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-hypothermia-handling-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-hypothermia" + ], + "retrievedDocIds": [ + "firstaid-hypothermia", + "shelter-cold-layering", + "power-emp-protection", + "food-canning-pressure", + "firstaid-bleeding-control" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-burn-cooling-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-burns" + ], + "retrievedDocIds": [ + "firstaid-burns", + "water-boiling", + "shelter-cold-layering", + "water-chemical-treatment", + "food-canning-pressure" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "firstaid-burn-ice-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-burns" + ], + "retrievedDocIds": [ + "firstaid-burns", + "water-boiling", + "water-chemical-treatment", + "shelter-cold-layering", + "firstaid-hypothermia" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-gmrs-license-01", + "tags": [ + "acronym" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-gmrs-frs" + ], + "retrievedDocIds": [ + "radio-gmrs-frs", + "radio-ham-bands", + "radio-antenna-basics", + "navigation-map-compass", + "water-filtration" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-gmrs-term-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-gmrs-frs" + ], + "retrievedDocIds": [ + "radio-gmrs-frs", + "radio-ham-bands", + "radio-antenna-basics", + "equipment-tr88-pump", + "food-storage-rotation" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-frs-power-01", + "tags": [ + "acronym", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-gmrs-frs" + ], + "retrievedDocIds": [ + "radio-gmrs-frs", + "radio-antenna-basics", + "radio-ham-bands", + "equipment-tr88-pump", + "power-solar-sizing" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-gmrs-power-01", + "tags": [ + "acronym", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-gmrs-frs" + ], + "retrievedDocIds": [ + "radio-gmrs-frs", + "radio-antenna-basics", + "radio-ham-bands", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-frs-license-01", + "tags": [ + "acronym-control" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-gmrs-frs" + ], + "retrievedDocIds": [ + "radio-gmrs-frs", + "radio-ham-bands", + "radio-antenna-basics", + "food-storage-rotation" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-tech-questions-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-ham-bands" + ], + "retrievedDocIds": [ + "radio-ham-bands", + "radio-gmrs-frs", + "radio-antenna-basics", + "power-emp-protection", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-extra-questions-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-ham-bands" + ], + "retrievedDocIds": [ + "radio-ham-bands", + "radio-gmrs-frs", + "power-solar-sizing", + "radio-antenna-basics", + "firstaid-ifak-contents" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-morse-01", + "tags": [ + "acronym" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-ham-bands" + ], + "retrievedDocIds": [ + "radio-ham-bands", + "radio-gmrs-frs", + "radio-antenna-basics", + "power-emp-protection", + "navigation-map-compass" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-counterpoise-01", + "tags": [ + "chunk-boundary", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-antenna-basics" + ], + "retrievedDocIds": [ + "radio-antenna-basics", + "power-solar-sizing", + "equipment-tr88-pump", + "navigation-map-compass", + "sanitation-waste" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-swr-threshold-01", + "tags": [ + "chunk-boundary", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-antenna-basics" + ], + "retrievedDocIds": [ + "radio-antenna-basics", + "radio-gmrs-frs", + "firstaid-hypothermia", + "radio-ham-bands" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-swr-meaning-01", + "tags": [ + "chunk-boundary" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-antenna-basics" + ], + "retrievedDocIds": [ + "radio-antenna-basics", + "radio-gmrs-frs", + "radio-ham-bands", + "firstaid-hypothermia" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-wavelength-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-antenna-basics" + ], + "retrievedDocIds": [ + "radio-antenna-basics", + "power-solar-sizing", + "radio-ham-bands", + "navigation-map-compass", + "radio-gmrs-frs" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "radio-height-vs-power-01", + "tags": [ + "chunk-boundary" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "radio-antenna-basics" + ], + "retrievedDocIds": [ + "radio-antenna-basics", + "radio-gmrs-frs", + "radio-ham-bands", + "food-canning-pressure" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-leadacid-dod-01", + "tags": [ + "multi-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-solar-sizing", + "power-battery-chemistry" + ], + "retrievedDocIds": [ + "power-battery-chemistry", + "power-solar-sizing", + "food-canning-pressure", + "food-canning-waterbath", + "power-emp-protection" + ], + "expectRefusal": false, + "recall": { + "1": 0.5, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-lifepo4-dod-01", + "tags": [ + "multi-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-solar-sizing", + "power-battery-chemistry" + ], + "retrievedDocIds": [ + "power-battery-chemistry", + "power-solar-sizing", + "equipment-tr88-pump", + "food-canning-pressure", + "food-mylar-oxygen" + ], + "expectRefusal": false, + "recall": { + "1": 0.5, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-derate-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-solar-sizing" + ], + "retrievedDocIds": [ + "power-solar-sizing", + "radio-antenna-basics", + "food-mylar-oxygen", + "power-battery-chemistry", + "regions-elevation-table" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-mppt-01", + "tags": [ + "acronym", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-solar-sizing" + ], + "retrievedDocIds": [ + "power-solar-sizing", + "power-battery-chemistry", + "power-emp-protection", + "radio-antenna-basics", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-peaksun-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-solar-sizing" + ], + "retrievedDocIds": [ + "power-solar-sizing", + "radio-antenna-basics", + "power-battery-chemistry", + "water-storage", + "firstaid-burns" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-lifepo4-cycles-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-battery-chemistry" + ], + "retrievedDocIds": [ + "power-battery-chemistry", + "power-solar-sizing", + "water-storage", + "food-storage-rotation", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-agm-cycles-01", + "tags": [ + "acronym", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-battery-chemistry" + ], + "retrievedDocIds": [ + "power-battery-chemistry", + "power-solar-sizing", + "equipment-tr88-pump", + "radio-gmrs-frs", + "water-storage" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-lifepo4-cold-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-battery-chemistry" + ], + "retrievedDocIds": [ + "power-battery-chemistry", + "power-solar-sizing", + "food-canning-pressure", + "power-emp-protection", + "food-canning-waterbath" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-usable-capacity-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-battery-chemistry" + ], + "retrievedDocIds": [ + "power-battery-chemistry", + "power-solar-sizing", + "equipment-tr88-pump", + "radio-antenna-basics", + "radio-gmrs-frs" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-emp-acronym-01", + "tags": [ + "acronym" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-emp-protection" + ], + "retrievedDocIds": [ + "power-emp-protection", + "radio-antenna-basics", + "radio-ham-bands", + "radio-gmrs-frs", + "firstaid-hypothermia" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-emp-spelled-01", + "tags": [ + "acronym-control" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-emp-protection" + ], + "retrievedDocIds": [ + "power-emp-protection", + "radio-antenna-basics", + "radio-ham-bands", + "food-mylar-oxygen", + "power-battery-chemistry" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "power-faraday-ground-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "power-emp-protection" + ], + "retrievedDocIds": [ + "power-emp-protection", + "radio-antenna-basics", + "shelter-cold-layering", + "food-storage-rotation", + "food-mylar-oxygen" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "gear-bob-acronym-01", + "tags": [ + "acronym", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "gear-bug-out-bag" + ], + "retrievedDocIds": [ + "gear-bug-out-bag", + "food-mylar-oxygen", + "water-storage", + "power-solar-sizing", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "gear-bob-spelled-01", + "tags": [ + "acronym-control", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "gear-bug-out-bag" + ], + "retrievedDocIds": [ + "gear-bug-out-bag", + "food-mylar-oxygen", + "water-storage", + "power-solar-sizing", + "firstaid-ifak-contents" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "gear-bob-weight-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "gear-bug-out-bag" + ], + "retrievedDocIds": [ + "gear-bug-out-bag", + "food-mylar-oxygen", + "sanitation-waste", + "firstaid-ifak-contents", + "food-canning-pressure" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "gear-edc-acronym-01", + "tags": [ + "acronym" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "gear-edc" + ], + "retrievedDocIds": [ + "gear-edc", + "power-emp-protection", + "gear-bug-out-bag", + "firstaid-ifak-contents", + "radio-ham-bands" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "shelter-cotton-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "shelter-cold-layering" + ], + "retrievedDocIds": [ + "shelter-cold-layering", + "firstaid-hypothermia", + "food-canning-waterbath", + "food-canning-pressure", + "water-chemical-treatment" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "shelter-layers-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "shelter-cold-layering" + ], + "retrievedDocIds": [ + "shelter-cold-layering", + "firstaid-hypothermia", + "power-emp-protection", + "gear-bug-out-bag", + "protocol-thornfield-cache" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "shelter-cold-hands-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "shelter-cold-layering" + ], + "retrievedDocIds": [ + "shelter-cold-layering", + "firstaid-burns", + "firstaid-hypothermia", + "sanitation-waste", + "firstaid-bleeding-control" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "sanitation-latrine-distance-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "sanitation-waste" + ], + "retrievedDocIds": [ + "sanitation-waste", + "water-boiling", + "food-canning-waterbath", + "food-canning-pressure", + "water-storage" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "sanitation-buckets-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "sanitation-waste" + ], + "retrievedDocIds": [ + "sanitation-waste", + "food-mylar-oxygen", + "water-storage", + "water-filtration", + "food-canning-waterbath" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "nav-declination-01", + "tags": [ + "single-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "navigation-map-compass" + ], + "retrievedDocIds": [ + "navigation-map-compass", + "power-emp-protection", + "protocol-thornfield-cache", + "radio-antenna-basics", + "shelter-cold-layering" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "nav-pacecount-01", + "tags": [ + "single-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "navigation-map-compass" + ], + "retrievedDocIds": [ + "navigation-map-compass", + "radio-ham-bands", + "radio-antenna-basics", + "regions-elevation-table", + "radio-gmrs-frs" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-thornfield-interval-01", + "tags": [ + "fictional", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "protocol-thornfield-cache" + ], + "retrievedDocIds": [ + "protocol-thornfield-cache", + "food-storage-rotation", + "regions-elevation-table", + "water-storage", + "navigation-map-compass" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-thornfield-states-01", + "tags": [ + "fictional" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "protocol-thornfield-cache" + ], + "retrievedDocIds": [ + "protocol-thornfield-cache", + "regions-elevation-table", + "sanitation-waste", + "radio-ham-bands", + "power-emp-protection" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-thornfield-marker-01", + "tags": [ + "fictional" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "protocol-thornfield-cache" + ], + "retrievedDocIds": [ + "protocol-thornfield-cache", + "regions-elevation-table", + "sanitation-waste", + "radio-antenna-basics", + "power-emp-protection" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-thornfield-custody-01", + "tags": [ + "fictional" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "protocol-thornfield-cache" + ], + "retrievedDocIds": [ + "protocol-thornfield-cache", + "regions-elevation-table", + "radio-ham-bands", + "radio-antenna-basics", + "sanitation-waste" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-tr88-flow-01", + "tags": [ + "fictional", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "equipment-tr88-pump" + ], + "retrievedDocIds": [ + "equipment-tr88-pump", + "food-mylar-oxygen", + "water-filtration", + "radio-antenna-basics", + "food-canning-pressure" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-tr88-diaphragm-01", + "tags": [ + "fictional", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "equipment-tr88-pump" + ], + "retrievedDocIds": [ + "equipment-tr88-pump", + "power-battery-chemistry", + "radio-gmrs-frs", + "water-storage", + "firstaid-hypothermia" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-tr88-maxtemp-01", + "tags": [ + "fictional", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "equipment-tr88-pump" + ], + "retrievedDocIds": [ + "equipment-tr88-pump", + "food-canning-pressure", + "firstaid-hypothermia", + "shelter-cold-layering", + "food-dehydrating" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "fict-tr88-spares-01", + "tags": [ + "fictional" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "equipment-tr88-pump" + ], + "retrievedDocIds": [ + "equipment-tr88-pump", + "power-battery-chemistry", + "gear-bug-out-bag", + "firstaid-ifak-contents", + "radio-ham-bands" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "multihop-kestrel-boil-01", + "tags": [ + "multi-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "regions-elevation-table", + "water-boiling" + ], + "retrievedDocIds": [ + "water-boiling", + "regions-elevation-table", + "food-canning-waterbath", + "food-canning-pressure", + "water-chemical-treatment" + ], + "expectRefusal": false, + "recall": { + "1": 0.5, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "multihop-ashmere-boil-01", + "tags": [ + "multi-hop", + "numeric" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "regions-elevation-table", + "water-boiling" + ], + "retrievedDocIds": [ + "water-boiling", + "regions-elevation-table", + "food-canning-waterbath", + "food-canning-pressure", + "water-chemical-treatment" + ], + "expectRefusal": false, + "recall": { + "1": 0.5, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "multihop-bellhollow-canning-01", + "tags": [ + "multi-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "regions-elevation-table", + "food-canning-pressure" + ], + "retrievedDocIds": [ + "food-canning-pressure", + "regions-elevation-table", + "firstaid-bleeding-control", + "food-canning-waterbath", + "water-chemical-treatment" + ], + "expectRefusal": false, + "recall": { + "1": 0.5, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.6666666666666666, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "distract-river-drink-01", + "tags": [ + "distractor" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-boiling", + "water-filtration", + "water-chemical-treatment" + ], + "retrievedDocIds": [ + "water-river-song", + "water-boiling", + "water-storage", + "water-filtration", + "sanitation-waste" + ], + "expectRefusal": false, + "recall": { + "1": 0, + "3": 0.3333333333333333, + "5": 0.6666666666666666, + "10": 0.6666666666666666 + }, + "hitRate": { + "1": 0, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 0, + "3": 0.3333333333333333, + "5": 0.4, + "10": 0.4 + }, + "ndcg": { + "1": 0, + "3": 0.2960819109658652, + "5": 0.49818925746641285, + "10": 0.49818925746641285 + }, + "reciprocalRank": 0.5, + "empty": false + }, + { + "id": "distract-river-legit-01", + "tags": [ + "distractor-control" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "water-river-song" + ], + "retrievedDocIds": [ + "water-river-song", + "sanitation-waste", + "shelter-cold-layering", + "water-boiling", + "equipment-tr88-pump" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "ooc-tire-pressure-01", + "tags": [ + "out-of-corpus" + ], + "retrievedCount": 5, + "relevantDocIds": [], + "retrievedDocIds": [ + "food-canning-pressure", + "radio-gmrs-frs", + "food-mylar-oxygen", + "equipment-tr88-pump", + "regions-elevation-table" + ], + "expectRefusal": true, + "recall": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "hitRate": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "precision": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "ndcg": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "reciprocalRank": null, + "empty": false + }, + { + "id": "ooc-bgp-01", + "tags": [ + "out-of-corpus" + ], + "retrievedCount": 5, + "relevantDocIds": [], + "retrievedDocIds": [ + "navigation-map-compass", + "radio-antenna-basics", + "regions-elevation-table", + "radio-gmrs-frs", + "power-solar-sizing" + ], + "expectRefusal": true, + "recall": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "hitRate": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "precision": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "ndcg": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "reciprocalRank": null, + "empty": false + }, + { + "id": "ooc-thornfield-founded-01", + "tags": [ + "out-of-corpus", + "adversarial" + ], + "retrievedCount": 5, + "relevantDocIds": [], + "retrievedDocIds": [ + "protocol-thornfield-cache", + "regions-elevation-table", + "firstaid-ifak-contents", + "equipment-tr88-pump", + "radio-gmrs-frs" + ], + "expectRefusal": true, + "recall": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "hitRate": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "precision": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "ndcg": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "reciprocalRank": null, + "empty": false + }, + { + "id": "ooc-tr88-warranty-01", + "tags": [ + "out-of-corpus", + "adversarial" + ], + "retrievedCount": 5, + "relevantDocIds": [], + "retrievedDocIds": [ + "equipment-tr88-pump", + "water-storage", + "food-storage-rotation", + "radio-gmrs-frs", + "food-mylar-oxygen" + ], + "expectRefusal": true, + "recall": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "hitRate": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "precision": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "ndcg": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "reciprocalRank": null, + "empty": false + }, + { + "id": "ooc-tr92-01", + "tags": [ + "out-of-corpus", + "adversarial" + ], + "retrievedCount": 5, + "relevantDocIds": [], + "retrievedDocIds": [ + "equipment-tr88-pump", + "radio-antenna-basics", + "food-mylar-oxygen", + "food-canning-pressure", + "radio-gmrs-frs" + ], + "expectRefusal": true, + "recall": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "hitRate": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "precision": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "ndcg": { + "1": null, + "3": null, + "5": null, + "10": null + }, + "reciprocalRank": null, + "empty": false + }, + { + "id": "multiturn-canning-temp-01", + "tags": [ + "multi-turn" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "food-canning-pressure" + ], + "retrievedDocIds": [ + "firstaid-hypothermia", + "food-canning-pressure", + "shelter-cold-layering", + "water-boiling", + "food-dehydrating" + ], + "expectRefusal": false, + "recall": { + "1": 0, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 0, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 0, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 0, + "3": 0.6309297535714575, + "5": 0.6309297535714575, + "10": 0.6309297535714575 + }, + "reciprocalRank": 0.5, + "empty": false + }, + { + "id": "multiturn-ifak-carry-01", + "tags": [ + "multi-turn" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "firstaid-ifak-contents" + ], + "retrievedDocIds": [ + "gear-edc", + "firstaid-ifak-contents", + "gear-bug-out-bag", + "radio-antenna-basics", + "radio-ham-bands" + ], + "expectRefusal": false, + "recall": { + "1": 0, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 0, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 0, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 0, + "3": 0.6309297535714575, + "5": 0.6309297535714575, + "10": 0.6309297535714575 + }, + "reciprocalRank": 0.5, + "empty": false + }, + { + "id": "multiturn-tr88-diaphragm-01", + "tags": [ + "multi-turn", + "fictional" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "equipment-tr88-pump" + ], + "retrievedDocIds": [ + "equipment-tr88-pump", + "food-storage-rotation", + "water-storage", + "firstaid-hypothermia", + "food-dehydrating" + ], + "expectRefusal": false, + "recall": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "reciprocalRank": 1, + "empty": false + }, + { + "id": "multiturn-boil-altitude-01", + "tags": [ + "multi-turn", + "multi-hop" + ], + "retrievedCount": 5, + "relevantDocIds": [ + "regions-elevation-table", + "water-boiling" + ], + "retrievedDocIds": [ + "regions-elevation-table", + "radio-antenna-basics", + "gear-bug-out-bag", + "protocol-thornfield-cache", + "radio-gmrs-frs" + ], + "expectRefusal": false, + "recall": { + "1": 0.5, + "3": 0.5, + "5": 0.5, + "10": 0.5 + }, + "hitRate": { + "1": 1, + "3": 1, + "5": 1, + "10": 1 + }, + "precision": { + "1": 1, + "3": 0.3333333333333333, + "5": 0.2, + "10": 0.2 + }, + "ndcg": { + "1": 1, + "3": 0.6131471927654584, + "5": 0.6131471927654584, + "10": 0.6131471927654584 + }, + "reciprocalRank": 1, + "empty": false + } + ] +} \ No newline at end of file diff --git a/admin/tests/eval/corpus/equipment-tr88-pump.md b/admin/tests/eval/corpus/equipment-tr88-pump.md new file mode 100644 index 0000000..dcd5a97 --- /dev/null +++ b/admin/tests/eval/corpus/equipment-tr88-pump.md @@ -0,0 +1,32 @@ +# Model TR-88 Manual Transfer Pump + +The TR-88 is a hand-operated diaphragm pump intended for moving potable water +between storage containers where no power is available. + +## Specifications + +- Rated flow: **4.5 liters per minute** at 60 strokes per minute +- Maximum suction lift: **6.5 meters** +- Wetted materials: polypropylene body, EPDM diaphragm, stainless steel hardware +- Dry weight: 1.9 kg +- Inlet and outlet: 25 mm barbed + +## Maintenance + +Replace the EPDM diaphragm every **500 operating hours** or every three years, +whichever comes first. The diaphragm is the only wear part that will strand the +pump; everything else degrades gracefully. + +Flush with clean water after every use. The TR-88 tolerates silty water but the +check valves will stick if sediment is allowed to dry inside the chambers. + +## Known limitation + +The TR-88 must not be used with any fluid above **40°C**. The EPDM diaphragm +softens above that point and will deform permanently against the seat, after +which the pump will not hold prime. + +## Spares kit + +The TR-88 spares kit, part number **TR88-SK2**, contains two diaphragms, a set of +four check valve umbrellas, and two barb O-rings. diff --git a/admin/tests/eval/corpus/firstaid-bleeding-control.md b/admin/tests/eval/corpus/firstaid-bleeding-control.md new file mode 100644 index 0000000..8ba679f --- /dev/null +++ b/admin/tests/eval/corpus/firstaid-bleeding-control.md @@ -0,0 +1,29 @@ +# Controlling Severe Bleeding + +Uncontrolled bleeding kills faster than almost anything else you can treat in +the field. Minutes matter. + +## Direct pressure first + +Press hard, directly on the wound, with whatever clean material you have. Do not +lift the dressing to check — that breaks the clot you just formed. Add layers on +top instead. + +## Tourniquets + +If direct pressure does not control arterial bleeding from a limb, apply a +tourniquet. Place it **2 to 3 inches above the wound**, never directly over a +joint. Tighten until the bleeding stops, which will hurt. + +**Write the time of application** on the tourniquet or on the patient's +forehead. This single piece of information drives every decision made downstream +of you. + +Do not loosen or remove a tourniquet once it is applied. That is a decision for +someone with surgical capability. + +## Wound packing + +For a deep wound in a junctional area where a tourniquet cannot be placed — the +groin, the armpit, the neck — pack the wound tightly with gauze and hold firm +pressure for a full **3 minutes** before reassessing. diff --git a/admin/tests/eval/corpus/firstaid-burns.md b/admin/tests/eval/corpus/firstaid-burns.md new file mode 100644 index 0000000..02c3539 --- /dev/null +++ b/admin/tests/eval/corpus/firstaid-burns.md @@ -0,0 +1,24 @@ +# Treating Burns + +## Immediate care + +Cool the burn under **cool running water for 20 minutes**. This works for up to +three hours after the injury, so it is still worth doing late. + +Do **not** use ice, do not use butter or oil, and do not break blisters. Ice +causes further tissue damage in an area that has already lost its ability to +regulate. + +Remove rings, watches, and tight clothing near the burn before swelling starts. + +## Covering + +Cover with a clean, non-adherent dressing or cling film laid on loosely — never +wrapped tightly around a limb. + +## When it is beyond field care + +Seek definitive care for any burn that is larger than the patient's palm, any +burn that is white, leathery, or painless (which indicates full thickness), and +any burn to the face, hands, feet, genitals, or across a joint. Burns that +circle a limb entirely are an emergency regardless of size. diff --git a/admin/tests/eval/corpus/firstaid-hypothermia.md b/admin/tests/eval/corpus/firstaid-hypothermia.md new file mode 100644 index 0000000..4b97dbd --- /dev/null +++ b/admin/tests/eval/corpus/firstaid-hypothermia.md @@ -0,0 +1,26 @@ +# Recognizing and Treating Hypothermia + +Hypothermia begins when core body temperature drops below **95°F (35°C)**. + +## Stages + +**Mild** — shivering, clumsiness, slurred speech, and poor judgement. The person +often insists they are fine. Confusion is the tell. + +**Moderate** — shivering stops. This is not improvement; it means the body has +run out of the energy to shiver. Consciousness clouds. + +**Severe** — shivering absent, muscles rigid, pulse and breathing very slow and +hard to detect. + +## Treatment + +Get the person out of the wind and out of wet clothing. Insulate them from the +ground, which steals heat faster than the air does. Apply heat to the torso — +armpits, chest, groin — not to the extremities. + +Handle a severely hypothermic person **gently**. Rough movement can trigger a +fatal cardiac arrhythmia in a cold heart. + +Give warm sweet drinks only if they are fully alert and can swallow. No alcohol: +it feels warming because it dilates surface vessels, which dumps core heat. diff --git a/admin/tests/eval/corpus/firstaid-ifak-contents.md b/admin/tests/eval/corpus/firstaid-ifak-contents.md new file mode 100644 index 0000000..912383d --- /dev/null +++ b/admin/tests/eval/corpus/firstaid-ifak-contents.md @@ -0,0 +1,25 @@ +# Building an IFAK + +An **IFAK** is an Individual First Aid Kit. It is not a boo-boo kit. Its purpose +is to treat life-threatening trauma on one person — you — long enough to reach +real care. + +## Core contents + +- One **CAT or SOF-T tourniquet**, carried where you can reach it one-handed +- Two packages of hemostatic or plain compressed gauze for wound packing +- One pressure bandage +- One vented chest seal, ideally two +- One nasopharyngeal airway with lubricant +- Trauma shears +- Nitrile gloves +- A permanent marker for writing the tourniquet time + +## Carry rules + +Carry the IFAK on your body, not in your pack. A kit in a bag you dropped when +you were injured is a kit you do not have. + +Everything in the kit should be something you have actually practiced with. A +tourniquet you have never applied under stress is a tourniquet you will apply +too loosely. diff --git a/admin/tests/eval/corpus/food-canning-pressure.md b/admin/tests/eval/corpus/food-canning-pressure.md new file mode 100644 index 0000000..636439c --- /dev/null +++ b/admin/tests/eval/corpus/food-canning-pressure.md @@ -0,0 +1,23 @@ +# Pressure Canning Low-Acid Foods + +Low-acid foods — vegetables, meats, poultry, seafood, and most soups — must be +pressure canned. A boiling water bath cannot reach a high enough temperature to +destroy Clostridium botulinum spores. + +## Temperature and pressure + +Pressure canning holds the jars at **240°F (116°C)**. At sea level this +corresponds to **10 psi** on a weighted gauge or **11 psi** on a dial gauge. + +Increase the pressure with altitude. On a weighted gauge, move from 10 psi to +15 psi above 1,000 feet. On a dial gauge, add roughly 1 psi for each additional +2,000 feet of elevation. + +## Do not improvise + +Processing times come from tested recipes and are specific to the food, the jar +size, and the pack style. Do not shorten a time, do not scale a recipe up, and +do not substitute a different vegetable into a tested recipe. + +Have a dial gauge tested for accuracy once a year. A gauge reading 2 psi high +means every batch that season was under-processed. diff --git a/admin/tests/eval/corpus/food-canning-waterbath.md b/admin/tests/eval/corpus/food-canning-waterbath.md new file mode 100644 index 0000000..7942c73 --- /dev/null +++ b/admin/tests/eval/corpus/food-canning-waterbath.md @@ -0,0 +1,23 @@ +# Water Bath Canning + +Water bath canning submerges filled jars in boiling water. The contents reach +212°F (100°C) at sea level and no higher, which is why this method is limited to +a specific class of foods. + +## Only high-acid foods + +Water bath canning is safe **only** for high-acid foods — those with a pH of +**4.6 or below**. That means most fruits, jams and jellies, pickles, and +properly acidified tomatoes. + +Tomatoes sit close to the line. Modern varieties are often above pH 4.6, so +tested recipes call for added bottled lemon juice or citric acid. Use bottled +lemon juice, not fresh: fresh juice varies in acidity. + +## Method + +Cover the jars with at least one inch of water. Start timing when the water +reaches a full boil, and add time for altitude — 5 additional minutes for +1,001 to 3,000 feet, and more above that. + +Never use this method for green beans, corn, meat, or soup. diff --git a/admin/tests/eval/corpus/food-dehydrating.md b/admin/tests/eval/corpus/food-dehydrating.md new file mode 100644 index 0000000..92871f6 --- /dev/null +++ b/admin/tests/eval/corpus/food-dehydrating.md @@ -0,0 +1,20 @@ +# Dehydrating Food + +Dehydration removes moisture so that bacteria, yeasts, and molds cannot grow. +Properly dried and stored food keeps for a year or more. + +## Temperatures + +Dry vegetables at **125°F (52°C)** and fruits at **135°F (57°C)**. Meat for +jerky is different: heat it to an internal **160°F (71°C)** before or after +drying to kill pathogens, then dry at 130-140°F. + +Drying too hot causes case hardening — the outside seals while the inside is +still wet, and the piece spoils from within. + +## Conditioning + +After drying, pack the pieces loosely in a sealed jar for **7 days** and shake it +daily. This is called conditioning, and it equalizes residual moisture between +pieces. If you see any condensation on the glass, the batch needs more drying +time. diff --git a/admin/tests/eval/corpus/food-mylar-oxygen.md b/admin/tests/eval/corpus/food-mylar-oxygen.md new file mode 100644 index 0000000..8036a2c --- /dev/null +++ b/admin/tests/eval/corpus/food-mylar-oxygen.md @@ -0,0 +1,30 @@ +# Mylar Bags and Oxygen Absorbers + +Mylar is a metallized polyester film. It blocks light and, unlike bare plastic +buckets, it is an effective oxygen and moisture barrier. + +## Thickness + +Use bags of at least **5 mil** thickness for long-term storage. Thinner bags +puncture on the sharp edges of dry pasta and rice. + +## Sizing oxygen absorbers + +Oxygen absorbers are rated in cubic centimeters (cc) of oxygen they can take up. +Size them to the bag, not the food: + +- 1 gallon bag: **300cc** +- 5 gallon bag: **2000cc** + +When in doubt, over-size. An absorber that is too large costs a few cents; one +that is too small leaves oxygen behind and the batch degrades. + +Absorbers begin working the moment air hits them. Open only what you will use +within about 15 minutes and keep the rest in a sealed jar. + +## Do not use with + +Do not use oxygen absorbers with sugar or brown sugar — removing the oxygen +turns them into a solid brick. Do not use them for anything with a moisture +content above about 10%; low oxygen plus moisture is the exact condition +botulism spores prefer. diff --git a/admin/tests/eval/corpus/food-storage-rotation.md b/admin/tests/eval/corpus/food-storage-rotation.md new file mode 100644 index 0000000..b68c349 --- /dev/null +++ b/admin/tests/eval/corpus/food-storage-rotation.md @@ -0,0 +1,22 @@ +# Rotating Stored Food + +A store you never rotate is a store that quietly expires. + +## First in, first out + +Run your pantry on **FIFO** — first in, first out. New purchases go to the back; +the oldest stock comes forward and gets eaten first. Date every container with a +marker on the day it enters the pantry, not the day it was manufactured. + +The single most common failure is a deep shelf. If you cannot see the back row, +you will not rotate it. Use shallow shelving, gravity-fed can racks, or bins you +can pull all the way out. + +## Shelf life by category + +Wheat, white rice, rolled oats, and dried beans stored cool and dry in sealed +containers with oxygen absorbers last **20 to 30 years**. Brown rice does not — +its oils go rancid in about 6 months at room temperature. + +Canned goods stay safe indefinitely if the seal holds, but quality declines +after 2 to 5 years. diff --git a/admin/tests/eval/corpus/gear-bug-out-bag.md b/admin/tests/eval/corpus/gear-bug-out-bag.md new file mode 100644 index 0000000..fe58eaa --- /dev/null +++ b/admin/tests/eval/corpus/gear-bug-out-bag.md @@ -0,0 +1,27 @@ +# The Bug Out Bag + +A **BOB** — bug out bag — is a pre-packed kit that sustains one person for +**72 hours** while moving on foot to a predetermined destination. It is not a +wilderness survival kit and it is not everything you own. + +## Weight + +Target **20% of body weight** loaded, and treat 25% as a hard ceiling. Most +first attempts come in far too heavy, because every individual item seems +reasonable in isolation. Weigh the packed bag and walk five miles with it. That +walk will remove more weight than any packing list. + +## Contents by priority + +1. **Water** — one liter carried, plus a filter and chemical backup +2. **Shelter** — tarp or bivy, and a sleeping bag rated for the season +3. **Fire** — ferro rod plus a lighter plus tinder; three methods +4. **Food** — 72 hours of no-cook calories +5. **First aid** — a real trauma kit, not a blister kit +6. **Navigation** — paper map and compass, plus a charged phone +7. **Documents** — copies of ID, deeds, prescriptions, on paper and encrypted + +## The destination + +A bug out bag with no destination is camping gear. Decide in advance where you +are going, who is expecting you, and what the two alternate routes are. diff --git a/admin/tests/eval/corpus/gear-edc.md b/admin/tests/eval/corpus/gear-edc.md new file mode 100644 index 0000000..dd63589 --- /dev/null +++ b/admin/tests/eval/corpus/gear-edc.md @@ -0,0 +1,21 @@ +# Everyday Carry + +**EDC** is every day carry: the small set of items on your person during a normal +day. Its value is that it is *always* present, which no larger kit can claim. + +## A minimal set + +- A light. A single-AAA or rechargeable pocket light. This is the item people + use most and pack last. +- A knife or multitool. +- A means of making fire. +- A phone with offline maps downloaded. +- A tourniquet, if you carry anything medical at all. It is the only item that + reverses a death in the timeframe you can act in. +- Cash in small bills. + +## The test + +The test of an EDC item is whether it is on you right now. An item you leave at +home when the clothes are inconvenient is not EDC — it is a bag item, and it +should be planned for as one. diff --git a/admin/tests/eval/corpus/navigation-map-compass.md b/admin/tests/eval/corpus/navigation-map-compass.md new file mode 100644 index 0000000..d723b94 --- /dev/null +++ b/admin/tests/eval/corpus/navigation-map-compass.md @@ -0,0 +1,31 @@ +# Map and Compass + +## Declination + +True north and magnetic north are not the same, and the angle between them is +**declination**. It varies by location and drifts over time. A map printed with a +declination diagram from 1998 is telling you about 1998. + +Set the declination on the compass housing if it adjusts. If it does not, write +the correction on the compass with a paint pen so you are not doing arithmetic +under stress. + +## Taking a bearing + +Lay the compass edge along your intended line of travel on the map, rotate the +housing until the orienting lines match the map's north-south grid, then lift the +compass and turn your body until the needle sits in the shed. Walk the direction +of travel arrow. + +## Pacing + +Know your pace count — the number of double-steps you take to cover 100 meters. +For most adults it is between **60 and 70** on flat ground, and it lengthens +downhill and shortens uphill and under load. Measure yours on a known distance; +do not assume an average. + +## Handrails and catching features + +Navigate along linear features — a stream, a ridge, a fence line — and pick a +"catching feature" beyond your target that tells you when you have gone too far. +This is more reliable than trying to hit a small point precisely. diff --git a/admin/tests/eval/corpus/power-battery-chemistry.md b/admin/tests/eval/corpus/power-battery-chemistry.md new file mode 100644 index 0000000..bddb0a3 --- /dev/null +++ b/admin/tests/eval/corpus/power-battery-chemistry.md @@ -0,0 +1,31 @@ +# Battery Chemistry for Storage Systems + +## Cycle life + +The number that matters is cycles at a given depth of discharge. + +- **LiFePO4** (lithium iron phosphate): **3,000 to 5,000 cycles** at 80% depth of + discharge +- **AGM** (absorbed glass mat lead-acid): **300 to 500 cycles** at 50% depth of + discharge +- **Flooded lead-acid**: 500 to 1,200 cycles at 50%, but requires watering and + ventilation + +LiFePO4 costs more per amp-hour up front and far less per usable cycle. + +## Temperature + +LiFePO4 must not be **charged below freezing (32°F / 0°C)**. Charging a cold +lithium cell plates metallic lithium onto the anode, which is permanent and +eventually causes an internal short. Discharging below freezing is fine, just +reduced in capacity. Any decent battery management system enforces this, but +cheap cells often ship without one. + +Lead-acid tolerates cold charging but loses capacity, and a discharged lead-acid +battery can freeze solid and split its case. + +## Usable capacity + +A 100Ah LiFePO4 battery gives you about 80Ah usable. A 100Ah AGM gives you about +50Ah before you start destroying it. Compare on usable capacity, never on +nameplate. diff --git a/admin/tests/eval/corpus/power-emp-protection.md b/admin/tests/eval/corpus/power-emp-protection.md new file mode 100644 index 0000000..efdba4f --- /dev/null +++ b/admin/tests/eval/corpus/power-emp-protection.md @@ -0,0 +1,28 @@ +# EMP and Faraday Cages + +**EMP** stands for electromagnetic pulse. A sufficiently strong pulse induces +currents in conductors, and small unshielded electronics with long attached leads +are the most vulnerable. + +## What a Faraday cage does + +A Faraday cage is a continuous conductive enclosure. Incoming fields induce +currents in the shell, which cancel the field inside. Three things determine +whether it works: + +1. **Continuity.** The conductive layer must be unbroken. A seam that is taped + rather than conductively bonded is a hole. +2. **Insulation.** Contents must not touch the conductive shell. A cardboard box + inside a metal trash can is the classic arrangement. +3. **Closure.** The lid must make contact all the way around. Copper tape along + the rim is a cheap fix. + +A galvanized steel trash can with a tight lid, contents boxed in cardboard, is a +genuinely effective cage. It does not need to be grounded to protect its +contents — grounding matters for personnel safety, not for shielding. + +## What to put in it + +Spares, not primaries: a backup handheld radio, a spare charge controller, an +old laptop with offline references, a spare inverter board. Anything already +plugged into long wiring during the event is likely lost regardless. diff --git a/admin/tests/eval/corpus/power-solar-sizing.md b/admin/tests/eval/corpus/power-solar-sizing.md new file mode 100644 index 0000000..aed758a --- /dev/null +++ b/admin/tests/eval/corpus/power-solar-sizing.md @@ -0,0 +1,28 @@ +# Sizing a Small Solar System + +Work backwards from your loads, never forwards from a panel you happened to buy. + +## Step one: measure the load + +Add up watt-hours per day for everything you intend to run. A device drawing 20 +watts for 6 hours is 120 watt-hours. Measure real devices with a meter; nameplate +ratings are near-useless. + +## Step two: size the battery + +Divide daily watt-hours by battery voltage to get amp-hours. Then divide by the +usable depth of discharge — **50%** for lead-acid, **80%** for LiFePO4. Then +multiply by the number of days of autonomy you want to survive without sun. + +## Step three: size the panels + +Divide daily watt-hours by your location's **peak sun hours** — typically 3 to 5, +not the number of daylight hours. Then add **25%** for charge controller +inefficiency, wiring loss, panel soiling, and the fact that panels never hit +their rated output in the real world. + +## Charge controllers + +An MPPT controller extracts roughly 20-30% more energy than a PWM controller from +the same panel, and the gap widens in cold weather and low light. PWM is only +sensible on very small systems where the controller cost dominates. diff --git a/admin/tests/eval/corpus/protocol-thornfield-cache.md b/admin/tests/eval/corpus/protocol-thornfield-cache.md new file mode 100644 index 0000000..69146b3 --- /dev/null +++ b/admin/tests/eval/corpus/protocol-thornfield-cache.md @@ -0,0 +1,37 @@ +# The Thornfield Cache Rotation Protocol + +The Thornfield protocol is a scheme for rotating buried supply caches so that no +single cache is ever both stale and unmonitored. It was written up by the +Thornfield Valley mutual aid group and has since been adopted by several +neighboring groups. + +## The rule of three + +Every cache site is one of exactly three states: **primed**, **resting**, or +**pulled**. A group maintains three sites and one site is in each state at all +times. + +- A **primed** cache is freshly stocked and sealed. It is checked but never + opened. +- A **resting** cache has been in the ground longest and is next in line. +- A **pulled** cache has been recovered, its contents inspected and consumed or + redistributed, and the site is left empty and marked as burned. + +## The rotation interval + +Sites rotate every **11 months**. The odd interval is deliberate: a 12 month +cycle means every inspection happens in the same season, and the group never +learns how the site behaves in a wet spring or a hard freeze. + +## Marking + +Sites are recorded by a three-word reference and never by coordinates on any +device. The canonical Thornfield marker is a **cairn of five stones** set 12 +paces magnetic north of the cache itself, which looks like nothing to a passer-by +and is unambiguous to someone who knows the count. + +## Who holds the map + +No single person holds the complete site list. The protocol splits it: each of +three custodians knows two of the three sites. Any two custodians together can +reconstruct the full list; any one alone cannot. diff --git a/admin/tests/eval/corpus/radio-antenna-basics.md b/admin/tests/eval/corpus/radio-antenna-basics.md new file mode 100644 index 0000000..6f1e0e5 --- /dev/null +++ b/admin/tests/eval/corpus/radio-antenna-basics.md @@ -0,0 +1,72 @@ +# Antenna Fundamentals for Field Operators + +The antenna matters more than the radio. A 5 watt handheld with a good antenna +at height will outperform a 50 watt mobile feeding a compromised antenna at +ground level, every time. If you have limited money and limited time, spend both +on the antenna system. + +## Why length matters + +An antenna is a resonant structure. It radiates most efficiently when its +physical length bears a specific relationship to the wavelength of the signal it +is carrying. Wavelength in meters is found by dividing 300 by the frequency in +megahertz. At 146 MHz, near the center of the 2 meter band, a full wavelength is +just over two meters. + +Most practical antennas are a fraction of a wavelength. A quarter-wave whip is +the most common compromise on handhelds because it is short enough to carry and +still works acceptably against a ground plane. + +## The rubber duck problem + +The short flexible antenna that ships with a handheld is a helically wound +quarter-wave that has been physically shortened. Shortening an antenna costs +efficiency. A stock rubber duck typically radiates a fraction of the power that +reaches it, with the remainder lost as heat in the loading coil and in ground +losses through your hand and body. + +Replacing it with a full-length whip is usually the single cheapest improvement +available to a handheld operator. A roll-up J-pole hung from a tree branch is +better still, and costs almost nothing to make from twin-lead. + +## Ground planes and counterpoises + +A quarter-wave antenna is only half of the radiating structure. The other half is +the ground plane — the conductive surface the antenna works against. On a vehicle +that is the metal roof. On a handheld it is your hand, your arm, and your body, +which is why holding a radio differently changes your signal report. + +A counterpoise fixes this. It is nothing more than a wire, cut to a quarter +wavelength, attached to the ground side of the antenna connector and allowed to +hang. On a 2 meter handheld a counterpoise is about **19.5 inches** long, and +adding one is frequently worth more than a new radio. + +## Feedline loss + +Coaxial cable loses signal, and the loss increases with frequency and with +length. RG-58 is convenient and lossy. LMR-400 is stiff, expensive, and much +better. On a short handheld setup feedline barely matters; on a 100 foot run to a +rooftop antenna at 440 MHz it can eat most of your transmitted power before it +reaches the antenna. + +Loss works in both directions. A lossy feedline degrades what you hear just as +much as what you send. + +## SWR and what it actually tells you + +Standing wave ratio measures how much power is reflected back down the feedline +instead of being radiated. A perfect match reads 1:1. Most operators aim to stay +below **2:1**, above which many solid-state transmitters begin folding back their +output power to protect the finals. + +A low SWR does not mean a good antenna. A dummy load has a perfect 1:1 SWR and +radiates nothing at all. SWR tells you about the match, not about the radiation. +This confuses people endlessly: they trim an antenna for the lowest possible SWR +and end up with something that matches beautifully and gets out poorly. + +## Height + +Above roughly 50 MHz, propagation is essentially line of sight, extended a little +by atmospheric refraction. Every foot of height buys range, and height beats +power by a wide margin. Ten feet of mast is worth more than doubling your +transmitter output. diff --git a/admin/tests/eval/corpus/radio-gmrs-frs.md b/admin/tests/eval/corpus/radio-gmrs-frs.md new file mode 100644 index 0000000..3dfe515 --- /dev/null +++ b/admin/tests/eval/corpus/radio-gmrs-frs.md @@ -0,0 +1,25 @@ +# GMRS and FRS Radios + +Two license classes cover most short-range handheld radio use in the United +States, and people confuse them constantly. + +## FRS + +**FRS** is the Family Radio Service. It requires **no license**. Power is capped +at **2 watts** on channels 1-7 and 15-22, and at 0.5 watts on channels 8-14. +Antennas are fixed and cannot be replaced. Realistic range is well under a mile +in terrain, despite what the blister pack claims. + +## GMRS + +**GMRS** is the General Mobile Radio Service. It **requires a license**, but +there is **no examination** — you pay a fee and the license is issued. One +license covers your entire immediate family, and it is valid for **10 years**. + +GMRS allows up to **50 watts** on the main channels, removable antennas, and +repeater use, which is what actually extends range beyond line of sight. + +## Sharing channels + +FRS channels 1-7 and 15-22 overlap with GMRS frequencies, so an FRS handheld and +a GMRS handheld can talk to each other on those channels. diff --git a/admin/tests/eval/corpus/radio-ham-bands.md b/admin/tests/eval/corpus/radio-ham-bands.md new file mode 100644 index 0000000..fb88d36 --- /dev/null +++ b/admin/tests/eval/corpus/radio-ham-bands.md @@ -0,0 +1,29 @@ +# Amateur Radio License Classes + +Amateur radio — **ham** radio — requires passing a written examination. There are +three current US license classes, each granting more spectrum. + +## Technician + +The entry class. **35 questions**, and it grants full privileges on all amateur +bands above 30 MHz, which covers the 2 meter and 70 centimeter bands most +handhelds use. Limited HF privileges on 10 meters and small CW segments. + +## General + +**35 questions** beyond Technician. Opens most of the HF spectrum, which is what +makes long-distance and over-the-horizon communication possible. + +## Amateur Extra + +**50 questions**. Grants the remaining HF segments, mostly the lower portions of +each band where the DX contests live. + +There is **no Morse code requirement** for any current US class; that was dropped +in 2007. + +## Emergency use + +Any station may use any means of radiocommunication at its disposal in a genuine +emergency involving immediate safety of life or property. That provision exists +for real emergencies, not for skipping the exam. diff --git a/admin/tests/eval/corpus/regions-elevation-table.md b/admin/tests/eval/corpus/regions-elevation-table.md new file mode 100644 index 0000000..d25287e --- /dev/null +++ b/admin/tests/eval/corpus/regions-elevation-table.md @@ -0,0 +1,19 @@ +# Settlement Elevation Reference + +Elevations for the settlements in the regional mutual aid network, surveyed at +the town marker. Use these when adjusting any procedure that depends on +elevation — canning pressure, boiling time, and engine tuning all change with +altitude. + +| Settlement | Elevation (feet) | Elevation (meters) | +|-----------------|------------------|--------------------| +| Ashmere Landing | 240 | 73 | +| Thornfield | 1,180 | 360 | +| Calder Junction | 3,400 | 1,036 | +| Kestrel Ridge | 8,900 | 2,713 | +| Vantry Flats | 620 | 189 | +| Bellhollow | 5,150 | 1,570 | + +Kestrel Ridge is the highest occupied site in the network and the only one above +the treeline. Bellhollow and Calder Junction sit in the band where altitude +adjustments start to matter but are easy to forget. diff --git a/admin/tests/eval/corpus/sanitation-waste.md b/admin/tests/eval/corpus/sanitation-waste.md new file mode 100644 index 0000000..653d006 --- /dev/null +++ b/admin/tests/eval/corpus/sanitation-waste.md @@ -0,0 +1,25 @@ +# Sanitation When Plumbing Fails + +More people die after a disaster from sanitation failures than from the event +itself. This is the least glamorous preparation and close to the most important. + +## The twin-bucket system + +Use two buckets: one for liquid waste, one for solid. Separating them is the +whole trick — mixed waste is what produces the smell and the pathogen load. + +Line the solid bucket with a heavy bag and cover each use with a scoop of an +absorbent carbon material: sawdust, peat moss, coconut coir, or shredded paper. +Seal and remove the bag when it is roughly two-thirds full. + +## Siting a latrine + +If you must dig, place a latrine at least **200 feet (60 meters)** from any water +source, and downhill from it. Dig it at least 6 inches deep, cover after each +use, and do not site it anywhere water pools after rain. + +## Handwashing + +A tippy-tap — a jug on a string with a foot lever — uses a fraction of the water +of pouring and is the single highest-value sanitation build. Soap and running +water beat hand sanitizer against the organisms that matter here. diff --git a/admin/tests/eval/corpus/shelter-cold-layering.md b/admin/tests/eval/corpus/shelter-cold-layering.md new file mode 100644 index 0000000..36eebdf --- /dev/null +++ b/admin/tests/eval/corpus/shelter-cold-layering.md @@ -0,0 +1,27 @@ +# Layering for Cold Weather + +Layering manages moisture as much as it manages temperature. Sweat is the enemy; +wet insulation is nearly worthless. + +## The three layers + +**Base layer** — wicks moisture off the skin. Merino wool or synthetic. Never +cotton: cotton holds water against the body and conducts heat away roughly 25 +times faster than dry air. + +**Mid layer** — insulates by trapping air. Fleece, down, or synthetic fill. Down +is lighter and more compressible; synthetic keeps insulating when wet. + +**Shell layer** — blocks wind and precipitation. Wind stripping the warm air out +of your insulation is what actually makes you cold on a dry day. + +## Vent before you sweat + +Open zips, remove the hat, or shed the mid layer *before* you start sweating, not +after. Once the base layer is soaked you will not dry it out in the field. + +## Extremities + +Heat is lost fastest from the head, neck, hands, and feet. A hat and a neck +gaiter do more per gram than any other item. If your hands are cold, put on a +hat — the body restricts blood flow to the extremities to defend the core. diff --git a/admin/tests/eval/corpus/water-boiling.md b/admin/tests/eval/corpus/water-boiling.md new file mode 100644 index 0000000..68a6f10 --- /dev/null +++ b/admin/tests/eval/corpus/water-boiling.md @@ -0,0 +1,27 @@ +# Boiling Water to Make It Safe + +Boiling is the most reliable way to kill bacteria, viruses, and protozoa in +questionable water. It does not remove chemical contaminants, heavy metals, or +salt. + +## How long to boil + +Bring the water to a **rolling boil for 1 minute** at normal elevations. At +elevations **above 6,500 feet (2,000 meters)**, boil for **3 minutes** — water +boils at a lower temperature as altitude increases, so it needs longer at the +boil to reach the same level of pathogen kill. + +Let the water cool naturally. Do not add ice to speed cooling; ice made from +untreated water re-contaminates the batch. + +## Improving the taste + +Boiled water tastes flat because boiling drives off dissolved air. Pour it back +and forth between two clean containers a few times to re-aerate it, or add a +small pinch of salt per quart. + +## Before you boil + +If the water is cloudy, let it settle and pour it through a clean cloth or +coffee filter first. Boiling cloudy water still works, but sediment shields +some organisms and makes the result unpleasant to drink. diff --git a/admin/tests/eval/corpus/water-chemical-treatment.md b/admin/tests/eval/corpus/water-chemical-treatment.md new file mode 100644 index 0000000..d103aa2 --- /dev/null +++ b/admin/tests/eval/corpus/water-chemical-treatment.md @@ -0,0 +1,27 @@ +# Chemical Water Treatment + +Chemical disinfection is lighter and faster than boiling when fuel is short. +It is less effective against protozoa such as Cryptosporidium than boiling or +filtration. + +## Unscented household bleach + +Use plain, unscented household bleach containing 5-6% sodium hypochlorite. Add +**8 drops per gallon** of clear water (about 1/8 teaspoon). Double that to 16 +drops per gallon if the water is cloudy or very cold. Stir and let it stand for +**30 minutes** before drinking. The water should have a faint chlorine smell; if +it does not, repeat the dose and wait another 15 minutes. + +Never use scented, color-safe, or additive-bearing bleach. + +## Chlorine dioxide + +Chlorine dioxide tablets are effective against bacteria, viruses, and — unlike +bleach — Cryptosporidium, but they need a long contact time: **4 hours** for +Cryptosporidium at cold temperatures. Follow the package dosing exactly. + +## Iodine + +Iodine works on bacteria and viruses but is unreliable against Cryptosporidium. +It is not suitable for pregnant people, anyone with thyroid conditions, or for +continuous use beyond a few weeks. diff --git a/admin/tests/eval/corpus/water-filtration.md b/admin/tests/eval/corpus/water-filtration.md new file mode 100644 index 0000000..bc1087b --- /dev/null +++ b/admin/tests/eval/corpus/water-filtration.md @@ -0,0 +1,25 @@ +# Water Filters: What They Do and Do Not Remove + +A filter is rated by its pore size, measured in microns. The rating tells you +what physically cannot pass through. + +## Pore sizes + +A filter with an **absolute 0.2 micron** rating removes bacteria and protozoa, +including Giardia and Cryptosporidium. It does **not** remove viruses, which are +far smaller. In regions where viral contamination is a concern, filter first and +then disinfect chemically or by boiling. + +"Nominal" ratings are marketing numbers and allow a percentage of larger +particles through. Look for the word "absolute". + +## What filters never remove + +Filters do not remove dissolved salts, heavy metals, or most chemicals. Activated +carbon elements improve taste and adsorb some organic compounds, but they are not +a substitute for testing when contamination is suspected. + +## Maintenance + +Backflush hollow-fiber filters before storage and never let them freeze — ice +crystals rupture the fibers and the damage is invisible from the outside. diff --git a/admin/tests/eval/corpus/water-river-song.md b/admin/tests/eval/corpus/water-river-song.md new file mode 100644 index 0000000..b695a83 --- /dev/null +++ b/admin/tests/eval/corpus/water-river-song.md @@ -0,0 +1,18 @@ +# The River Song + +*A traditional verse, collected in the highlands.* + + The river runs clean where the tall pines drink, + and the cold water sings on the stone; + I filled my cup at the mountain's brink + and boiled the night down to the bone. + + Three days of rain and the fords ran high, + the crossing lost to the flood — + we watched the clean water hurrying by + and tasted the silt and the mud. + +They sang this at the spring gathering, when the snowmelt came down and the +lower fields flooded. The verse is older than the settlement itself, and nobody +remembers who wrote it. It is about patience, and about the distance between +water you can see and water you can drink. diff --git a/admin/tests/eval/corpus/water-storage.md b/admin/tests/eval/corpus/water-storage.md new file mode 100644 index 0000000..17d3488 --- /dev/null +++ b/admin/tests/eval/corpus/water-storage.md @@ -0,0 +1,22 @@ +# Storing Water + +Store at least **one gallon per person per day**, with a two-week supply as a +practical target for a fixed location. + +## Containers + +Use food-grade containers marked with recycling code 2 (HDPE). Never reuse +containers that held milk or fruit juice — residual sugars and proteins support +bacterial growth no matter how well you rinse. + +Keep containers off bare concrete. Plasticizers migrate and the floor stays cold +enough to cause condensation cycles. + +## Rotation + +Commercially bottled water carries a manufacturer date; rotate it every two +years. Water you bottled yourself from a treated municipal supply should be +rotated every **six months**. + +Store away from sunlight and away from gasoline, pesticides, or solvents — +vapors permeate plastic over time. diff --git a/admin/tests/eval/goldens/core.jsonl b/admin/tests/eval/goldens/core.jsonl new file mode 100644 index 0000000..58c04eb --- /dev/null +++ b/admin/tests/eval/goldens/core.jsonl @@ -0,0 +1,99 @@ +{"id": "water-boil-time-01", "query": "How long should I boil water to make it safe to drink?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["\\b(1|one) minute"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-boil-altitude-01", "query": "How long do I need to boil water at high altitude?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["\\b(3|three) minutes?"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-boil-altitude-threshold-01", "query": "Above what elevation do I need to boil water longer?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["6,?500|2,?000\\s*(m|meter)"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-boil-taste-01", "query": "Why does boiled water taste flat and how do I fix it?", "turns": [], "relevantDocIds": ["water-boiling"], "mustInclude": ["aerat|pour|air"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "water-bleach-dose-01", "query": "How much unscented bleach do I add to a gallon of clear water?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["8 drops"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-bleach-cloudy-01", "query": "How much bleach should I use if the water is cloudy?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["16 drops"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-bleach-wait-01", "query": "After adding bleach to water, how long do I wait before drinking it?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["30 minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-cldioxide-crypto-01", "query": "How long do chlorine dioxide tablets need to work against Cryptosporidium?", "turns": [], "relevantDocIds": ["water-chemical-treatment"], "mustInclude": ["4 hours"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-filter-pore-01", "query": "What filter pore size removes Giardia and Cryptosporidium?", "turns": [], "relevantDocIds": ["water-filtration"], "mustInclude": ["0\\.2"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-filter-virus-01", "query": "Will a 0.2 micron water filter remove viruses?", "turns": [], "relevantDocIds": ["water-filtration"], "mustInclude": ["virus"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "water-filter-freeze-01", "query": "Why must I keep a hollow fiber water filter from freezing?", "turns": [], "relevantDocIds": ["water-filtration"], "mustInclude": ["fiber|fibre|ruptur|crack|damage"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "water-storage-amount-01", "query": "How much drinking water should I store per person per day?", "turns": [], "relevantDocIds": ["water-storage"], "mustInclude": ["\\b(1|one) gallon"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-storage-rotation-01", "query": "How often should I rotate water I bottled myself from the tap?", "turns": [], "relevantDocIds": ["water-storage"], "mustInclude": ["six months|6 months"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "water-storage-containers-01", "query": "Which containers should I never reuse for storing drinking water?", "turns": [], "relevantDocIds": ["water-storage"], "mustInclude": ["milk|juice"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "food-pressure-temp-01", "query": "What temperature does pressure canning reach?", "turns": [], "relevantDocIds": ["food-canning-pressure"], "mustInclude": ["240"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-pressure-psi-01", "query": "What pressure should a weighted gauge canner run at, at sea level?", "turns": [], "relevantDocIds": ["food-canning-pressure"], "mustInclude": ["10 ?psi"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-greenbeans-method-01", "query": "Can I water bath can green beans?", "turns": [], "relevantDocIds": ["food-canning-pressure", "food-canning-waterbath"], "mustInclude": ["pressure"], "mustNotInclude": [], "expectRefusal": false, "tags": ["near-miss", "multi-hop"]} +{"id": "food-waterbath-ph-01", "query": "What pH do foods need to be safe for water bath canning?", "turns": [], "relevantDocIds": ["food-canning-waterbath"], "mustInclude": ["4\\.6"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-tomato-acid-01", "query": "Why do tomato canning recipes call for added lemon juice?", "turns": [], "relevantDocIds": ["food-canning-waterbath"], "mustInclude": ["acid|pH|4\\.6"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "food-dehydrate-veg-temp-01", "query": "What temperature should I dehydrate vegetables at?", "turns": [], "relevantDocIds": ["food-dehydrating"], "mustInclude": ["125"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-jerky-temp-01", "query": "What internal temperature does meat need to reach when making jerky?", "turns": [], "relevantDocIds": ["food-dehydrating"], "mustInclude": ["160"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-conditioning-01", "query": "What is conditioning dried food and how long does it take?", "turns": [], "relevantDocIds": ["food-dehydrating"], "mustInclude": ["(7|seven) days"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-fifo-01", "query": "What does FIFO mean for pantry rotation?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["first in,? first out"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]} +{"id": "food-fifo-spelled-01", "query": "How should I organize my pantry so the oldest food gets eaten first?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["first in,? first out|oldest|front|rotat"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]} +{"id": "food-brownrice-01", "query": "How long does brown rice last in storage?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["6 months|six months"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-wheat-shelflife-01", "query": "How long will wheat and white rice keep in sealed containers?", "turns": [], "relevantDocIds": ["food-storage-rotation"], "mustInclude": ["20|30 years"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-oxyabsorber-5gal-01", "query": "What size oxygen absorber do I need for a 5 gallon mylar bag?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["2,?000"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-oxyabsorber-1gal-01", "query": "What size oxygen absorber goes in a 1 gallon mylar bag?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["300"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-mylar-thickness-01", "query": "How thick should mylar bags be for long term food storage?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["5 ?mil"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "food-oxyabsorber-sugar-01", "query": "Can I use oxygen absorbers with sugar?", "turns": [], "relevantDocIds": ["food-mylar-oxygen"], "mustInclude": ["brick|hard|solid"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "firstaid-tq-placement-01", "query": "Where do I place a tourniquet relative to the wound?", "turns": [], "relevantDocIds": ["firstaid-bleeding-control"], "mustInclude": ["2 ?(to|-|\u2013) ?3 inch|above"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "firstaid-tq-time-01", "query": "What should I write down after applying a tourniquet?", "turns": [], "relevantDocIds": ["firstaid-bleeding-control"], "mustInclude": ["time"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "firstaid-woundpack-01", "query": "How long do I hold pressure after packing a wound?", "turns": [], "relevantDocIds": ["firstaid-bleeding-control"], "mustInclude": ["(3|three) minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "firstaid-ifak-acronym-01", "query": "What does IFAK stand for?", "turns": [], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["individual first aid kit"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]} +{"id": "firstaid-ifak-spelled-01", "query": "What should go in an individual first aid kit?", "turns": [], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["tourniquet"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]} +{"id": "firstaid-ifak-carry-01", "query": "Where should I carry my IFAK?", "turns": [], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["body|person|on you"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]} +{"id": "firstaid-hypothermia-temp-01", "query": "At what core body temperature does hypothermia begin?", "turns": [], "relevantDocIds": ["firstaid-hypothermia"], "mustInclude": ["95|35"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "firstaid-hypothermia-shivering-01", "query": "What does it mean when a cold person stops shivering?", "turns": [], "relevantDocIds": ["firstaid-hypothermia"], "mustInclude": ["moderate|worse|energy|serious|not.{0,15}improv"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "firstaid-hypothermia-handling-01", "query": "Why must a severely hypothermic person be handled gently?", "turns": [], "relevantDocIds": ["firstaid-hypothermia"], "mustInclude": ["arrhythmia|cardiac|heart"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "firstaid-burn-cooling-01", "query": "How long should I cool a burn under running water?", "turns": [], "relevantDocIds": ["firstaid-burns"], "mustInclude": ["20 minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "firstaid-burn-ice-01", "query": "What should I use to cool a burn?", "turns": [], "relevantDocIds": ["firstaid-burns"], "mustInclude": ["cool running water|cool water"], "mustNotInclude": ["\\buse ice\\b|apply ice"], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "radio-gmrs-license-01", "query": "Do I need to pass a test to get a GMRS license?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["no exam|no examination|without.{0,20}exam|not.{0,20}exam"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]} +{"id": "radio-gmrs-term-01", "query": "How long is a GMRS license valid?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["10 years|ten years"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "radio-frs-power-01", "query": "What is the maximum power for an FRS radio?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["2 ?watt"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]} +{"id": "radio-gmrs-power-01", "query": "How much power can a GMRS radio transmit?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["50 ?watt"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]} +{"id": "radio-frs-license-01", "query": "Does the Family Radio Service require a license?", "turns": [], "relevantDocIds": ["radio-gmrs-frs"], "mustInclude": ["no licen|not require|without a licen"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]} +{"id": "radio-tech-questions-01", "query": "How many questions are on the Technician amateur radio exam?", "turns": [], "relevantDocIds": ["radio-ham-bands"], "mustInclude": ["\\b35\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "radio-extra-questions-01", "query": "How many questions are on the Amateur Extra exam?", "turns": [], "relevantDocIds": ["radio-ham-bands"], "mustInclude": ["\\b50\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "radio-morse-01", "query": "Do I still need to learn Morse code to get a ham license?", "turns": [], "relevantDocIds": ["radio-ham-bands"], "mustInclude": ["no longer|not required|dropped|2007|no morse"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]} +{"id": "radio-counterpoise-01", "query": "How long should a counterpoise be for a 2 meter handheld?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["19\\.5"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary", "numeric"]} +{"id": "radio-swr-threshold-01", "query": "What SWR should I stay below on a solid state transmitter?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["2:1|2 ?to ?1"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary", "numeric"]} +{"id": "radio-swr-meaning-01", "query": "Does a low SWR mean I have a good antenna?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["dummy load|not mean|does not|match"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary"]} +{"id": "radio-wavelength-01", "query": "How do I calculate wavelength from frequency?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["300"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "radio-height-vs-power-01", "query": "Is it better to raise my antenna or increase transmit power?", "turns": [], "relevantDocIds": ["radio-antenna-basics"], "mustInclude": ["height"], "mustNotInclude": [], "expectRefusal": false, "tags": ["chunk-boundary"]} +{"id": "power-leadacid-dod-01", "query": "What depth of discharge should I design for with lead acid batteries?", "turns": [], "relevantDocIds": ["power-solar-sizing", "power-battery-chemistry"], "mustInclude": ["50 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]} +{"id": "power-lifepo4-dod-01", "query": "How deeply can I discharge a LiFePO4 battery?", "turns": [], "relevantDocIds": ["power-solar-sizing", "power-battery-chemistry"], "mustInclude": ["80 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]} +{"id": "power-derate-01", "query": "How much margin should I add when sizing solar panels?", "turns": [], "relevantDocIds": ["power-solar-sizing"], "mustInclude": ["25 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "power-mppt-01", "query": "How much better is an MPPT charge controller than PWM?", "turns": [], "relevantDocIds": ["power-solar-sizing"], "mustInclude": ["20|30"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]} +{"id": "power-peaksun-01", "query": "What are peak sun hours and what value should I use?", "turns": [], "relevantDocIds": ["power-solar-sizing"], "mustInclude": ["3|4|5"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "power-lifepo4-cycles-01", "query": "How many cycles does a LiFePO4 battery last?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["3,?000|5,?000"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "power-agm-cycles-01", "query": "How many cycles will an AGM battery give me?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["300|500"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]} +{"id": "power-lifepo4-cold-01", "query": "Can I charge a lithium iron phosphate battery below freezing?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["32|0 ?\u00b0?C|freezing"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "power-usable-capacity-01", "query": "How many usable amp hours do I get from a 100Ah LiFePO4 battery?", "turns": [], "relevantDocIds": ["power-battery-chemistry"], "mustInclude": ["\\b80\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "power-emp-acronym-01", "query": "What does EMP stand for?", "turns": [], "relevantDocIds": ["power-emp-protection"], "mustInclude": ["electromagnetic pulse"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]} +{"id": "power-emp-spelled-01", "query": "How do I protect spare electronics from an electromagnetic pulse?", "turns": [], "relevantDocIds": ["power-emp-protection"], "mustInclude": ["faraday|trash can|conductive|enclos"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control"]} +{"id": "power-faraday-ground-01", "query": "Does a Faraday cage need to be grounded to protect what is inside it?", "turns": [], "relevantDocIds": ["power-emp-protection"], "mustInclude": ["not need|no need|not required|does not|doesn't"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "gear-bob-acronym-01", "query": "How many hours should a BOB sustain me?", "turns": [], "relevantDocIds": ["gear-bug-out-bag"], "mustInclude": ["72|three days|3 days"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym", "numeric"]} +{"id": "gear-bob-spelled-01", "query": "How many hours should a bug out bag sustain me?", "turns": [], "relevantDocIds": ["gear-bug-out-bag"], "mustInclude": ["72|three days|3 days"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym-control", "numeric"]} +{"id": "gear-bob-weight-01", "query": "How heavy should my bug out bag be?", "turns": [], "relevantDocIds": ["gear-bug-out-bag"], "mustInclude": ["20 ?%"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "gear-edc-acronym-01", "query": "What does EDC mean?", "turns": [], "relevantDocIds": ["gear-edc"], "mustInclude": ["every ?day carry"], "mustNotInclude": [], "expectRefusal": false, "tags": ["acronym"]} +{"id": "shelter-cotton-01", "query": "Why should I avoid cotton base layers in cold weather?", "turns": [], "relevantDocIds": ["shelter-cold-layering"], "mustInclude": ["25 times|conduct|holds water|wet|moist"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "shelter-layers-01", "query": "What are the three clothing layers for cold weather?", "turns": [], "relevantDocIds": ["shelter-cold-layering"], "mustInclude": ["base", "mid", "shell"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "shelter-cold-hands-01", "query": "My hands are cold even in gloves. What should I do?", "turns": [], "relevantDocIds": ["shelter-cold-layering"], "mustInclude": ["hat|head|core"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "sanitation-latrine-distance-01", "query": "How far from a water source should I dig a latrine?", "turns": [], "relevantDocIds": ["sanitation-waste"], "mustInclude": ["200 ?(feet|ft)|60 ?(meters|metres|m)\\b"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "sanitation-buckets-01", "query": "Why separate liquid and solid waste in a bucket toilet?", "turns": [], "relevantDocIds": ["sanitation-waste"], "mustInclude": ["smell|odou?r|pathogen"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "nav-declination-01", "query": "What is magnetic declination?", "turns": [], "relevantDocIds": ["navigation-map-compass"], "mustInclude": ["true north|magnetic north|angle|differ"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop"]} +{"id": "nav-pacecount-01", "query": "What is a typical pace count for 100 meters?", "turns": [], "relevantDocIds": ["navigation-map-compass"], "mustInclude": ["60|70"], "mustNotInclude": [], "expectRefusal": false, "tags": ["single-hop", "numeric"]} +{"id": "fict-thornfield-interval-01", "query": "How often do Thornfield cache sites rotate?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["11 months|eleven months"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]} +{"id": "fict-thornfield-states-01", "query": "What are the three states of a Thornfield cache site?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["primed", "resting", "pulled"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]} +{"id": "fict-thornfield-marker-01", "query": "How is a Thornfield cache site marked on the ground?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["five stones|5 stones|cairn"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]} +{"id": "fict-thornfield-custody-01", "query": "Who holds the full list of Thornfield cache sites?", "turns": [], "relevantDocIds": ["protocol-thornfield-cache"], "mustInclude": ["no single|two of the three|split|nobody|no one"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]} +{"id": "fict-tr88-flow-01", "query": "What is the rated flow of the TR-88 transfer pump?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["4\\.5"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]} +{"id": "fict-tr88-diaphragm-01", "query": "How often should the TR-88 diaphragm be replaced?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["500"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]} +{"id": "fict-tr88-maxtemp-01", "query": "What is the maximum fluid temperature for the TR-88 pump?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["40"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional", "numeric"]} +{"id": "fict-tr88-spares-01", "query": "What is the part number for the TR-88 spares kit?", "turns": [], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["TR88-SK2"], "mustNotInclude": [], "expectRefusal": false, "tags": ["fictional"]} +{"id": "multihop-kestrel-boil-01", "query": "How long should I boil water at Kestrel Ridge?", "turns": [], "relevantDocIds": ["regions-elevation-table", "water-boiling"], "mustInclude": ["(3|three) minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]} +{"id": "multihop-ashmere-boil-01", "query": "How long should I boil water at Ashmere Landing?", "turns": [], "relevantDocIds": ["regions-elevation-table", "water-boiling"], "mustInclude": ["\\b(1|one) minute"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop", "numeric"]} +{"id": "multihop-bellhollow-canning-01", "query": "Do I need to adjust my canner pressure at Bellhollow?", "turns": [], "relevantDocIds": ["regions-elevation-table", "food-canning-pressure"], "mustInclude": ["15 ?psi|increase|adjust|higher|yes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-hop"]} +{"id": "distract-river-drink-01", "query": "Is river water clean enough to drink straight from the stream?", "turns": [], "relevantDocIds": ["water-boiling", "water-filtration", "water-chemical-treatment"], "mustInclude": ["boil|filter|treat|disinfect"], "mustNotInclude": ["\\bpoem\\b|\\bverse\\b|\\bsong\\b|tall pines|highland"], "expectRefusal": false, "tags": ["distractor"]} +{"id": "distract-river-legit-01", "query": "What does the river song say happened after three days of rain?", "turns": [], "relevantDocIds": ["water-river-song"], "mustInclude": ["flood|high|ford"], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]} +{"id": "ooc-tire-pressure-01", "query": "What is the recommended tire pressure for a Toyota Hilux?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus"]} +{"id": "ooc-bgp-01", "query": "How do I configure BGP route reflectors on a Juniper router?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus"]} +{"id": "ooc-thornfield-founded-01", "query": "What year was the Thornfield Valley mutual aid group founded?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus", "adversarial"]} +{"id": "ooc-tr88-warranty-01", "query": "What is the warranty period on the TR-88 pump?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus", "adversarial"]} +{"id": "ooc-tr92-01", "query": "How much does the Model TR-92 pump weigh?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["out-of-corpus", "adversarial"]} +{"id": "multiturn-canning-temp-01", "query": "What temperature does it reach?", "turns": [{"role": "user", "content": "Tell me about pressure canning low acid foods."}, {"role": "assistant", "content": "Pressure canning is required for low-acid foods such as vegetables, meats, and soups, because a boiling water bath cannot destroy botulism spores."}], "relevantDocIds": ["food-canning-pressure"], "mustInclude": ["240"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn"]} +{"id": "multiturn-ifak-carry-01", "query": "Where should I carry it?", "turns": [{"role": "user", "content": "What should I put in an IFAK?"}, {"role": "assistant", "content": "An IFAK holds trauma gear: a tourniquet, hemostatic gauze, a pressure bandage, a chest seal, an airway, shears, and gloves."}], "relevantDocIds": ["firstaid-ifak-contents"], "mustInclude": ["body|person|on you"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn"]} +{"id": "multiturn-tr88-diaphragm-01", "query": "How often does the diaphragm need replacing?", "turns": [{"role": "user", "content": "Tell me about the TR-88 transfer pump."}, {"role": "assistant", "content": "The TR-88 is a hand-operated diaphragm pump for moving potable water between containers without power."}], "relevantDocIds": ["equipment-tr88-pump"], "mustInclude": ["500"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn", "fictional"]} +{"id": "multiturn-boil-altitude-01", "query": "What about up at Kestrel Ridge?", "turns": [{"role": "user", "content": "How long should I boil water to make it safe?"}, {"role": "assistant", "content": "Bring it to a rolling boil for one minute at normal elevations."}], "relevantDocIds": ["regions-elevation-table", "water-boiling"], "mustInclude": ["(3|three) minutes"], "mustNotInclude": [], "expectRefusal": false, "tags": ["multi-turn", "multi-hop"]} diff --git a/admin/tests/eval/reports/.gitignore b/admin/tests/eval/reports/.gitignore new file mode 100644 index 0000000..7b0a940 --- /dev/null +++ b/admin/tests/eval/reports/.gitignore @@ -0,0 +1,12 @@ +# Eval reports are run artifacts: they carry a timestamp, a host, and a git SHA, +# and a new one is written every time someone runs the harness. Committing them +# would add noise to every RAG pull request without making anything reproducible. +# +# Baselines are the opposite — those live in ../baselines// +# and ARE committed, because the whole team needs to gate against the same +# numbers. Promote a report with: +# +# node ace eval:compare --promote= +*.json +*.md +!.gitignore diff --git a/admin/tests/unit/eval_generation_metrics.spec.ts b/admin/tests/unit/eval_generation_metrics.spec.ts new file mode 100644 index 0000000..5dd3068 --- /dev/null +++ b/admin/tests/unit/eval_generation_metrics.spec.ts @@ -0,0 +1,294 @@ +/** + * Tests for the deterministic answer scorers. + * + * The refusal and leakage detectors are regex families, and a regex family is + * only as good as the phrasings it was tested against — so the cases below are + * real model phrasings, including the near-misses that must NOT trip them. + * A false positive on leakage would flag a perfectly good answer as a prompt + * regression, which is the fastest way to make people ignore the harness. + * + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + detectLeakage, + detectRefusal, + extractNumbers, + hasThinkTagLeak, + isMarkdownFormatted, + matchPatterns, + numericGroundedness, + scoreAnswer, + summarizeNumeric, + summarizeRepeats, +} from '../../app/utils/eval/generation_metrics.js' + +// --- refusal detection --------------------------------------------------------- + +const REFUSALS = [ + "I don't know the answer to that.", + 'I do not have that information.', + "I'm not able to answer that question.", + 'I am unable to determine that.', + "I couldn't find anything about that.", + 'There is no information available on that topic.', + 'That is not mentioned anywhere.', + 'The documentation does not specify a warranty period.', + "That isn't covered here.", + "I'd need more detail to answer that.", +] + +for (const [i, text] of REFUSALS.entries()) { + test(`detects refusal phrasing #${i + 1}: ${text.slice(0, 40)}`, () => { + assert.equal(detectRefusal(text), true) + }) +} + +const NON_REFUSALS = [ + 'Boil the water for three minutes above 6,500 feet.', + 'A tourniquet goes 2 to 3 inches above the wound.', + // "not" appears, but the answer is substantive — must not trip the detector. + 'Do not use ice on a burn; cool it under running water for 20 minutes.', + 'Water bath canning is not suitable for green beans, so use a pressure canner.', + 'You know the water is safe once it has held a rolling boil.', +] + +for (const [i, text] of NON_REFUSALS.entries()) { + test(`does not misread a substantive answer as a refusal #${i + 1}`, () => { + assert.equal(detectRefusal(text), false, `false positive on: ${text}`) + }) +} + +// --- leakage detection ----------------------------------------------------------- + +const LEAKS = [ + 'According to the context, boil for three minutes.', + 'Based on the provided context, use 8 drops per gallon.', + 'Context 1 describes the boiling procedure.', + 'The knowledge base does not cover that.', + 'The retrieved documents mention a 500 hour interval.', + "The context doesn't address your question.", + 'I found no relevant context for this.', + 'From the search results, the pressure is 10 psi.', +] + +for (const [i, text] of LEAKS.entries()) { + test(`detects retrieval narration #${i + 1}`, () => { + assert.ok(detectLeakage(text).length > 0, `missed leakage in: ${text}`) + }) +} + +const CLEAN = [ + 'Boil the water for three minutes.', + 'The TR-88 has a rated flow of 4.5 litres per minute.', + // "context" used in an ordinary sense, not as retrieval narration. + 'In a wilderness context, a tourniquet is the priority.', + 'Store one gallon per person per day.', +] + +for (const [i, text] of CLEAN.entries()) { + test(`does not flag a clean answer as leakage #${i + 1}`, () => { + assert.deepEqual(detectLeakage(text), [], `false positive on: ${text}`) + }) +} + +// --- think tags ------------------------------------------------------------------- + +test('detects surviving reasoning tags', () => { + assert.equal(hasThinkTagLeak('hmmThe answer is 3 minutes.'), true) + assert.equal(hasThinkTagLeak('x answer'), true) +}) + +test('does not flag ordinary prose containing the word think', () => { + assert.equal(hasThinkTagLeak('I think you should boil it longer.'), false) +}) + +// --- markdown --------------------------------------------------------------------- + +test('recognises common markdown structures', () => { + assert.equal(isMarkdownFormatted('## Heading\n\ntext'), true) + assert.equal(isMarkdownFormatted('- one\n- two'), true) + assert.equal(isMarkdownFormatted('1. first'), true) + assert.equal(isMarkdownFormatted('use **bold** here'), true) + assert.equal(isMarkdownFormatted('| a | b |'), true) +}) + +test('flags an answer with no formatting at all', () => { + assert.equal(isMarkdownFormatted('Just a plain sentence with no structure.'), false) +}) + +// --- numbers and grounding ---------------------------------------------------------- + +test('extracts and normalizes numbers', () => { + assert.deepEqual(extractNumbers('2,000cc and 300cc'), ['2000', '300']) +}) + +test('treats 2,000 and 2000 as the same number', () => { + const { score, ungrounded } = numericGroundedness('Use a 2,000cc absorber.', 'Use a 2000cc absorber.') + assert.equal(score, 1) + assert.deepEqual(ungrounded, []) +}) + +test('flags a number the context does not support', () => { + // The classic fabrication: a plausible figure that appears nowhere. + const { score, ungrounded } = numericGroundedness( + 'Replace the diaphragm every 250 hours.', + 'Replace the EPDM diaphragm every 500 operating hours.' + ) + assert.equal(score, 0) + assert.deepEqual(ungrounded, ['250']) +}) + +test('scores partial grounding', () => { + const { score } = numericGroundedness('Boil 3 minutes above 9000 feet.', 'Boil 3 minutes above 6500 feet.') + // 3 is excluded as a small integer; 9000 is claimed and unsupported. + assert.equal(score, 0) +}) + +test('ignores small integers, which appear incidentally in any prose', () => { + // "3 layers" and "step 2" would otherwise dominate the score with noise. + const { score } = numericGroundedness('There are 3 layers and 2 rules.', 'Wear a base, mid, and shell layer.') + assert.equal(score, null) +}) + +test('returns null when the answer makes no numeric claims', () => { + const { score } = numericGroundedness('Cool the burn under running water.', 'Cool for twenty minutes.') + assert.equal(score, null) +}) + +test('grounding is null, not 1.0, for qualitative answers', () => { + // Folding these in as perfect would quietly inflate the aggregate. + assert.equal(numericGroundedness('Use a pressure canner.', 'context').score, null) +}) + +// --- pattern matching ----------------------------------------------------------------- + +test('matchPatterns reports both sides', () => { + const { matched, missed } = matchPatterns('boil for three minutes', [ + String.raw`\b(3|three) minutes?`, + 'never appears', + ]) + assert.equal(matched.length, 1) + assert.deepEqual(missed, ['never appears']) +}) + +test('pattern matching is case insensitive', () => { + assert.equal(matchPatterns('BOIL FOR 3 MINUTES', ['3 minutes']).matched.length, 1) +}) + +// --- scoreAnswer ------------------------------------------------------------------------ + +test('a correct answer passes every assertion', () => { + const s = scoreAnswer({ + answer: 'Boil the water for **3 minutes** above 6,500 feet.', + context: 'boil for 3 minutes above 6,500 feet', + mustInclude: [String.raw`\b(3|three) minutes?`], + mustNotInclude: ['distill'], + expectRefusal: false, + }) + assert.equal(s.correct, true) + assert.deepEqual(s.missedRequired, []) + assert.deepEqual(s.hitForbidden, []) + assert.equal(s.refusalCorrect, true) + assert.deepEqual(s.leakage, []) +}) + +test('a forbidden phrase fails the case even when the required one matched', () => { + const s = scoreAnswer({ + answer: 'Boil for 3 minutes, or distill it.', + context: 'boil for 3 minutes', + mustInclude: ['3 minutes'], + mustNotInclude: ['distill'], + expectRefusal: false, + }) + assert.equal(s.correct, false) + assert.deepEqual(s.hitForbidden, ['distill']) +}) + +test('declining an out-of-corpus question is scored correct', () => { + const s = scoreAnswer({ + answer: "I don't have information about the warranty period.", + context: '', + mustInclude: [], + mustNotInclude: [], + expectRefusal: true, + }) + assert.equal(s.refused, true) + assert.equal(s.refusalCorrect, true) +}) + +test('inventing an answer to an out-of-corpus question is scored incorrect', () => { + // This is the failure that matters most: a confident, fabricated reply. + const s = scoreAnswer({ + answer: 'The TR-88 carries a 2 year warranty.', + context: '', + mustInclude: [], + mustNotInclude: [], + expectRefusal: true, + }) + assert.equal(s.refused, false) + assert.equal(s.refusalCorrect, false) +}) + +test('hedging on an answerable question is scored incorrect', () => { + // The exact regression that started NOMAD's RAG work: the model has good + // context and hedges anyway. + const s = scoreAnswer({ + answer: "I couldn't find specific context, but generally you boil water.", + context: 'boil for 3 minutes', + mustInclude: [], + mustNotInclude: [], + expectRefusal: false, + }) + assert.equal(s.refusalCorrect, false) + assert.ok(s.leakage.length > 0, 'should also flag the retrieval narration') +}) + +// --- repeat aggregation -------------------------------------------------------------------- + +test('all-pass is stable', () => { + const s = summarizeRepeats([true, true, true]) + assert.equal(s.passRate, 1) + assert.equal(s.unstable, false) +}) + +test('all-fail is stable', () => { + assert.equal(summarizeRepeats([false, false, false]).unstable, false) +}) + +test('a mixed outcome is flagged unstable and kept out of gating', () => { + const s = summarizeRepeats([true, false, true]) + assert.equal(s.passes, 2) + assert.ok(Math.abs(s.passRate - 2 / 3) < 1e-9) + assert.equal(s.unstable, true) +}) + +test('a single repeat is never called unstable', () => { + assert.equal(summarizeRepeats([true]).unstable, false) + assert.equal(summarizeRepeats([false]).unstable, false) +}) + +test('numeric summary reports mean and stddev', () => { + const s = summarizeNumeric([1, 1, 1])! + assert.equal(s.mean, 1) + assert.equal(s.stddev, 0) + assert.equal(s.n, 3) +}) + +test('numeric summary ignores nulls', () => { + const s = summarizeNumeric([1, null, 3])! + assert.equal(s.mean, 2) + assert.equal(s.n, 2) +}) + +test('numeric summary is null when there is nothing to summarize', () => { + assert.equal(summarizeNumeric([null, null]), null) +}) + +test('stddev is non-zero when runs disagree, which is the noise warning', () => { + const s = summarizeNumeric([0, 1])! + assert.equal(s.mean, 0.5) + assert.equal(s.stddev, 0.5) +}) diff --git a/admin/tests/unit/eval_golden_set.spec.ts b/admin/tests/unit/eval_golden_set.spec.ts new file mode 100644 index 0000000..1cef17b --- /dev/null +++ b/admin/tests/unit/eval_golden_set.spec.ts @@ -0,0 +1,227 @@ +/** + * Tests for golden-set parsing, validation, and the corpus fingerprint. + * + * The validation here is the difference between "the eval scored 0.62" and "the + * eval scored 0.62 because a golden had a typo'd doc id and could never match". + * Every failure mode below is one that would otherwise be silent. + * + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + assertGoldensMatchCorpus, + computeCorpusFingerprint, + GoldenSetError, + parseGoldens, +} from '../../app/utils/eval/golden_set.js' + +const line = (obj: Record) => + JSON.stringify({ + id: 'x', + query: 'q?', + relevantDocIds: ['doc-a'], + mustInclude: [], + mustNotInclude: [], + expectRefusal: false, + tags: [], + ...obj, + }) + +/** + * Deterministic stand-in for sha256. Returns 64 hex chars like the real thing, + * so the truncation behaviour under test is the behaviour that ships — a short + * fake would make the length assertion vacuous. + */ +const fakeHash = (s: string) => { + let a = 0x811c9dc5 + const out: string[] = [] + for (let round = 0; round < 8; round++) { + for (const ch of s) a = Math.imul(a ^ ch.charCodeAt(0), 0x01000193) >>> 0 + a = (a ^ round) >>> 0 + out.push(a.toString(16).padStart(8, '0')) + } + return out.join('') +} + +// --- parsing ----------------------------------------------------------------- + +test('parses a well-formed golden', () => { + const [g] = parseGoldens(line({ id: 'water-01', query: 'How long?', tags: ['numeric'] })) + assert.equal(g.id, 'water-01') + assert.equal(g.query, 'How long?') + assert.deepEqual(g.tags, ['numeric']) + assert.deepEqual(g.turns, []) +}) + +test('skips blank lines and // comments', () => { + const jsonl = ['', '// a note', line({ id: 'a' }), ' ', line({ id: 'b' })].join('\n') + assert.equal(parseGoldens(jsonl).length, 2) +}) + +test('rejects duplicate ids', () => { + const jsonl = [line({ id: 'dup' }), line({ id: 'dup' })].join('\n') + assert.throws(() => parseGoldens(jsonl), /duplicate id "dup"/) +}) + +test('rejects malformed JSON with the line number', () => { + assert.throws(() => parseGoldens('{not json'), /goldens:1: not valid JSON/) +}) + +test('rejects a missing query', () => { + assert.throws(() => parseGoldens(JSON.stringify({ id: 'a', relevantDocIds: ['d'] })), /missing "query"/) +}) + +test('rejects an empty query', () => { + assert.throws(() => parseGoldens(line({ query: ' ' })), /"query" must be a non-empty string/) +}) + +// --- the two contradictions that would silently score wrong ------------------- + +test('rejects a refusal case that also lists relevant documents', () => { + // One of the two is wrong, and either way the case is scored against a + // contradiction. Better to fail at load than to quietly grade nonsense. + assert.throws( + () => parseGoldens(line({ expectRefusal: true, relevantDocIds: ['doc-a'] })), + /expects a refusal but also lists relevant documents/ + ) +}) + +test('rejects a non-refusal case with no relevant documents', () => { + assert.throws( + () => parseGoldens(line({ expectRefusal: false, relevantDocIds: [] })), + /can never be scored/ + ) +}) + +test('accepts a refusal case with no relevant documents', () => { + const [g] = parseGoldens(line({ expectRefusal: true, relevantDocIds: [] })) + assert.equal(g.expectRefusal, true) + assert.deepEqual(g.relevantDocIds, []) +}) + +// --- regex validation --------------------------------------------------------- + +test('compiles mustInclude patterns at load so a bad regex fails immediately', () => { + assert.throws(() => parseGoldens(line({ mustInclude: ['(unclosed'] })), /invalid regex/) +}) + +test('compiles mustNotInclude patterns too', () => { + assert.throws(() => parseGoldens(line({ mustNotInclude: ['[z-a]'] })), /invalid regex/) +}) + +test('accepts alternation, which is how one entry covers "3 minutes" and "three minutes"', () => { + const [g] = parseGoldens(line({ mustInclude: [String.raw`\b(3|three) minutes?`] })) + assert.match('boil for three minutes', new RegExp(g.mustInclude[0], 'i')) + assert.match('boil for 3 minutes', new RegExp(g.mustInclude[0], 'i')) +}) + +// --- turns -------------------------------------------------------------------- + +test('accepts multi-turn history', () => { + const [g] = parseGoldens( + line({ turns: [{ role: 'user', content: 'first' }, { role: 'assistant', content: 'reply' }] }) + ) + assert.equal(g.turns.length, 2) +}) + +test('rejects a turn with a bogus role', () => { + assert.throws(() => parseGoldens(line({ turns: [{ role: 'system', content: 'x' }] })), /"turns" must be/) +}) + +test('rejects an empty golden file rather than silently reporting a perfect score', () => { + assert.throws(() => parseGoldens('\n\n// only comments\n'), /no goldens found/) +}) + +// --- corpus cross-check -------------------------------------------------------- + +test('accepts goldens whose documents all exist', () => { + const goldens = parseGoldens(line({ relevantDocIds: ['doc-a', 'doc-b'] })) + assert.doesNotThrow(() => assertGoldensMatchCorpus(goldens, ['doc-a', 'doc-b', 'doc-c'])) +}) + +test('rejects a golden pointing at a document that is not in the corpus', () => { + // A typo'd doc id makes recall unhittable for that question forever, and the + // only symptom is a slightly lower score. This is the check that catches it. + const goldens = parseGoldens(line({ relevantDocIds: ['doc-typo'] })) + assert.throws( + () => assertGoldensMatchCorpus(goldens, ['doc-a']), + (err: unknown) => err instanceof GoldenSetError && /doc-typo/.test((err as Error).message) + ) +}) + +// --- fingerprint ---------------------------------------------------------------- + +const inputs = (over: Record = {}) => ({ + documents: new Map([ + ['a', 'alpha'], + ['b', 'beta'], + ]), + chunkTokens: 1500, + chunkOverlapTokens: 150, + charToTokenRatio: 2, + embeddingModel: 'nomic-embed-text:v1.5', + embeddingDimension: 768, + ...over, +}) + +test('fingerprint is stable across runs', () => { + assert.equal(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(inputs(), fakeHash)) +}) + +test('fingerprint ignores document insertion order', () => { + const reversed = inputs({ + documents: new Map([ + ['b', 'beta'], + ['a', 'alpha'], + ]), + }) + assert.equal(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(reversed, fakeHash)) +}) + +test('fingerprint changes when a document changes', () => { + const edited = inputs({ documents: new Map([['a', 'alpha!'], ['b', 'beta']]) }) + assert.notEqual(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(edited, fakeHash)) +}) + +test('fingerprint changes when a document is added', () => { + const added = inputs({ documents: new Map([['a', 'alpha'], ['b', 'beta'], ['c', 'gamma']]) }) + assert.notEqual(computeCorpusFingerprint(inputs(), fakeHash), computeCorpusFingerprint(added, fakeHash)) +}) + +test('fingerprint changes when the chunk size changes', () => { + // This is the one that matters most: re-chunking invalidates every prior + // score, and the fingerprint is what stops us comparing across the change. + assert.notEqual( + computeCorpusFingerprint(inputs(), fakeHash), + computeCorpusFingerprint(inputs({ chunkTokens: 512 }), fakeHash) + ) +}) + +test('fingerprint changes when the token-estimate ratio changes', () => { + // CHAR_TO_TOKEN_RATIO feeds the chunker as a character count, so changing it + // silently re-chunks the corpus. It must be in the fingerprint. + assert.notEqual( + computeCorpusFingerprint(inputs(), fakeHash), + computeCorpusFingerprint(inputs({ charToTokenRatio: 3 }), fakeHash) + ) +}) + +test('fingerprint changes when the embedding model changes', () => { + assert.notEqual( + computeCorpusFingerprint(inputs(), fakeHash), + computeCorpusFingerprint(inputs({ embeddingModel: 'mxbai-embed-large' }), fakeHash) + ) +}) + +test('fingerprint is short enough to paste into a filename', () => { + assert.equal(computeCorpusFingerprint(inputs(), fakeHash).length, 16) +}) + +test('document contents cannot collide by concatenation', () => { + // Naive joining lets {a: "xy", b: ""} hash the same as {a: "x", b: "y"}. + const one = inputs({ documents: new Map([['a', 'xy'], ['b', '']]) }) + const two = inputs({ documents: new Map([['a', 'x'], ['b', 'y']]) }) + assert.notEqual(computeCorpusFingerprint(one, fakeHash), computeCorpusFingerprint(two, fakeHash)) +}) diff --git a/admin/tests/unit/eval_report.spec.ts b/admin/tests/unit/eval_report.spec.ts new file mode 100644 index 0000000..024fcad --- /dev/null +++ b/admin/tests/unit/eval_report.spec.ts @@ -0,0 +1,172 @@ +/** + * Tests for baseline diffing and the regression gate. + * + * This is the logic that decides whether a pull request is blocked, so the + * cases below focus on the two ways a gate loses trust: firing on noise, and + * comparing runs that were never comparable in the first place. + * + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + compareReports, + DEFAULT_TOLERANCE, + flattenByK, + renderComparisonMarkdown, + type EvalReport, +} from '../../app/utils/eval/report.js' + +const report = ( + metrics: Record, + over: Partial = {} +): EvalReport => ({ + meta: { + kind: 'retrieval', + createdAt: '2026-08-12T00:00:00.000Z', + corpusFingerprint: 'abc123', + gitSha: 'deadbeef', + gitBranch: 'dev', + gitDirty: false, + nomadVersion: '1.34.0', + platform: { cpuArchitecture: 'x64', osName: 'Linux', nodeVersion: 'v24' }, + params: {}, + ...over, + }, + metrics, + byTag: {}, + cases: [], +}) + +// --- the comparability veto ------------------------------------------------------ + +test('refuses to compare across corpus fingerprints', () => { + // Diffing runs against different corpora would manufacture a regression out + // of a chunk-size change. Refusing is the correct behaviour. + const result = compareReports( + report({ mrr: 0.9 }), + report({ mrr: 0.5 }, { corpusFingerprint: 'different' }) + ) + assert.equal(result.comparable, false) + assert.match(result.incomparableReason!, /fingerprint changed/) + assert.deepEqual(result.regressions, []) +}) + +test('refuses to compare a retrieval report against a generation report', () => { + const result = compareReports(report({ mrr: 0.9 }), report({ correctness: 0.9 }, { kind: 'generation' })) + assert.equal(result.comparable, false) + assert.match(result.incomparableReason!, /retrieval report against a generation/) +}) + +test('compares happily when the fingerprints match', () => { + assert.equal(compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 })).comparable, true) +}) + +// --- direction awareness ----------------------------------------------------------- + +test('a drop in a higher-is-better metric is a regression', () => { + const r = compareReports(report({ 'recall@5': 0.9 }), report({ 'recall@5': 0.5 })) + assert.equal(r.regressions.length, 1) + assert.equal(r.regressions[0].metric, 'recall@5') +}) + +test('a rise in a lower-is-better metric is a regression', () => { + // leakageRate going up means the model started narrating retrieval again. + const r = compareReports(report({ leakageRate: 0.0 }), report({ leakageRate: 0.4 })) + assert.equal(r.regressions.length, 1) + assert.equal(r.regressions[0].metric, 'leakageRate') +}) + +test('a fall in a lower-is-better metric is an improvement, not a regression', () => { + const r = compareReports(report({ leakageRate: 0.4 }), report({ leakageRate: 0.0 })) + assert.deepEqual(r.regressions, []) + assert.equal(r.improvements.length, 1) +}) + +test('emptyRateOnAnswerable is treated as lower-is-better', () => { + const r = compareReports(report({ emptyRateOnAnswerable: 0.1 }), report({ emptyRateOnAnswerable: 0.6 })) + assert.equal(r.regressions.length, 1) +}) + +test('an unknown metric defaults to higher-is-better rather than being ignored', () => { + const r = compareReports(report({ somethingNew: 0.9 }), report({ somethingNew: 0.1 })) + assert.equal(r.regressions.length, 1) +}) + +// --- the tolerance band --------------------------------------------------------------- + +test('a movement inside the tolerance band is neither a regression nor an improvement', () => { + // A gate that fires on 0.001 gets switched off within a week. + const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 - DEFAULT_TOLERANCE / 2 })) + assert.deepEqual(r.regressions, []) + assert.deepEqual(r.improvements, []) +}) + +test('a movement exactly at the tolerance is not yet a regression', () => { + const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 - DEFAULT_TOLERANCE })) + assert.deepEqual(r.regressions, []) +}) + +test('tolerance is configurable', () => { + const loose = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.85 }), 0.1) + const tight = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.85 }), 0.01) + assert.deepEqual(loose.regressions, []) + assert.equal(tight.regressions.length, 1) +}) + +// --- nulls and asymmetric metric sets --------------------------------------------------- + +test('a null on either side is never a regression', () => { + // "not measurable in this run" must not read as "collapsed to zero". + assert.deepEqual(compareReports(report({ mrr: 0.9 }), report({ mrr: null })).regressions, []) + assert.deepEqual(compareReports(report({ mrr: null }), report({ mrr: 0.1 })).regressions, []) +}) + +test('a metric present only in the current report is marked new, not regressed', () => { + const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9, ndcgNew: 0.1 })) + const added = r.deltas.find((d) => d.metric === 'ndcgNew')! + assert.equal(added.onlyIn, 'current') + assert.equal(added.regressed, false) +}) + +test('a metric dropped from the current report is marked removed', () => { + const r = compareReports(report({ mrr: 0.9, gone: 0.5 }), report({ mrr: 0.9 })) + const removed = r.deltas.find((d) => d.metric === 'gone')! + assert.equal(removed.onlyIn, 'baseline') + assert.equal(removed.regressed, false) +}) + +// --- helpers and rendering ------------------------------------------------------------- + +test('flattenByK produces name@k keys', () => { + assert.deepEqual(flattenByK('recall', { 1: 0.5, 5: 0.9 }), { 'recall@1': 0.5, 'recall@5': 0.9 }) +}) + +test('markdown leads with the regression count', () => { + const r = compareReports(report({ 'recall@5': 0.9 }), report({ 'recall@5': 0.4 })) + const md = renderComparisonMarkdown('base.json', 'cur.json', r) + assert.match(md, /## 1 regression\(s\)/) + assert.match(md, /\*\*REGRESSED\*\*/) +}) + +test('markdown says so plainly when nothing regressed', () => { + const md = renderComparisonMarkdown('b.json', 'c.json', compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 }))) + assert.match(md, /## No regressions/) +}) + +test('markdown explains an incomparable pair instead of printing an empty table', () => { + const r = compareReports(report({ mrr: 0.9 }), report({ mrr: 0.9 }, { corpusFingerprint: 'other' })) + const md = renderComparisonMarkdown('b.json', 'c.json', r) + assert.match(md, /## Not comparable/) + assert.match(md, /fingerprint changed/) +}) + +test('markdown orders the worst regression first', () => { + const r = compareReports( + report({ small: 0.9, big: 0.9 }), + report({ small: 0.85, big: 0.2 }) + ) + const md = renderComparisonMarkdown('b.json', 'c.json', r) + assert.ok(md.indexOf('| big |') < md.indexOf('| small |'), 'worst regression should come first') +}) diff --git a/admin/tests/unit/eval_retrieval_metrics.spec.ts b/admin/tests/unit/eval_retrieval_metrics.spec.ts new file mode 100644 index 0000000..5657761 --- /dev/null +++ b/admin/tests/unit/eval_retrieval_metrics.spec.ts @@ -0,0 +1,264 @@ +/** + * Tests for the retrieval metrics. + * + * Expected values are hand-computed from the formulas in the module doc, not + * captured from a run — a snapshot test of a wrong implementation just freezes + * the wrong answer. nDCG in particular is easy to get subtly wrong (log base, + * off-by-one in the rank, what the ideal ranking is normalized against), and a + * silently wrong metric is worse than no metric. + * + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + aggregate, + aggregateByTag, + describeScores, + hitRateAtK, + meanOf, + ndcgAtK, + precisionAtK, + recallAtK, + reciprocalRank, + scoreCase, + toDocumentRanking, + type RetrievalCase, + type ScoredChunk, +} from '../../app/utils/eval/retrieval_metrics.js' + +/** Build a ranked chunk list from doc ids, best first. */ +const chunks = (...docIds: Array): ScoredChunk[] => + docIds.map((docId, i) => ({ docId, score: 1 - i * 0.05, semanticScore: 0.9 - i * 0.05 })) + +const close = (actual: number | null, expected: number, msg?: string) => { + assert.ok(actual !== null, msg ?? 'expected a value, got null') + assert.ok(Math.abs(actual - expected) < 1e-9, `${msg ?? ''} expected ${expected}, got ${actual}`) +} + +// --- document ranking --------------------------------------------------------- + +test('document ranking keeps each document at its best rank', () => { + assert.deepEqual(toDocumentRanking(chunks('a', 'b', 'a', 'c', 'b')), ['a', 'b', 'c']) +}) + +test('document ranking drops chunks with no resolvable document', () => { + assert.deepEqual(toDocumentRanking(chunks('a', null, 'b')), ['a', 'b']) +}) + +// --- recall ------------------------------------------------------------------- + +test('recall@k finds both documents of a multi-hop question', () => { + close(recallAtK(chunks('a', 'x', 'b'), ['a', 'b'], 5), 1) +}) + +test('recall@k is partial when only one required document is in the window', () => { + // 'b' sits at rank 3, outside k=2. One of two relevant docs found. + close(recallAtK(chunks('a', 'x', 'b'), ['a', 'b'], 2), 0.5) +}) + +test('recall@k counts distinct documents, not chunks', () => { + // Three chunks, all from 'a'. That is one document found, not three. + close(recallAtK(chunks('a', 'a', 'a'), ['a', 'b'], 5), 0.5) +}) + +test('recall is null for out-of-corpus cases rather than 0', () => { + // Scoring these as 0 would drag the mean down for questions that are + // supposed to retrieve nothing — punishing correct behaviour. + assert.equal(recallAtK(chunks('x'), [], 5), null) +}) + +test('recall is 0 when nothing relevant was retrieved', () => { + close(recallAtK(chunks('x', 'y'), ['a'], 5), 0) +}) + +// --- hit rate ----------------------------------------------------------------- + +test('hit rate is 1 when any relevant document appears', () => { + assert.equal(hitRateAtK(chunks('x', 'a'), ['a', 'b'], 5), 1) +}) + +test('hit rate and recall diverge on multi-hop, which is the point of having both', () => { + const retrieved = chunks('a', 'x', 'y') + assert.equal(hitRateAtK(retrieved, ['a', 'b'], 5), 1) + close(recallAtK(retrieved, ['a', 'b'], 5), 0.5) +}) + +test('hit rate respects the k window', () => { + assert.equal(hitRateAtK(chunks('x', 'y', 'a'), ['a'], 2), 0) +}) + +// --- precision ---------------------------------------------------------------- + +test('precision@k is chunk level and deliberately not deduped', () => { + // Two of four injected chunks are noise, regardless of how many documents + // they came from — the small model pays for all four. + close(precisionAtK(chunks('a', 'a', 'x', 'y'), ['a'], 4), 0.5) +}) + +test('precision divides by what was actually retrieved, not by k', () => { + // Returning 2 good chunks should score 1.0, not 0.4 — declining to pad the + // context with noise is correct behaviour and must not be penalised. + close(precisionAtK(chunks('a', 'a'), ['a'], 5), 1) +}) + +test('precision is 0 when nothing was retrieved', () => { + close(precisionAtK([], ['a'], 5), 0) +}) + +// --- reciprocal rank ---------------------------------------------------------- + +test('reciprocal rank is 1 when the first chunk is relevant', () => { + close(reciprocalRank(chunks('a', 'x'), ['a']), 1) +}) + +test('reciprocal rank is 1/3 when the first relevant chunk is third', () => { + close(reciprocalRank(chunks('x', 'y', 'a'), ['a']), 1 / 3) +}) + +test('reciprocal rank is 0 when nothing relevant was retrieved', () => { + close(reciprocalRank(chunks('x', 'y'), ['a']), 0) +}) + +// --- nDCG --------------------------------------------------------------------- + +test('nDCG is 1 when the single relevant document ranks first', () => { + close(ndcgAtK(chunks('a', 'x', 'y'), ['a'], 5), 1) +}) + +test('nDCG at rank 2 equals 1/log2(3)', () => { + // DCG = 1/log2(2+1) = 1/1.58496 = 0.63093 + // IDCG = 1/log2(1+1) = 1 + close(ndcgAtK(chunks('x', 'a'), ['a'], 5), 1 / Math.log2(3)) +}) + +test('nDCG normalizes against the known relevant count, not the retrieved set', () => { + // One of three required documents, ranked first. + // DCG = 1/log2(2) = 1 + // IDCG = 1/log2(2) + 1/log2(3) + 1/log2(4) = 1 + 0.63093 + 0.5 = 2.13093 + // Normalizing against the retrieved set instead would report a perfect 1.0 + // for a run that missed two thirds of the answer. + const idcg = 1 + 1 / Math.log2(3) + 1 / Math.log2(4) + close(ndcgAtK(chunks('a', 'x', 'y'), ['a', 'b', 'c'], 5), 1 / idcg) +}) + +test('nDCG punishes ordering even when recall is unchanged', () => { + // This is the regression nDCG exists to catch: same documents retrieved, + // pushed down the list, recall identical. + const good = chunks('a', 'b', 'x', 'y') + const bad = chunks('x', 'y', 'a', 'b') + close(recallAtK(good, ['a', 'b'], 5), 1) + close(recallAtK(bad, ['a', 'b'], 5), 1) + const nGood = ndcgAtK(good, ['a', 'b'], 5)! + const nBad = ndcgAtK(bad, ['a', 'b'], 5)! + assert.equal(nGood, 1) + assert.ok(nBad < nGood, `expected ${nBad} < ${nGood}`) +}) + +test('nDCG ideal ranking is capped at k', () => { + // Three relevant docs but k=1: the best achievable is one hit at rank 1. + close(ndcgAtK(chunks('a', 'b', 'c'), ['a', 'b', 'c'], 1), 1) +}) + +test('nDCG collapses duplicate chunks from the same document', () => { + // Three chunks of 'a' must not be credited as three separate hits. + const idcg = 1 + 1 / Math.log2(3) + close(ndcgAtK(chunks('a', 'a', 'a'), ['a', 'b'], 5), 1 / idcg) +}) + +test('nDCG is 0 when nothing relevant is retrieved', () => { + close(ndcgAtK(chunks('x', 'y'), ['a'], 5), 0) +}) + +// --- score distribution --------------------------------------------------------- + +test('describeScores reports values that actually occurred', () => { + const d = describeScores([0.1, 0.2, 0.3, 0.4, 0.5])! + assert.equal(d.count, 5) + assert.equal(d.min, 0.1) + assert.equal(d.max, 0.5) + assert.equal(d.median, 0.3) + close(d.mean, 0.3) + // Nearest-rank, so every reported percentile is a real observation. + assert.ok([0.1, 0.2, 0.3, 0.4, 0.5].includes(d.p10)) + assert.ok([0.1, 0.2, 0.3, 0.4, 0.5].includes(d.p90)) +}) + +test('describeScores returns null for an empty sample', () => { + assert.equal(describeScores([]), null) +}) + +test('meanOf ignores nulls and returns null when everything is null', () => { + close(meanOf([1, null, 3]), 2) + assert.equal(meanOf([null, null]), null) +}) + +// --- aggregation ---------------------------------------------------------------- + +const mkCase = (over: Partial): RetrievalCase => ({ + id: 'c', + tags: [], + retrieved: chunks('a'), + relevantDocIds: ['a'], + expectRefusal: false, + ...over, +}) + +test('aggregate separates answerable cases from refusal cases', () => { + const cases = [ + mkCase({ id: 'q1', retrieved: chunks('a'), relevantDocIds: ['a'] }), + mkCase({ id: 'q2', retrieved: [], relevantDocIds: [], expectRefusal: true }), + ] + const results = cases.map((c) => scoreCase(c)) + const agg = aggregate(cases, results) + assert.equal(agg.cases, 2) + assert.equal(agg.answerable, 1) + // The refusal case contributes null to recall, so the mean is over q1 alone. + close(agg.recall[5], 1) +}) + +test('empty rate on answerable questions surfaces an over-tight threshold', () => { + const cases = [ + mkCase({ id: 'q1', retrieved: [] }), + mkCase({ id: 'q2', retrieved: chunks('a') }), + ] + const agg = aggregate(cases, cases.map((c) => scoreCase(c))) + close(agg.emptyRateOnAnswerable, 0.5) +}) + +test('non-empty rate on refusal questions surfaces an over-loose threshold', () => { + const cases = [ + mkCase({ id: 'r1', retrieved: chunks('x'), relevantDocIds: [], expectRefusal: true }), + mkCase({ id: 'r2', retrieved: [], relevantDocIds: [], expectRefusal: true }), + ] + const agg = aggregate(cases, cases.map((c) => scoreCase(c))) + close(agg.nonEmptyRateOnRefusal, 0.5) +}) + +test('score distributions split relevant from irrelevant chunks', () => { + const cases = [mkCase({ retrieved: chunks('a', 'x', 'y'), relevantDocIds: ['a'] })] + const agg = aggregate(cases, cases.map((c) => scoreCase(c))) + assert.equal(agg.relevantScores!.count, 1) + assert.equal(agg.irrelevantScores!.count, 2) + // The relevant chunk ranked first, so it should score above the noise. + assert.ok(agg.relevantScores!.median > agg.irrelevantScores!.median) +}) + +test('refusal cases contribute only to the irrelevant score population', () => { + const cases = [mkCase({ retrieved: chunks('x', 'y'), relevantDocIds: [], expectRefusal: true })] + const agg = aggregate(cases, cases.map((c) => scoreCase(c))) + assert.equal(agg.relevantScores, null) + assert.equal(agg.irrelevantScores!.count, 2) +}) + +test('per-tag aggregation slices the same cases without recomputing them wrong', () => { + const cases = [ + mkCase({ id: 'q1', tags: ['single-hop'], retrieved: chunks('a'), relevantDocIds: ['a'] }), + mkCase({ id: 'q2', tags: ['multi-hop'], retrieved: chunks('a'), relevantDocIds: ['a', 'b'] }), + ] + const byTag = aggregateByTag(cases, cases.map((c) => scoreCase(c))) + close(byTag['single-hop'].recall[5], 1) + close(byTag['multi-hop'].recall[5], 0.5) + assert.equal(byTag['single-hop'].cases, 1) +}) diff --git a/admin/tests/unit/eval_source_guard.spec.ts b/admin/tests/unit/eval_source_guard.spec.ts new file mode 100644 index 0000000..bd9ff5a --- /dev/null +++ b/admin/tests/unit/eval_source_guard.spec.ts @@ -0,0 +1,84 @@ +/** + * Tests for the eval-corpus leak guard. + * + * The eval corpus shares the `nomad_knowledge_base` Qdrant collection with the + * developer's real documents, isolated by a `collection: __nomad_eval__` payload + * filter. That filter is applied by Qdrant during search and does hold — but a + * harness that depends on a filter must be able to *prove* the filter held, not + * assume it. + * + * `docIdFromSource` is that proof: anything it cannot resolve to a corpus + * document is counted as an unresolved chunk, and a non-zero count fails the + * run. These tests exist because the first version checked only for a `.md` + * extension, which silently accepted a developer's own markdown — and NOMAD + * embeds its own `admin/docs/*.md` on first run, so that was not hypothetical. + * + * npm run test:eval + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' +import { join, resolve } from 'node:path' + +import { docIdFromSource } from '../../app/utils/eval/corpus_source.js' + +const CORPUS = resolve('/srv/nomad/admin/tests/eval/corpus') +const inCorpus = (name: string) => join(CORPUS, name) + +const resolveId = (source: unknown) => docIdFromSource(source, CORPUS) + +test('resolves a genuine corpus document', () => { + assert.equal(resolveId(inCorpus('water-boiling.md')), 'water-boiling') +}) + +test('resolves every corpus document to its filename without the extension', () => { + assert.equal(resolveId(inCorpus('regions-elevation-table.md')), 'regions-elevation-table') + assert.equal(resolveId(inCorpus('equipment-tr88-pump.md')), 'equipment-tr88-pump') +}) + +test("rejects the developer's own markdown, even though it ends in .md", () => { + // The regression this guard exists for. NOMAD embeds admin/docs/*.md into the + // knowledge base on first run, so if the collection filter ever leaked, this + // is the exact shape that would come back. Resolving it to "faq" would have + // been counted as a merely-irrelevant chunk and quietly lowered precision + // instead of failing the run. + assert.equal(resolveId('/srv/nomad/admin/docs/faq.md'), null) + assert.equal(resolveId('/srv/nomad/admin/docs/release-notes.md'), null) +}) + +test('rejects an uploaded knowledge-base file', () => { + assert.equal(resolveId('/srv/nomad/admin/storage/kb_uploads/notes-abc123.txt'), null) + assert.equal(resolveId('/srv/nomad/admin/storage/kb_uploads/notes-abc123.md'), null) +}) + +test('rejects a ZIM article source', () => { + assert.equal(resolveId('/srv/nomad/admin/storage/zim/wikipedia_en_100_mini.zim'), null) +}) + +test('rejects a sibling directory whose path merely starts with the corpus path', () => { + // "…/corpus-backup/x.md" shares a string prefix with "…/corpus". Without the + // trailing separator in the check this would pass. + assert.equal(resolveId('/srv/nomad/admin/tests/eval/corpus-backup/water-boiling.md'), null) +}) + +test('rejects a path that escapes the corpus via traversal', () => { + assert.equal(resolveId(join(CORPUS, '..', '..', '..', 'docs', 'faq.md')), null) +}) + +test('rejects a non-markdown file inside the corpus directory', () => { + assert.equal(resolveId(inCorpus('README.txt')), null) +}) + +test('rejects a missing or non-string source', () => { + assert.equal(resolveId(undefined), null) + assert.equal(resolveId(null), null) + assert.equal(resolveId(42), null) + assert.equal(resolveId(''), null) +}) + +test('rejects the corpus directory itself', () => { + assert.equal(resolveId(CORPUS), null) +}) + +test('accepts a nested document, should the corpus ever grow subdirectories', () => { + assert.equal(resolveId(inCorpus('water/boiling.md')), 'boiling') +}) diff --git a/admin/tests/unit/rag_pipeline_prompt.spec.ts b/admin/tests/unit/rag_pipeline_prompt.spec.ts new file mode 100644 index 0000000..67be486 --- /dev/null +++ b/admin/tests/unit/rag_pipeline_prompt.spec.ts @@ -0,0 +1,189 @@ +/** + * Characterization tests for the prompt-assembly helpers extracted out of + * OllamaController into RagPipelineService. + * + * These lock in the *current* behaviour, quirks included, so the extraction is + * provably a no-op and so any later deliberate change to context budgeting or + * the num_ctx ladder shows up as a failing test rather than a silent shift in + * answer quality. + * + * Pure functions only — no MySQL, Redis, Qdrant, or Ollama needed: + * npm run test:unit + */ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { + buildContextBlock, + deriveNumCtx, + getContextLimitsForModel, + NUM_CTX_TRIGGER_TOKENS, + PROMPT_CHARS_PER_TOKEN, + trimToContextBudget, + type ContextLimitTier, +} from '../../app/utils/rag_prompt.js' + +/** Mirrors RAG_CONTEXT_LIMITS in constants/ollama.ts. Passed in explicitly so + * these tests need no AdonisJS-flavoured imports, per the kb_ratio_lookup + * precedent — and so a change to the shipped tiers shows up here as a + * deliberate edit rather than a silently-passing test. */ +const TIERS: ContextLimitTier[] = [ + { maxParams: 3, maxResults: 2, maxTokens: 1000 }, + { maxParams: 8, maxResults: 4, maxTokens: 2500 }, + { maxParams: Infinity, maxResults: 5, maxTokens: 0 }, +] + +const limits = (model: string) => getContextLimitsForModel(model, TIERS) + +const chunk = (text: string, metadata: Record = {}) => ({ + text, + score: 0.5, + metadata, +}) + +// --- getContextLimitsForModel ------------------------------------------------ + +test('context limits: 1-3B models get the tightest budget', () => { + assert.deepEqual(limits('llama3.2:1b'), { maxResults: 2, maxTokens: 1000 }) + assert.deepEqual(limits('qwen2.5:3b'), { maxResults: 2, maxTokens: 1000 }) +}) + +test('context limits: fractional sizes parse correctly', () => { + // "1.5b" must read as 1.5, not 1 or 15 — it decides which tier the model lands in. + assert.deepEqual(limits('qwen2.5:1.5b'), { maxResults: 2, maxTokens: 1000 }) +}) + +test('context limits: 4-8B tier', () => { + assert.deepEqual(limits('llama3.1:8b'), { maxResults: 4, maxTokens: 2500 }) +}) + +test('context limits: 13B+ is uncapped', () => { + assert.deepEqual(limits('llama2:70b'), { maxResults: 5, maxTokens: 0 }) +}) + +test('context limits: unparseable model name is assumed to be 8B', () => { + // Documented quirk, not an endorsement: "phi3" has no size token, so it is + // handed the 4-8B budget regardless of what it actually is. + assert.deepEqual(limits('phi3'), { maxResults: 4, maxTokens: 2500 }) +}) + +test('context limits: quantization suffixes do not confuse the size parse', () => { + assert.deepEqual(limits('llama3.1:8b-text-q4_1'), { + maxResults: 4, + maxTokens: 2500, + }) +}) + +// --- trimToContextBudget ----------------------------------------------------- + +test('trim: caps the number of results', () => { + const docs = [chunk('a'), chunk('b'), chunk('c'), chunk('d')] + const out = trimToContextBudget(docs, { maxResults: 2, maxTokens: 0 }) + assert.equal(out.length, 2) + assert.deepEqual( + out.map((d: { text: string }) => d.text), + ['a', 'b'] + ) +}) + +test('trim: maxTokens of 0 means uncapped', () => { + const docs = [chunk('x'.repeat(50_000)), chunk('y'.repeat(50_000))] + const out = trimToContextBudget(docs, { maxResults: 5, maxTokens: 0 }) + assert.equal(out.length, 2) +}) + +test('trim: the top result survives even when it alone blows the budget', () => { + // This is the guard that stops a small model from getting *no* context at all + // when the single best chunk happens to be enormous. + const huge = chunk('x'.repeat(100_000)) + const out = trimToContextBudget([huge, chunk('small')], { maxResults: 5, maxTokens: 1000 }) + assert.equal(out.length, 1) + assert.equal(out[0].text.length, 100_000) +}) + +test('trim: drops later results once the character cap is exceeded', () => { + const capChars = 1000 * PROMPT_CHARS_PER_TOKEN // 3500 + const docs = [chunk('a'.repeat(1000)), chunk('b'.repeat(1000)), chunk('c'.repeat(3000))] + const out = trimToContextBudget(docs, { maxResults: 5, maxTokens: 1000 }) + // Running totals: 1000 (kept, idx 0), 2000 (<= 3500, kept), 5000 (> 3500, dropped). + assert.deepEqual( + out.map((d: { text: string }) => d.text[0]), + ['a', 'b'] + ) + assert.ok(capChars === 3500) +}) + +test('trim: the count cap is applied before the token cap', () => { + const docs = [chunk('a'), chunk('b'), chunk('c')] + const out = trimToContextBudget(docs, { maxResults: 1, maxTokens: 1000 }) + assert.equal(out.length, 1) +}) + +// --- buildContextBlock ------------------------------------------------------- + +test('context block: numbers each chunk from 1', () => { + const out = buildContextBlock([chunk('first'), chunk('second')]) + assert.equal(out, '[Context 1]\nfirst\n\n[Context 2]\nsecond') +}) + +test('context block: labels with full_title when present', () => { + const out = buildContextBlock([chunk('body', { full_title: 'Water - Boiling' })]) + assert.equal(out, '[Context 1 — Water - Boiling]\nbody') +}) + +test('context block: falls back to article_title', () => { + const out = buildContextBlock([chunk('body', { article_title: 'Water' })]) + assert.equal(out, '[Context 1 — Water]\nbody') +}) + +test('context block: full_title wins over article_title', () => { + const out = buildContextBlock([chunk('body', { full_title: 'A - B', article_title: 'A' })]) + assert.equal(out, '[Context 1 — A - B]\nbody') +}) + +test('context block: never leaks the relevance score to the model', () => { + // Deliberate: nomic cosine scores for genuinely relevant passages sit around + // 0.4-0.6, and showing the model "42%" primes it to distrust correct context. + const out = buildContextBlock([chunk('body', { source: 'f.md', semantic_score: 0.42 })]) + assert.ok(!out.includes('0.42')) + assert.ok(!out.includes('42')) +}) + +// --- deriveNumCtx ------------------------------------------------------------ + +const sys = (chars: number) => ({ role: 'system' as const, content: 'x'.repeat(chars) }) + +test('numCtx: unset below the trigger, so the backend default applies', () => { + // Ollama's default is a silent 2048. Below the trigger we deliberately say + // nothing — a known risk, captured here so a future fix is a visible change. + assert.equal(deriveNumCtx([sys(100)]), undefined) +}) + +test('numCtx: exactly at the trigger is still unset (strict greater-than)', () => { + const chars = NUM_CTX_TRIGGER_TOKENS * PROMPT_CHARS_PER_TOKEN // 10500 -> exactly 3000 tokens + assert.equal(deriveNumCtx([sys(chars)]), undefined) +}) + +test('numCtx: one character past the trigger steps onto the ladder', () => { + const chars = NUM_CTX_TRIGGER_TOKENS * PROMPT_CHARS_PER_TOKEN + 1 + assert.equal(deriveNumCtx([sys(chars)]), 8192) +}) + +test('numCtx: climbs the ladder as the system prompt grows', () => { + // ~7000 tokens of system prompt + 2048 headroom = 9048 -> next rung is 16384. + assert.equal(deriveNumCtx([sys(7000 * PROMPT_CHARS_PER_TOKEN)]), 16384) +}) + +test('numCtx: saturates at the top of the ladder rather than failing', () => { + assert.equal(deriveNumCtx([sys(1_000_000)]), 65536) +}) + +test('numCtx: only system messages count toward the budget', () => { + const messages = [sys(100), { role: 'user' as const, content: 'y'.repeat(1_000_000) }] + assert.equal(deriveNumCtx(messages), undefined) +}) + +test('numCtx: system messages are summed, not measured individually', () => { + const half = (NUM_CTX_TRIGGER_TOKENS * PROMPT_CHARS_PER_TOKEN) / 2 + 10 + assert.equal(deriveNumCtx([sys(half), sys(half)]), 8192) +}) diff --git a/admin/types/rag.ts b/admin/types/rag.ts index 86c7bfe..383a93a 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -1,3 +1,5 @@ +import type { OllamaChatMessage } from './ollama.js' + export type EmbedJobWithProgress = { jobId: string fileName: string @@ -42,6 +44,74 @@ export type RerankedRAGResult = Omit & { finalScore: number } +/** One entry in a recorded retrieval stage: just enough to score a ranking. */ +export type StageEntry = { source?: string; score: number } + +/** + * The three ranked lists retrieval produces internally, captured so the eval + * harness can score each stage separately and show whether the heuristic + * reranker and the source-diversity penalty are earning their complexity. + * + * `dense` is the raw cosine ordering from Qdrant, `reranked` adds the + * keyword/heading boosts, `diversified` adds the same-document penalty. + */ +export type RetrievalStages = { + dense?: StageEntry[] + reranked?: StageEntry[] + diversified?: StageEntry[] +} + +/** + * A chunk as returned by `RagService.searchSimilarDocuments` — the shape the + * chat pipeline consumes and the eval harness scores. + */ +export type RetrievedChunk = { + text: string + score: number + metadata?: Record +} + +/** + * Knobs on a single pipeline run. Everything is optional: the defaults + * reproduce production chat exactly. The non-default paths exist so the eval + * harness can ablate one stage at a time without a parallel implementation. + */ +export type PipelineOptions = { + topK?: number + scoreThreshold?: number + collection?: string + /** Skip the history-aware rewrite (which is an LLM call, and therefore + * non-deterministic). Retrieval then runs on the raw last user message. */ + skipQueryRewrite?: boolean + /** Bypass retrieval entirely and inject these chunks as the context. Used by + * the `oracle` eval mode to isolate generation quality from retrieval. */ + oracleContext?: RetrievedChunk[] + /** Ignore the user's NOMAD.md. Off in production; on in evals, where a + * developer's personal instructions would silently skew every result. */ + skipNomadMd?: boolean +} + +/** + * Everything the pipeline decided on the way to a prompt. The controller uses + * only `messages` and `numCtx`; the eval harness scores the rest. Returning it + * unconditionally keeps one code path for both. + */ +export type PipelineTrace = { + /** null when retrieval was skipped entirely (empty KB, or no user message). */ + rewrittenQuery: string | null + /** True when the rewrite LLM call actually ran (it is skipped on turn 1). */ + didRewrite: boolean + /** Everything retrieval returned, pre-trim. */ + retrieved: RetrievedChunk[] + /** What actually made it into the prompt, post model-size trim. */ + injected: RetrievedChunk[] + /** The exact payload handed to Ollama. */ + messages: OllamaChatMessage[] + numCtx: number | undefined + contextLimits: { maxResults: number; maxTokens: number } + timings: { rewriteMs: number; retrievalMs: number } +} + export type FileWarning = | { kind: 'zero_chunks'; fileSizeBytes: number } | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number }