From b8961d27dd56cd7e10464699c0ab8067107e95fc Mon Sep 17 00:00:00 2001 From: jakeaturner Date: Mon, 1 Jun 2026 17:34:32 +0000 Subject: [PATCH] fix(rag): improve context-reliance hedging and use heading metadata at query time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Users often saw the assistant disclaim ("Sorry, I couldn't find specific context regarding X, but here's a general answer...") even when material that directly answered the query was embedded. Two compounding causes, both on the read/generation side: 1. The rag_context system prompt explicitly authorized the hedge: it told the model to fall back to general knowledge and "acknowledge the limitations," and mandated "According to the information available..." citations that pushed small models into meta-commentary preambles. Rewrote it to treat the retrieved context as the primary, authoritative source, lead with the answer, fall back to general knowledge *silently*, and never emit "couldn't find specific context" phrasing, while still allowing the model itself to make some "sanity-check" decisions if the retrieved context seems wildly incorrect or unrelated to the user's query. 2. The injected context labeled each block with a raw relevance score (e.g. "Relevance: 42.3%"). nomic-embed cosine scores for genuinely relevant passages sit ~0.4-0.6, so the number primed the model to distrust correct context. Dropped the model-visible score (it stays in the logs) and replaced it with a neutral source-title label. Also added a conservative, score-scaled heading boost in rerankResults: when query keywords match a chunk's section/article title (ZIM metadata we already fetch), nudge its rank. Same diminishing-returns shape as the existing boosts and gated behind the existing semantic-quality threshold, so it can't promote a weak match. Scope note: this removes the visible hedge and improves ranking of already-retrieved chunks. It does NOT fix retrieval recall — diluted 1500-token chunks that never reach dense search's top-k are unaffected. That's a follow-up (smaller chunks + BM25/RRF hybrid). --- admin/app/controllers/ollama_controller.ts | 11 +++++++++- admin/app/services/rag_service.ts | 23 ++++++++++++++++++++ admin/constants/ollama.ts | 25 +++++++++++++++------- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 3bf9e73..9174616 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -106,8 +106,17 @@ export default class OllamaController { `[RAG] Injecting ${trimmedDocs.length}/${relevantDocs.length} results (model: ${reqData.model}, maxResults: ${maxResults}, maxTokens: ${maxTokens || 'unlimited'})` ) + // Label each context block with its source title when available (a neutral, + // honest provenance signal) but never the raw relevance score — nomic cosine + // scores for genuinely relevant passages sit ~0.4-0.6, and surfacing e.g. + // "42%" primes the model to distrust correct context. Scores stay in the logs + // above for debugging. const contextText = trimmedDocs - .map((doc, idx) => `[Context ${idx + 1}] (Relevance: ${(doc.score * 100).toFixed(1)}%)\n${doc.text}`) + .map((doc, idx) => { + const title = doc.metadata?.full_title || doc.metadata?.article_title + const label = title ? `[Context ${idx + 1} — ${title}]` : `[Context ${idx + 1}]` + return `${label}\n${doc.text}` + }) .join('\n\n') const systemMessage = { diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 2a2110b..203913a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -1011,6 +1011,29 @@ export class RagService { } } + // Boost when query keywords match the chunk's section/article heading. ZIM + // content carries this structural metadata (already fetched, no extra cost), + // and a query term appearing in a heading is a strong relevance signal that + // body-text matching alone misses. Same conservative, score-scaled, diminishing + // -returns shape as the boosts above, so it can't promote a weak match. + const headingText = [result.full_title, result.section_title, result.article_title] + .filter(Boolean) + .join(' ') + .toLowerCase() + if (headingText) { + const headingHits = queryKeywords.filter((kw) => + headingText.includes(kw.toLowerCase()) + ).length + if (headingHits > 0) { + const headingRatio = headingHits / Math.max(queryKeywords.length, 1) + const headingBoost = Math.sqrt(headingRatio) * 0.1 * result.score + logger.debug( + `[RAG] Heading match: ${headingHits}/${queryKeywords.length} - Boost: ${headingBoost.toFixed(3)}` + ) + finalScore += headingBoost + } + } + finalScore = Math.min(1.0, finalScore + keywordBoost) return { diff --git a/admin/constants/ollama.ts b/admin/constants/ollama.ts index 23df8f0..17268d8 100644 --- a/admin/constants/ollama.ts +++ b/admin/constants/ollama.ts @@ -86,18 +86,27 @@ export const SYSTEM_PROMPTS = { - Use tables when presenting structured data. `, rag_context: (context: string) => ` -You have access to relevant information from the knowledge base. This context has been retrieved based on semantic similarity to the user's question. +Information has been retrieved from the NOMAD knowledge base that MAY be relevant to the +user's question. It was selected by automated similarity search, which is imperfect — some +or all of it may be unrelated to what the user actually asked. [Knowledge Base Context] ${context} -IMPORTANT INSTRUCTIONS: -1. If the user's question is directly related to the context above, use this information to provide accurate, detailed answers. -2. Always cite or reference the context when using it (e.g., "According to the information available..." or "Based on the knowledge base..."). -3. If the context is only partially relevant, combine it with your general knowledge but be clear about what comes from the knowledge base. -4. If the context is not relevant to the user's question, you can respond using your general knowledge without forcing the context into your answer. Do not mention the context if it's not relevant. -5. Never fabricate information that isn't in the context or your training data. -6. If you're unsure or you don't have enough information to answer the user's question, acknowledge the limitations. +HOW TO ANSWER: +1. First, silently judge whether the context genuinely addresses the user's question. Use + it ONLY when it really contains relevant information. Do not force a connection that + isn't there: poetic, narrative, tangential, or topically-unrelated passages are NOT + relevant just because they share a word with the question — ignore them. +2. When the context is relevant, base your answer on it and answer directly and specifically. +3. When the context does not actually address the question, ignore it completely and answer + from your own general knowledge. Do this silently — do not mention the knowledge base, + the context, or the fact that it lacked an answer, and do not apologize. +4. Never narrate your retrieval or reasoning process. Do not write "according to Context 1", + "the context is unrelated, but", "I couldn't find specific context", or similar. Just + give the answer as if you simply knew it. +5. Do not fabricate specifics (numbers, names, procedures) that are neither supported by + genuinely relevant context nor part of your reliable knowledge. Format your response using markdown for readability. `,