fix(KB): stop partial_stall warning firing on atypical ZIMs (link-out/PDF-heavy)

The Stored Files "partial stall" warning compares chunks in Qdrant against an
expected count from the ratio registry. The registry has an empty-pattern
catch-all (100 chunks/MB) that matches any filename, so a ZIM that matches no
specific pattern still gets a size-based estimate. For archives that are mostly
PDFs, images, or link-out stubs (e.g. irp.fas.org military-medicine), byte size
wildly over-predicts embeddable text: a 75 MB ZIM estimates ~7,236 chunks but
produces ~1, tripping a false "ingestion may have stalled" warning that re-embed
can't clear.

The catch-all is fine for rough aggregate disk-cost estimates, but it should not
drive a per-file stall signal. Add an `ignoreCatchAll` option to the ratio
lookup that excludes the empty-pattern row (returning null when only the
fallback would match), and use it in the warnings path so partial_stall only
fires when the registry has a *specific* expectation for the file. Files that
match a real pattern (wikipedia_, devdocs_, ifixit_, ...) are unaffected;
disk-cost/batch estimates keep using the fallback.

Closes #913

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

admin/app/models/kb_ratio_registry.ts

@@ -49,10 +49,22 @@ export default class KbRatioRegistry extends BaseModel {
     return findChunksPerMb(filename, rows)
   }
 
-  /** Estimate total chunks for a file of the given size on disk. */
-  static async estimateChunks(filename: string, fileSizeBytes: number): Promise<number | null> {
+  /**
+   * Estimate total chunks for a file of the given size on disk.
+   *
+   * `ignoreCatchAll` excludes the empty-pattern fallback, returning `null` for
+   * filenames that only the catch-all would match. The partial_stall warning
+   * uses this so it never flags ZIMs the registry can't specifically
+   * characterize (e.g. PDF/link-out-heavy archives whose byte size wildly
+   * over-predicts embeddable chunks). See #913.
+   */
+  static async estimateChunks(
+    filename: string,
+    fileSizeBytes: number,
+    opts: { ignoreCatchAll?: boolean } = {}
+  ): Promise<number | null> {
     const rows = await this.all()
-    return estimateChunkCount(filename, fileSizeBytes, rows)
+    return estimateChunkCount(filename, fileSizeBytes, rows, opts)
   }
 
   /**

admin/app/services/rag_service.ts

@@ -1242,9 +1242,14 @@ export class RagService {
         const fileSizeBytes = sizeByPath.get(source) ?? 0
         const chunksInQdrant = chunksBySource.get(source) ?? 0
         const fileName = source.split(/[/\\]/).pop() ?? source
+        // ignoreCatchAll: the partial_stall warning must only fire when the
+        // registry has a *specific* expectation for this file. The empty-pattern
+        // fallback (100 chunks/MB) over-predicts wildly for atypical ZIMs that
+        // are mostly PDFs/images/link-outs (e.g. military-medicine), producing
+        // false "ingestion stalled" warnings. Suppress Warning B in that case. (#913)
         const expectedChunks =
           fileSizeBytes > 0
-            ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes)
+            ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes, { ignoreCatchAll: true })
             : null
 
         const warnings = decideWarnings({ fileSizeBytes, chunksInQdrant, expectedChunks })

admin/app/utils/kb_ratio_lookup.ts

@@ -66,10 +66,21 @@ export function estimateBatch(
  *
  * Returns `null` if no row matches and no empty-string fallback is present —
  * caller decides whether to surface "unknown" or use its own default.
+ *
+ * `ignoreCatchAll` excludes the empty-string catch-all row from matching, so a
+ * filename that only the fallback would have matched returns `null` instead.
+ * Callers that need a *specific* expectation (e.g. the partial_stall warning,
+ * which must not fire on atypical ZIMs the registry can't actually characterize)
+ * pass this; rough aggregate estimates (disk cost) leave it off. See #913.
  */
-export function findChunksPerMb(filename: string, rows: RatioRow[]): number | null {
+export function findChunksPerMb(
+  filename: string,
+  rows: RatioRow[],
+  opts: { ignoreCatchAll?: boolean } = {}
+): number | null {
   let best: RatioRow | null = null
   for (const row of rows) {
+    if (opts.ignoreCatchAll && row.pattern === '') continue
     if (!filename.startsWith(row.pattern)) continue
     if (best === null || row.pattern.length > best.pattern.length) {
       best = row
@@ -88,9 +99,10 @@ export function findChunksPerMb(filename: string, rows: RatioRow[]): number | nu
 export function estimateChunkCount(
   filename: string,
   fileSizeBytes: number,
-  rows: RatioRow[]
+  rows: RatioRow[],
+  opts: { ignoreCatchAll?: boolean } = {}
 ): number | null {
-  const ratio = findChunksPerMb(filename, rows)
+  const ratio = findChunksPerMb(filename, rows, opts)
   if (ratio === null) return null
   const megabytes = fileSizeBytes / (1024 * 1024)
   return Math.round(ratio * megabytes)

admin/tests/unit/kb_ratio_lookup.spec.ts

@@ -66,6 +66,34 @@ test('estimateChunkCount returns null when no match and no fallback', () => {
   )
 })
 
+test('ignoreCatchAll: returns null for filenames only the empty-pattern fallback would match (#913)', () => {
+  // military-medicine matches no specific prefix; without ignoreCatchAll it would
+  // hit the 100 chunks/MB fallback and wildly over-predict, falsely tripping
+  // the partial_stall warning on a PDF/link-out-heavy ZIM.
+  assert.equal(
+    findChunksPerMb('irp.fas.org_en_military-medicine_2026-05.zim', SEEDED_ROWS, {
+      ignoreCatchAll: true,
+    }),
+    null
+  )
+  assert.equal(
+    estimateChunkCount('irp.fas.org_en_military-medicine_2026-05.zim', 75 * 1024 * 1024, SEEDED_ROWS, {
+      ignoreCatchAll: true,
+    }),
+    null
+  )
+})
+
+test('ignoreCatchAll: still honors a specific (non-empty) pattern match', () => {
+  // A file that matches a real prefix should be unaffected by ignoreCatchAll.
+  assert.equal(
+    findChunksPerMb('wikipedia_en_simple_all_nopic_2026-02.zim', SEEDED_ROWS, {
+      ignoreCatchAll: true,
+    }),
+    270
+  )
+})
+
 test('estimateBatch sums chunks and bytes for matched files', () => {
   const files = [
     // 100 MB devdocs -> 110,000 chunks