diff --git a/admin/app/models/kb_ratio_registry.ts b/admin/app/models/kb_ratio_registry.ts index 13731b8..1d8e2e3 100644 --- a/admin/app/models/kb_ratio_registry.ts +++ b/admin/app/models/kb_ratio_registry.ts @@ -49,10 +49,22 @@ export default class KbRatioRegistry extends BaseModel { return findChunksPerMb(filename, rows) } - /** Estimate total chunks for a file of the given size on disk. */ - static async estimateChunks(filename: string, fileSizeBytes: number): Promise { + /** + * Estimate total chunks for a file of the given size on disk. + * + * `ignoreCatchAll` excludes the empty-pattern fallback, returning `null` for + * filenames that only the catch-all would match. The partial_stall warning + * uses this so it never flags ZIMs the registry can't specifically + * characterize (e.g. PDF/link-out-heavy archives whose byte size wildly + * over-predicts embeddable chunks). See #913. + */ + static async estimateChunks( + filename: string, + fileSizeBytes: number, + opts: { ignoreCatchAll?: boolean } = {} + ): Promise { const rows = await this.all() - return estimateChunkCount(filename, fileSizeBytes, rows) + return estimateChunkCount(filename, fileSizeBytes, rows, opts) } /** diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 203913a..8c046e8 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -1242,9 +1242,14 @@ export class RagService { const fileSizeBytes = sizeByPath.get(source) ?? 0 const chunksInQdrant = chunksBySource.get(source) ?? 0 const fileName = source.split(/[/\\]/).pop() ?? source + // ignoreCatchAll: the partial_stall warning must only fire when the + // registry has a *specific* expectation for this file. The empty-pattern + // fallback (100 chunks/MB) over-predicts wildly for atypical ZIMs that + // are mostly PDFs/images/link-outs (e.g. military-medicine), producing + // false "ingestion stalled" warnings. Suppress Warning B in that case. (#913) const expectedChunks = fileSizeBytes > 0 - ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes) + ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes, { ignoreCatchAll: true }) : null const warnings = decideWarnings({ fileSizeBytes, chunksInQdrant, expectedChunks }) diff --git a/admin/app/utils/kb_ratio_lookup.ts b/admin/app/utils/kb_ratio_lookup.ts index 1abd5c6..e5e7185 100644 --- a/admin/app/utils/kb_ratio_lookup.ts +++ b/admin/app/utils/kb_ratio_lookup.ts @@ -66,10 +66,21 @@ export function estimateBatch( * * Returns `null` if no row matches and no empty-string fallback is present — * caller decides whether to surface "unknown" or use its own default. + * + * `ignoreCatchAll` excludes the empty-string catch-all row from matching, so a + * filename that only the fallback would have matched returns `null` instead. + * Callers that need a *specific* expectation (e.g. the partial_stall warning, + * which must not fire on atypical ZIMs the registry can't actually characterize) + * pass this; rough aggregate estimates (disk cost) leave it off. See #913. */ -export function findChunksPerMb(filename: string, rows: RatioRow[]): number | null { +export function findChunksPerMb( + filename: string, + rows: RatioRow[], + opts: { ignoreCatchAll?: boolean } = {} +): number | null { let best: RatioRow | null = null for (const row of rows) { + if (opts.ignoreCatchAll && row.pattern === '') continue if (!filename.startsWith(row.pattern)) continue if (best === null || row.pattern.length > best.pattern.length) { best = row @@ -88,9 +99,10 @@ export function findChunksPerMb(filename: string, rows: RatioRow[]): number | nu export function estimateChunkCount( filename: string, fileSizeBytes: number, - rows: RatioRow[] + rows: RatioRow[], + opts: { ignoreCatchAll?: boolean } = {} ): number | null { - const ratio = findChunksPerMb(filename, rows) + const ratio = findChunksPerMb(filename, rows, opts) if (ratio === null) return null const megabytes = fileSizeBytes / (1024 * 1024) return Math.round(ratio * megabytes) diff --git a/admin/tests/unit/kb_ratio_lookup.spec.ts b/admin/tests/unit/kb_ratio_lookup.spec.ts index 330e2a3..2c1ae61 100644 --- a/admin/tests/unit/kb_ratio_lookup.spec.ts +++ b/admin/tests/unit/kb_ratio_lookup.spec.ts @@ -66,6 +66,34 @@ test('estimateChunkCount returns null when no match and no fallback', () => { ) }) +test('ignoreCatchAll: returns null for filenames only the empty-pattern fallback would match (#913)', () => { + // military-medicine matches no specific prefix; without ignoreCatchAll it would + // hit the 100 chunks/MB fallback and wildly over-predict, falsely tripping + // the partial_stall warning on a PDF/link-out-heavy ZIM. + assert.equal( + findChunksPerMb('irp.fas.org_en_military-medicine_2026-05.zim', SEEDED_ROWS, { + ignoreCatchAll: true, + }), + null + ) + assert.equal( + estimateChunkCount('irp.fas.org_en_military-medicine_2026-05.zim', 75 * 1024 * 1024, SEEDED_ROWS, { + ignoreCatchAll: true, + }), + null + ) +}) + +test('ignoreCatchAll: still honors a specific (non-empty) pattern match', () => { + // A file that matches a real prefix should be unaffected by ignoreCatchAll. + assert.equal( + findChunksPerMb('wikipedia_en_simple_all_nopic_2026-02.zim', SEEDED_ROWS, { + ignoreCatchAll: true, + }), + 270 + ) +}) + test('estimateBatch sums chunks and bytes for matched files', () => { const files = [ // 100 MB devdocs -> 110,000 chunks