diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 67e8627..81145f8 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -532,9 +532,12 @@ export class RagService { } } - // Count unique articles processed in this batch + // Count unique articles processed in this batch. hasMoreBatches gates on the article + // count — zimChunks.length counts section-level chunks (multiple per article under the + // 'structured' strategy), so comparing it to ZIM_BATCH_SIZE (an article limit) caps + // processing at the first batch for any real archive. const articlesInBatch = new Set(zimChunks.map((c) => c.documentId)).size - const hasMoreBatches = zimChunks.length === ZIM_BATCH_SIZE + const hasMoreBatches = articlesInBatch >= ZIM_BATCH_SIZE logger.info( `[RAG] Successfully embedded ${totalChunks} total chunks from ${articlesInBatch} articles (hasMore: ${hasMoreBatches})` @@ -1252,8 +1255,12 @@ export class RagService { logger.info(`[RAG] Found ${sourcesInQdrant.size} unique sources in Qdrant`) - // Find files that are in storage but not in Qdrant - const filesToEmbed = filesInStorage.filter((filePath) => !sourcesInQdrant.has(filePath)) + // Find files that are in storage, not already in Qdrant, and have an embeddable type. + // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise be + // dispatched to EmbedFileJob, fail with "Unsupported file type", and retry on every sync. + const filesToEmbed = filesInStorage.filter( + (filePath) => !sourcesInQdrant.has(filePath) && determineFileType(filePath) !== 'unknown' + ) logger.info(`[RAG] Found ${filesToEmbed.length} files that need embedding`) diff --git a/admin/app/services/zim_extraction_service.ts b/admin/app/services/zim_extraction_service.ts index b3594b6..c48b72d 100644 --- a/admin/app/services/zim_extraction_service.ts +++ b/admin/app/services/zim_extraction_service.ts @@ -216,7 +216,10 @@ export class ZIMExtractionService { const sections: Array<{ heading: string; text: string; level: number }> = []; let currentSection = { heading: 'Introduction', content: [] as string[], level: 2 }; - $('body').children().each((_, element) => { + // Walk the full DOM rather than only direct children of . Modern ZIMs (Devdocs, + // Wikipedia, FreeCodeCamp, etc.) wrap article content in a container div, which under + // .children() would be a single non-heading/non-paragraph element and yield zero sections. + $('body').find('h2, h3, h4, p, ul, ol, dl, table').each((_, element) => { const $el = $(element); const tagName = element.tagName?.toLowerCase(); @@ -253,6 +256,20 @@ export class ZIMExtractionService { }); } + // Fallback: if the selector walk produced no sections but the body has meaningful + // text (unusual structure, minimal markup), emit one section with the full body text + // so the article still contributes to the knowledge base. + if (sections.length === 0) { + const bodyText = $('body').text().replace(/\s+/g, ' ').trim(); + if (bodyText.length > 0) { + sections.push({ + heading: title || 'Content', + text: bodyText, + level: 2, + }); + } + } + return { title, sections,