diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 426608a..5a3e682 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -7,6 +7,7 @@ import { OllamaService } from '#services/ollama_service' import { createHash } from 'crypto' import logger from '@adonisjs/core/services/logger' import fs from 'node:fs/promises' +import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' export interface EmbedFileJobParams { filePath: string @@ -93,9 +94,25 @@ export class EmbedFileJob { logger.info(`[EmbedFileJob] Processing file: ${filePath}`) - // Progress callback: maps service-reported 0-100% into the 5-95% job range + // Progress callback. For multi-batch ZIM ingestions, scale the service-reported + // 0-100% (which is % through the current batch's chunks) into the overall-file + // frame so the UI gauge climbs monotonically across the many continuation jobs + // BullMQ creates per file. Without this, every new continuation jobId resets the + // gauge to ~5% and the user sees ingestion progress "jumping around" between + // each batch's local frame and the end-of-batch overall-file overwrite below. + // + // For single-batch files (uploaded PDFs, txts) totalArticles is undefined and + // we fall back to the original 5-95% per-job range, which is what the UI expects + // for a one-shot file with no continuations. const onProgress = async (percent: number) => { - await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9))) + const useOverallFrame = totalArticles && totalArticles > 0 + if (useOverallFrame) { + const articlesDone = (batchOffset || 0) + (percent / 100) * ZIM_BATCH_SIZE + const overallPercent = Math.min(99, Math.round((articlesDone / totalArticles) * 100)) + await this.safeUpdateProgress(job, overallPercent) + } else { + await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9))) + } } // Process and embed the file diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index bd5371d..2a31bd1 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -500,13 +500,13 @@ export class RagService { `[RAG] Extracting ZIM content (batch: offset=${startOffset}, size=${ZIM_BATCH_SIZE})` ) - const zimChunks = await zimExtractionService.extractZIMContent(filepath, { - startOffset, - batchSize: ZIM_BATCH_SIZE, - }) + const { chunks: zimChunks, totalArticles } = await zimExtractionService.extractZIMContent( + filepath, + { startOffset, batchSize: ZIM_BATCH_SIZE } + ) logger.info( - `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata` + `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata (file totalArticles=${totalArticles})` ) // Process each chunk individually with its metadata @@ -582,6 +582,7 @@ export class RagService { chunks: totalChunks, hasMoreBatches, articlesProcessed: articlesInBatch, + totalArticles, } } diff --git a/admin/app/services/zim_extraction_service.ts b/admin/app/services/zim_extraction_service.ts index c48b72d..196add9 100644 --- a/admin/app/services/zim_extraction_service.ts +++ b/admin/app/services/zim_extraction_service.ts @@ -40,7 +40,10 @@ export class ZIMExtractionService { * @param filePath - Path to the ZIM file * @param opts - Options including maxArticles, strategy, onProgress, startOffset, and batchSize */ - async extractZIMContent(filePath: string, opts: ExtractZIMContentOptions = {}): Promise { + async extractZIMContent( + filePath: string, + opts: ExtractZIMContentOptions = {} + ): Promise<{ chunks: ZIMContentChunk[]; totalArticles: number }> { try { logger.info(`[ZIMExtractionService]: Processing ZIM file at path: ${filePath}`) @@ -161,7 +164,7 @@ export class ZIMExtractionService { textPreview: c.text.substring(0, 100) }))) logger.debug("Total structured sections extracted:", toReturn.length) - return toReturn + return { chunks: toReturn, totalArticles: archive.articleCount } } catch (error) { logger.error('Error processing ZIM file:', error) throw error