diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index cdb775b..65af19b 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -59,6 +59,15 @@ export class EmbedFileJob { async handle(job: Job) { const { filePath, fileName, batchOffset, totalArticles, collection } = job.data as EmbedFileJobParams + // Only the direct KB-upload controller passes `collection` on dispatch; the other + // six dispatch sites (download auto-index, scan/sync, re-embed, local ZIM upload, + // replaced-file reconcile, and this job's own ZIM batch continuation) do not. Fall + // back to whatever the file is already assigned to, so an assignment made *before* + // the file was indexed still reaches the vectors. Resolving it here rather than at + // each dispatch site keeps one source of truth and covers batch continuations too. + const effectiveCollection = + collection ?? (await KbIngestState.findBy('file_path', filePath))?.collection ?? undefined + const isZimBatch = batchOffset !== undefined const batchInfo = isZimBatch ? ` (batch offset: ${batchOffset})` : '' logger.info(`[EmbedFileJob] Starting embedding process for: ${fileName}${batchInfo}`) @@ -138,7 +147,7 @@ export class EmbedFileJob { allowDeletion, batchOffset, onProgress, - collection + effectiveCollection ) if (!result.success) { @@ -192,6 +201,9 @@ export class EmbedFileJob { totalArticles: totalArticles || result.totalArticles, isFinalBatch: false, // Explicitly not final chunksSoFar: chunksSoFarNext, + // Carry the collection across batches, otherwise only batch 1 of a ZIM + // would be tagged and the rest would land uncategorized. + ...(effectiveCollection ? { collection: effectiveCollection } : {}), }) // Calculate progress based on articles processed. @@ -244,7 +256,7 @@ export class EmbedFileJob { // BullMQ's :completed retention (50 jobs) ages out, so the state row is // the only durable record of "this file finished embedding". try { - await KbIngestState.markIndexed(filePath, totalChunks, collection) + await KbIngestState.markIndexed(filePath, totalChunks, effectiveCollection) } catch (stateErr) { logger.warn( `[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`, diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 6c932b9..13f0ec7 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -529,7 +529,8 @@ export class RagService { filepath: string, deleteAfterEmbedding: boolean, batchOffset?: number, - onProgress?: (percent: number) => Promise + onProgress?: (percent: number) => Promise, + collection?: string ): Promise { const zimExtractionService = new ZIMExtractionService() @@ -556,6 +557,9 @@ export class RagService { const result = await this.embedAndStoreText(zimChunk.text, { source: filepath, content_type: 'zim_article', + // Without this the ZIM path writes points with no `collection` at all, so + // getKnowledgeCollections() (which facets on it) never sees them. + ...(collection ? { collection } : {}), // Article-level context article_title: zimChunk.articleTitle, @@ -783,7 +787,7 @@ export class RagService { // Process based on file type // ZIM files are handled specially since they have their own embedding workflow if (fileType === 'zim') { - return await this.processZIMFile(filepath, deleteAfterEmbedding, batchOffset, onProgress) + return await this.processZIMFile(filepath, deleteAfterEmbedding, batchOffset, onProgress, collection) } // Extract text based on file type @@ -1240,11 +1244,13 @@ export class RagService { filter: { must: [{ key: 'source', match: { value: source } }] }, }) - const row = await KbIngestState.query().where('file_path', source).first() - if (row) { - row.collection = collection - await row.save() - } + // The setPayload above only reaches points that already exist, so for a file + // that has not been indexed yet it matches nothing. The row is therefore the + // only durable record of the choice until EmbedFileJob picks it up -- create + // it when absent rather than reporting success and storing the value nowhere. + const row = await KbIngestState.getOrCreate(source) + row.collection = collection + await row.save() return { success: true, message: collection ? `Moved to "${collection}".` : 'Moved to Uncategorized.' } } catch (error) {