From 5571e9cd1e9c6019368b2e09161e83805b885abc Mon Sep 17 00:00:00 2001 From: John Cortright Date: Sat, 18 Jul 2026 15:18:39 -0700 Subject: [PATCH] feat(rag): add explicit opt-in for ZIM files in the knowledge base ZIM/Kiwix files were previously auto-discovered and embedded into the AI knowledge base as soon as they existed on disk, with no way to exclude one without deleting it entirely. This made browsing content in Kiwix and including it in AI search the same decision, which isn't what users expect or consented to. - Stop auto-discovering ZIM_STORAGE_PATH during KB sync; ZIM files must be explicitly added via the Content Manager's new 'Add to Knowledge Base' action (embedSingleFile validates directly against disk instead). - Add a per-file active/inactive toggle (kb_ingest_state.active) so a file already indexed can be excluded from/re-included in search instantly, without re-embedding. - searchSimilarDocuments excludes inactive sources via a Qdrant filter. - Hard-deleting an indexed ZIM now also cleans up its kb_ingest_state row and Qdrant points, instead of leaving an orphaned KB entry behind. - Batch the kb_ingest_state lookup in ZimService.list() into a single query instead of one per file. --- admin/app/controllers/rag_controller.ts | 13 +++- admin/app/models/kb_ingest_state.ts | 11 ++++ admin/app/services/rag_service.ts | 44 ++++++++++--- admin/app/services/zim_service.ts | 57 ++++++++++++----- admin/app/validators/rag.ts | 7 +++ ...300872597_add_active_to_kb_ingest_state.ts | 17 +++++ admin/inertia/lib/api.ts | 7 +++ admin/inertia/pages/settings/zim/index.tsx | 62 +++++++++++++++++++ admin/start/routes.ts | 1 + admin/types/zim.ts | 3 + 10 files changed, 199 insertions(+), 23 deletions(-) create mode 100644 admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 7738d0f..77d7286 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app' import { randomBytes } from 'node:crypto' import { sanitizeFilename } from '../utils/fs.js' import { basename } from 'node:path' -import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag' +import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema, toggleActiveSchema } from '#validators/rag' import logger from '@adonisjs/core/services/logger' @inject() @@ -97,6 +97,17 @@ export default class RagController { return response.status(202).json({ message: result.message }) } + public async toggleActive({ request, response }: HttpContext) { + const { source, active } = await request.validateUsing(toggleActiveSchema) + const found = await this.ragService.toggleFileActive(source, active) + if (!found) { + return response.status(404).json({ error: 'File is not a tracked knowledge-base source.' }) + } + return response.status(200).json({ + message: active ? 'File re-included in search.' : 'File excluded from search.', + }) + } + public async getFailedJobs({ response }: HttpContext) { const jobs = await EmbedFileJob.listFailedJobs() return response.status(200).json(jobs) diff --git a/admin/app/models/kb_ingest_state.ts b/admin/app/models/kb_ingest_state.ts index 2add407..642599b 100644 --- a/admin/app/models/kb_ingest_state.ts +++ b/admin/app/models/kb_ingest_state.ts @@ -31,6 +31,9 @@ export default class KbIngestState extends BaseModel { @column() declare last_error: string | null + @column() + declare active: boolean + @column.dateTime({ autoCreate: true }) declare created_at: DateTime @@ -71,6 +74,14 @@ export default class KbIngestState extends BaseModel { await row.save() } + static async setActive(filePath: string, active: boolean): Promise { + const row = await this.query().where('file_path', filePath).first() + if (!row) return null + row.active = active + await row.save() + return row + } + static async remove(filePath: string): Promise { await this.query().where('file_path', filePath).delete() } diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 29b65cf..2842566 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -868,11 +868,21 @@ export class RagService { `[RAG] Searching for top ${searchLimit} semantic matches (threshold: ${scoreThreshold})` ) + // Exclude sources the user has explicitly toggled inactive (e.g. a ZIM + // added to the KB but temporarily excluded from search) without + // touching their underlying embeddings -- see KbIngestState.active. + const inactiveSources = ( + await KbIngestState.query().where('active', false).select('file_path') + ).map((row) => row.file_path) + const searchResults = await this.qdrant!.search(RagService.CONTENT_COLLECTION_NAME, { vector: response.embeddings[0], limit: searchLimit, score_threshold: scoreThreshold, with_payload: true, + ...(inactiveSources.length > 0 + ? { filter: { must_not: [{ key: 'source', match: { any: inactiveSources } }] } } + : {}), }) logger.debug(`[RAG] Found ${searchResults.length} results above threshold ${scoreThreshold}`) @@ -1530,19 +1540,21 @@ export class RagService { } /** - * Walk kb_uploads and zim storage directories, returning the full path of - * every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are - * filtered out so they aren't dispatched only to fail with "Unsupported file - * type" and retry on every sync. + * Walk the kb_uploads storage directory, returning the full path of every + * embeddable file. ZIM_STORAGE_PATH is intentionally NOT walked here -- ZIM + * / Kiwix content must be explicitly opted into the knowledge base via + * embedSingleFile() (e.g. the Content Manager's "Add to Knowledge Base" + * action) rather than being auto-discovered by Sync Storage / Always-policy + * ingestion. Non-embeddable types (e.g. kiwix-library.xml) are filtered out + * so they aren't dispatched only to fail with "Unsupported file type" and + * retry on every sync. */ private async _discoverKbFiles(): Promise { const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) - const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) const filesInStorage: string[] = [] for (const [label, dirPath] of [ [RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const, - [ZIM_STORAGE_PATH, ZIM_PATH] as const, ]) { try { const contents = await listDirectoryContentsRecursive(dirPath) @@ -1623,8 +1635,13 @@ export class RagService { ): Promise { const stateRow = await KbIngestState.query().where('file_path', source).first() if (!stateRow) { - const knownFiles = await this._discoverKbFiles() - if (!knownFiles.includes(source)) { + // Deliberate opt-in path (e.g. "Add to Knowledge Base" on a ZIM file): + // validate directly against disk rather than via _discoverKbFiles(), + // which intentionally excludes ZIM_STORAGE_PATH from automatic + // discovery/sync. A file can still be explicitly added even though it + // won't be picked up by Sync Storage. + const stats = await getFileStatsIfExists(source) + if (!stats || determineFileType(source) === 'unknown') { return { success: false, code: 'not_found', @@ -1672,6 +1689,17 @@ export class RagService { } } + /** + * Flip whether a tracked knowledge-base source is eligible for RAG search, + * without touching its underlying Qdrant points or re-embedding anything. + * Returns null if there's no kb_ingest_state row for this source (nothing + * to toggle -- it was never indexed). + */ + public async toggleFileActive(source: string, active: boolean): Promise { + const row = await KbIngestState.setActive(source, active) + return row !== null + } + /** * Delete all Qdrant points whose `source` payload matches the given path. * Unlike deleteFileBySource(), this does NOT touch the file on disk — used diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index c305939..704b4f2 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -25,6 +25,7 @@ import vine from '@vinejs/vine' import { wikipediaOptionsFileSchema } from '#validators/curated_collections' import WikipediaSelection from '#models/wikipedia_selection' import InstalledResource from '#models/installed_resource' +import KbIngestState from '#models/kb_ingest_state' import CollectionManifest from '#models/collection_manifest' import { RunDownloadJob } from '#jobs/run_download_job' import { SERVICE_NAMES } from '../../constants/service_names.js' @@ -48,16 +49,34 @@ export class ZimService { const all = await listDirectoryContents(dirPath) const zimEntries = all.filter((item) => item.name.endsWith('.zim')) + // Resolve each entry's file path up front so the kb_ingest_state lookup + // can be a single batched query instead of one query per file (N+1) -- + // matters once there are dozens/hundreds of ZIMs (see #followup-proposal). + const filePaths = zimEntries.map((entry) => + entry.type === 'file' ? entry.key : join(dirPath, entry.name) + ) + const kbStates = filePaths.length + ? await KbIngestState.query().whereIn('file_path', filePaths) + : [] + const kbStateByPath = new Map(kbStates.map((row) => [row.file_path, row])) + const files = await Promise.all( - zimEntries.map(async (entry) => { - const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name) + zimEntries.map(async (entry, i) => { + const filePath = filePaths[i] const stats = await getFileStatsIfExists(filePath) + const kbState = kbStateByPath.get(filePath) + const kb_status: 'not_added' | 'active' | 'inactive' = !kbState + ? 'not_added' + : kbState.active + ? 'active' + : 'inactive' return { ...entry, title: null, summary: null, author: null, size_bytes: stats ? Number(stats.size) : null, + kb_status, } }) ) @@ -552,18 +571,11 @@ export class ZimService { logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error) } - const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama') - if (ollamaUrl) { - try { - const { EmbedFileJob } = await import('#jobs/embed_file_job') - await EmbedFileJob.dispatch({ - fileName: filename, - filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename), - }) - } catch (error) { - logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error) - } - } + // NOTE: uploaded ZIM files are intentionally NOT auto-embedded into the + // knowledge base. ZIM/Kiwix content must be explicitly opted in via + // RagService.embedSingleFile() (e.g. the Content Manager's "Add to + // Knowledge Base" action) so browsing content in Kiwix never implies + // consent to include it in AI search. return { added } } @@ -605,6 +617,23 @@ export class ZimService { logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`) } + // If this ZIM was indexed into the AI knowledge base, clean up its Qdrant + // points + kb_ingest_state row too -- otherwise a hard delete here leaves + // an orphaned "Indexed" entry behind that never gets cleaned up until a + // full Reset & Rebuild. + try { + const kbRow = await KbIngestState.query().where('file_path', fullPath).first() + if (kbRow) { + const appModule = await import('@adonisjs/core/services/app') + const { RagService } = await import('#services/rag_service') + const ragService = await appModule.default.container.make(RagService) + await ragService.deleteFileBySource(fullPath) + logger.info(`[ZimService] Cleaned up knowledge-base entry for deleted ZIM: ${fileName}`) + } + } catch (error) { + logger.error(`[ZimService] Failed to clean up knowledge-base entry for ${fileName}:`, error) + } + // If this file was the active Wikipedia selection, clear the selection try { const selection = await WikipediaSelection.query().first() diff --git a/admin/app/validators/rag.ts b/admin/app/validators/rag.ts index b57fcee..c350c40 100644 --- a/admin/app/validators/rag.ts +++ b/admin/app/validators/rag.ts @@ -19,6 +19,13 @@ export const embedFileSchema = vine.compile( }) ) +export const toggleActiveSchema = vine.compile( + vine.object({ + source: vine.string().minLength(1), + active: vine.boolean(), + }) +) + export const fileSourceSchema = vine.compile( vine.object({ source: vine.string().minLength(1), diff --git a/admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts b/admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts new file mode 100644 index 0000000..329168e --- /dev/null +++ b/admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts @@ -0,0 +1,17 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'kb_ingest_state' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.boolean('active').notNullable().defaultTo(true) + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('active') + }) + } +} diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 5b93a90..6b494c4 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -507,6 +507,13 @@ class API { })() } + async toggleRagFileActive(source: string, active: boolean) { + return catchInternal(async () => { + const response = await this.client.post<{ message: string }>('/rag/files/toggle-active', { source, active }) + return response.data + })() + } + async getKbFileWarnings() { return catchInternal(async () => { const response = await this.client.get('/rag/file-warnings') diff --git a/admin/inertia/pages/settings/zim/index.tsx b/admin/inertia/pages/settings/zim/index.tsx index 922cbfb..e9fe7cf 100644 --- a/admin/inertia/pages/settings/zim/index.tsx +++ b/admin/inertia/pages/settings/zim/index.tsx @@ -11,6 +11,7 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Alert from '~/components/Alert' import { useNotifications } from '~/context/NotificationContext' import ZimUploader from '~/components/ZimUploader' +import Switch from '~/components/inputs/Switch' import { ZimFileWithMetadata } from '../../../../types/zim' import { SERVICE_NAMES } from '../../../../constants/service_names' import { formatBytes } from '~/lib/util' @@ -126,6 +127,34 @@ export default function ZimPage() { }, }) + const embedMutation = useMutation({ + mutationFn: async (file: ZimFileWithMetadata) => + api.embedSingleRAGFile(file.type === 'file' ? file.key : file.name), + onSuccess: (result) => { + if (!result) return + queryClient.invalidateQueries({ queryKey: ['zim-files'] }) + addNotification({ + type: 'success', + message: result.message || 'Added to the knowledge base. It will be searchable once embedding finishes.', + }) + }, + }) + + const toggleActiveMutation = useMutation({ + mutationFn: async ({ file, active }: { file: ZimFileWithMetadata; active: boolean }) => + api.toggleRagFileActive(file.type === 'file' ? file.key : file.name, active), + onSuccess: (result, { active }) => { + if (!result) return + queryClient.invalidateQueries({ queryKey: ['zim-files'] }) + addNotification({ + type: 'success', + message: active + ? 'Re-included in AI search.' + : 'Excluded from AI search. It stays indexed, so re-enabling is instant.', + }) + }, + }) + return ( @@ -223,6 +252,39 @@ export default function ZimPage() { ), }, + { + accessor: 'kb_status', + title: 'Knowledge Base', + render: (record) => { + if (record.kb_status === 'not_added') { + const pending = + embedMutation.isPending && embedMutation.variables?.name === record.name + return ( + embedMutation.mutate(record)} + > + Add to Knowledge Base + + ) + } + const isActive = record.kb_status === 'active' + const pending = + toggleActiveMutation.isPending && + toggleActiveMutation.variables?.file.name === record.name + return ( + toggleActiveMutation.mutate({ file: record, active: checked })} + disabled={pending} + label={isActive ? 'Active' : 'Inactive'} + /> + ) + }, + }, { accessor: 'actions', title: 'Actions', diff --git a/admin/start/routes.ts b/admin/start/routes.ts index a9278a0..7b17f4f 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -149,6 +149,7 @@ router router.get('/file-warnings', [RagController, 'getFileWarnings']) router.delete('/files', [RagController, 'deleteFile']) router.post('/files/embed', [RagController, 'embedFile']) + router.post('/files/toggle-active', [RagController, 'toggleActive']) router.get('/files/content', [RagController, 'getFileContent']) router.get('/files/download', [RagController, 'downloadFile']) router.get('/active-jobs', [RagController, 'getActiveJobs']) diff --git a/admin/types/zim.ts b/admin/types/zim.ts index eb6e2b4..a3e3d9f 100644 --- a/admin/types/zim.ts +++ b/admin/types/zim.ts @@ -1,10 +1,13 @@ import { FileEntry } from './files.js' +export type ZimKbStatus = 'not_added' | 'active' | 'inactive' + export type ZimFileWithMetadata = FileEntry & { title: string | null summary: string | null author: string | null size_bytes: number | null + kb_status: ZimKbStatus } export type ListZimFilesResponse = {