diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 7738d0f..77d7286 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app' import { randomBytes } from 'node:crypto' import { sanitizeFilename } from '../utils/fs.js' import { basename } from 'node:path' -import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag' +import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema, toggleActiveSchema } from '#validators/rag' import logger from '@adonisjs/core/services/logger' @inject() @@ -97,6 +97,17 @@ export default class RagController { return response.status(202).json({ message: result.message }) } + public async toggleActive({ request, response }: HttpContext) { + const { source, active } = await request.validateUsing(toggleActiveSchema) + const found = await this.ragService.toggleFileActive(source, active) + if (!found) { + return response.status(404).json({ error: 'File is not a tracked knowledge-base source.' }) + } + return response.status(200).json({ + message: active ? 'File re-included in search.' : 'File excluded from search.', + }) + } + public async getFailedJobs({ response }: HttpContext) { const jobs = await EmbedFileJob.listFailedJobs() return response.status(200).json(jobs) diff --git a/admin/app/models/kb_ingest_state.ts b/admin/app/models/kb_ingest_state.ts index 2add407..642599b 100644 --- a/admin/app/models/kb_ingest_state.ts +++ b/admin/app/models/kb_ingest_state.ts @@ -31,6 +31,9 @@ export default class KbIngestState extends BaseModel { @column() declare last_error: string | null + @column() + declare active: boolean + @column.dateTime({ autoCreate: true }) declare created_at: DateTime @@ -71,6 +74,14 @@ export default class KbIngestState extends BaseModel { await row.save() } + static async setActive(filePath: string, active: boolean): Promise { + const row = await this.query().where('file_path', filePath).first() + if (!row) return null + row.active = active + await row.save() + return row + } + static async remove(filePath: string): Promise { await this.query().where('file_path', filePath).delete() } diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 29b65cf..2842566 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -868,11 +868,21 @@ export class RagService { `[RAG] Searching for top ${searchLimit} semantic matches (threshold: ${scoreThreshold})` ) + // Exclude sources the user has explicitly toggled inactive (e.g. a ZIM + // added to the KB but temporarily excluded from search) without + // touching their underlying embeddings -- see KbIngestState.active. + const inactiveSources = ( + await KbIngestState.query().where('active', false).select('file_path') + ).map((row) => row.file_path) + const searchResults = await this.qdrant!.search(RagService.CONTENT_COLLECTION_NAME, { vector: response.embeddings[0], limit: searchLimit, score_threshold: scoreThreshold, with_payload: true, + ...(inactiveSources.length > 0 + ? { filter: { must_not: [{ key: 'source', match: { any: inactiveSources } }] } } + : {}), }) logger.debug(`[RAG] Found ${searchResults.length} results above threshold ${scoreThreshold}`) @@ -1530,19 +1540,21 @@ export class RagService { } /** - * Walk kb_uploads and zim storage directories, returning the full path of - * every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are - * filtered out so they aren't dispatched only to fail with "Unsupported file - * type" and retry on every sync. + * Walk the kb_uploads storage directory, returning the full path of every + * embeddable file. ZIM_STORAGE_PATH is intentionally NOT walked here -- ZIM + * / Kiwix content must be explicitly opted into the knowledge base via + * embedSingleFile() (e.g. the Content Manager's "Add to Knowledge Base" + * action) rather than being auto-discovered by Sync Storage / Always-policy + * ingestion. Non-embeddable types (e.g. kiwix-library.xml) are filtered out + * so they aren't dispatched only to fail with "Unsupported file type" and + * retry on every sync. */ private async _discoverKbFiles(): Promise { const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) - const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) const filesInStorage: string[] = [] for (const [label, dirPath] of [ [RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const, - [ZIM_STORAGE_PATH, ZIM_PATH] as const, ]) { try { const contents = await listDirectoryContentsRecursive(dirPath) @@ -1623,8 +1635,13 @@ export class RagService { ): Promise { const stateRow = await KbIngestState.query().where('file_path', source).first() if (!stateRow) { - const knownFiles = await this._discoverKbFiles() - if (!knownFiles.includes(source)) { + // Deliberate opt-in path (e.g. "Add to Knowledge Base" on a ZIM file): + // validate directly against disk rather than via _discoverKbFiles(), + // which intentionally excludes ZIM_STORAGE_PATH from automatic + // discovery/sync. A file can still be explicitly added even though it + // won't be picked up by Sync Storage. + const stats = await getFileStatsIfExists(source) + if (!stats || determineFileType(source) === 'unknown') { return { success: false, code: 'not_found', @@ -1672,6 +1689,17 @@ export class RagService { } } + /** + * Flip whether a tracked knowledge-base source is eligible for RAG search, + * without touching its underlying Qdrant points or re-embedding anything. + * Returns null if there's no kb_ingest_state row for this source (nothing + * to toggle -- it was never indexed). + */ + public async toggleFileActive(source: string, active: boolean): Promise { + const row = await KbIngestState.setActive(source, active) + return row !== null + } + /** * Delete all Qdrant points whose `source` payload matches the given path. * Unlike deleteFileBySource(), this does NOT touch the file on disk — used diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index c305939..704b4f2 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -25,6 +25,7 @@ import vine from '@vinejs/vine' import { wikipediaOptionsFileSchema } from '#validators/curated_collections' import WikipediaSelection from '#models/wikipedia_selection' import InstalledResource from '#models/installed_resource' +import KbIngestState from '#models/kb_ingest_state' import CollectionManifest from '#models/collection_manifest' import { RunDownloadJob } from '#jobs/run_download_job' import { SERVICE_NAMES } from '../../constants/service_names.js' @@ -48,16 +49,34 @@ export class ZimService { const all = await listDirectoryContents(dirPath) const zimEntries = all.filter((item) => item.name.endsWith('.zim')) + // Resolve each entry's file path up front so the kb_ingest_state lookup + // can be a single batched query instead of one query per file (N+1) -- + // matters once there are dozens/hundreds of ZIMs (see #followup-proposal). + const filePaths = zimEntries.map((entry) => + entry.type === 'file' ? entry.key : join(dirPath, entry.name) + ) + const kbStates = filePaths.length + ? await KbIngestState.query().whereIn('file_path', filePaths) + : [] + const kbStateByPath = new Map(kbStates.map((row) => [row.file_path, row])) + const files = await Promise.all( - zimEntries.map(async (entry) => { - const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name) + zimEntries.map(async (entry, i) => { + const filePath = filePaths[i] const stats = await getFileStatsIfExists(filePath) + const kbState = kbStateByPath.get(filePath) + const kb_status: 'not_added' | 'active' | 'inactive' = !kbState + ? 'not_added' + : kbState.active + ? 'active' + : 'inactive' return { ...entry, title: null, summary: null, author: null, size_bytes: stats ? Number(stats.size) : null, + kb_status, } }) ) @@ -552,18 +571,11 @@ export class ZimService { logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error) } - const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama') - if (ollamaUrl) { - try { - const { EmbedFileJob } = await import('#jobs/embed_file_job') - await EmbedFileJob.dispatch({ - fileName: filename, - filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename), - }) - } catch (error) { - logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error) - } - } + // NOTE: uploaded ZIM files are intentionally NOT auto-embedded into the + // knowledge base. ZIM/Kiwix content must be explicitly opted in via + // RagService.embedSingleFile() (e.g. the Content Manager's "Add to + // Knowledge Base" action) so browsing content in Kiwix never implies + // consent to include it in AI search. return { added } } @@ -605,6 +617,23 @@ export class ZimService { logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`) } + // If this ZIM was indexed into the AI knowledge base, clean up its Qdrant + // points + kb_ingest_state row too -- otherwise a hard delete here leaves + // an orphaned "Indexed" entry behind that never gets cleaned up until a + // full Reset & Rebuild. + try { + const kbRow = await KbIngestState.query().where('file_path', fullPath).first() + if (kbRow) { + const appModule = await import('@adonisjs/core/services/app') + const { RagService } = await import('#services/rag_service') + const ragService = await appModule.default.container.make(RagService) + await ragService.deleteFileBySource(fullPath) + logger.info(`[ZimService] Cleaned up knowledge-base entry for deleted ZIM: ${fileName}`) + } + } catch (error) { + logger.error(`[ZimService] Failed to clean up knowledge-base entry for ${fileName}:`, error) + } + // If this file was the active Wikipedia selection, clear the selection try { const selection = await WikipediaSelection.query().first() diff --git a/admin/app/validators/rag.ts b/admin/app/validators/rag.ts index b57fcee..c350c40 100644 --- a/admin/app/validators/rag.ts +++ b/admin/app/validators/rag.ts @@ -19,6 +19,13 @@ export const embedFileSchema = vine.compile( }) ) +export const toggleActiveSchema = vine.compile( + vine.object({ + source: vine.string().minLength(1), + active: vine.boolean(), + }) +) + export const fileSourceSchema = vine.compile( vine.object({ source: vine.string().minLength(1), diff --git a/admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts b/admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts new file mode 100644 index 0000000..329168e --- /dev/null +++ b/admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts @@ -0,0 +1,17 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'kb_ingest_state' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.boolean('active').notNullable().defaultTo(true) + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('active') + }) + } +} diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 5b93a90..6b494c4 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -507,6 +507,13 @@ class API { })() } + async toggleRagFileActive(source: string, active: boolean) { + return catchInternal(async () => { + const response = await this.client.post<{ message: string }>('/rag/files/toggle-active', { source, active }) + return response.data + })() + } + async getKbFileWarnings() { return catchInternal(async () => { const response = await this.client.get('/rag/file-warnings') diff --git a/admin/inertia/pages/settings/zim/index.tsx b/admin/inertia/pages/settings/zim/index.tsx index 922cbfb..e9fe7cf 100644 --- a/admin/inertia/pages/settings/zim/index.tsx +++ b/admin/inertia/pages/settings/zim/index.tsx @@ -11,6 +11,7 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus' import Alert from '~/components/Alert' import { useNotifications } from '~/context/NotificationContext' import ZimUploader from '~/components/ZimUploader' +import Switch from '~/components/inputs/Switch' import { ZimFileWithMetadata } from '../../../../types/zim' import { SERVICE_NAMES } from '../../../../constants/service_names' import { formatBytes } from '~/lib/util' @@ -126,6 +127,34 @@ export default function ZimPage() { }, }) + const embedMutation = useMutation({ + mutationFn: async (file: ZimFileWithMetadata) => + api.embedSingleRAGFile(file.type === 'file' ? file.key : file.name), + onSuccess: (result) => { + if (!result) return + queryClient.invalidateQueries({ queryKey: ['zim-files'] }) + addNotification({ + type: 'success', + message: result.message || 'Added to the knowledge base. It will be searchable once embedding finishes.', + }) + }, + }) + + const toggleActiveMutation = useMutation({ + mutationFn: async ({ file, active }: { file: ZimFileWithMetadata; active: boolean }) => + api.toggleRagFileActive(file.type === 'file' ? file.key : file.name, active), + onSuccess: (result, { active }) => { + if (!result) return + queryClient.invalidateQueries({ queryKey: ['zim-files'] }) + addNotification({ + type: 'success', + message: active + ? 'Re-included in AI search.' + : 'Excluded from AI search. It stays indexed, so re-enabling is instant.', + }) + }, + }) + return ( @@ -223,6 +252,39 @@ export default function ZimPage() { ), }, + { + accessor: 'kb_status', + title: 'Knowledge Base', + render: (record) => { + if (record.kb_status === 'not_added') { + const pending = + embedMutation.isPending && embedMutation.variables?.name === record.name + return ( + embedMutation.mutate(record)} + > + Add to Knowledge Base + + ) + } + const isActive = record.kb_status === 'active' + const pending = + toggleActiveMutation.isPending && + toggleActiveMutation.variables?.file.name === record.name + return ( + toggleActiveMutation.mutate({ file: record, active: checked })} + disabled={pending} + label={isActive ? 'Active' : 'Inactive'} + /> + ) + }, + }, { accessor: 'actions', title: 'Actions', diff --git a/admin/start/routes.ts b/admin/start/routes.ts index a9278a0..7b17f4f 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -149,6 +149,7 @@ router router.get('/file-warnings', [RagController, 'getFileWarnings']) router.delete('/files', [RagController, 'deleteFile']) router.post('/files/embed', [RagController, 'embedFile']) + router.post('/files/toggle-active', [RagController, 'toggleActive']) router.get('/files/content', [RagController, 'getFileContent']) router.get('/files/download', [RagController, 'downloadFile']) router.get('/active-jobs', [RagController, 'getActiveJobs']) diff --git a/admin/types/zim.ts b/admin/types/zim.ts index eb6e2b4..a3e3d9f 100644 --- a/admin/types/zim.ts +++ b/admin/types/zim.ts @@ -1,10 +1,13 @@ import { FileEntry } from './files.js' +export type ZimKbStatus = 'not_added' | 'active' | 'inactive' + export type ZimFileWithMetadata = FileEntry & { title: string | null summary: string | null author: string | null size_bytes: number | null + kb_status: ZimKbStatus } export type ListZimFilesResponse = {