diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 7ca7b6d..da17101 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app' import { randomBytes } from 'node:crypto' import { sanitizeFilename } from '../utils/fs.js' import { basename } from 'node:path' -import { deleteFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag' +import { deleteFileSchema, embedFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag' import logger from '@adonisjs/core/services/logger' @inject() @@ -82,6 +82,15 @@ export default class RagController { return response.status(200).json({ message: result.message }) } + public async embedFile({ request, response }: HttpContext) { + const { source, force } = await request.validateUsing(embedFileSchema) + const result = await this.ragService.embedSingleFile(source, force ?? false) + if (!result.success) { + return response.status(409).json({ error: result.message }) + } + return response.status(202).json({ message: result.message }) + } + public async getFailedJobs({ response }: HttpContext) { const jobs = await EmbedFileJob.listFailedJobs() return response.status(200).json(jobs) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 21e302f..20a418a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -20,7 +20,8 @@ import KbIngestState from '#models/kb_ingest_state' import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js' import KbRatioRegistry from '#models/kb_ratio_registry' import { decideWarnings } from '../utils/kb_warning_decision.js' -import type { FileWarning, FileWarningsResult } from '../../types/rag.js' +import type { FileWarning, FileWarningsResult, StoredFileInfo } from '../../types/rag.js' +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' @@ -1051,7 +1052,7 @@ export class RagService { } } - public async getStoredFiles(): Promise { + public async getStoredFiles(): Promise { try { await this._ensureCollection( RagService.CONTENT_COLLECTION_NAME, @@ -1090,10 +1091,15 @@ export class RagService { // in particular) have no row to attach to. The state machine is the // authoritative "what's on disk?" view; Qdrant is "what made it into // the vector store?". Both are needed to render the KB UI honestly. + const stateByPath = new Map() try { - const stateRows = await KbIngestState.query().select('file_path') + const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded') for (const row of stateRows) { sources.add(row.file_path) + stateByPath.set(row.file_path, { + state: row.state, + chunks_embedded: row.chunks_embedded, + }) } } catch (error) { // Non-fatal: if the state machine query fails for any reason we'd @@ -1104,7 +1110,14 @@ export class RagService { ) } - return Array.from(sources) + return Array.from(sources).map((source) => { + const row = stateByPath.get(source) + return { + source, + state: row?.state ?? null, + chunksEmbedded: row?.chunks_embedded ?? 0, + } + }) } catch (error) { logger.error('Error retrieving stored files:', error) return [] @@ -1401,6 +1414,70 @@ export class RagService { return { queuedCount, dedupedCount, failedPaths } } + /** + * Dispatch an embed job for a single stored file. Wraps `_dispatchEmbedJobsFor` + * with the safety checks needed for a user-triggered per-row action: + * 1. The source must be known to the scanner OR have a state row — prevents + * arbitrary path dispatch from the public API. + * 2. We refuse if any inflight job (waiting/active/delayed/paused) already + * targets this filePath. Otherwise a double-click or a rapid retry could + * enqueue duplicate jobs, producing duplicate chunks. + * 3. When `force` is true (Re-embed of an already-indexed file), we + * pre-delete the prior Qdrant points so the new run doesn't stack on + * top of the old ones. For force=false (Index of a never-embedded file), + * there's nothing to clear. + */ + public async embedSingleFile( + source: string, + force: boolean = false + ): Promise<{ success: boolean; message: string }> { + const stateRow = await KbIngestState.query().where('file_path', source).first() + if (!stateRow) { + const knownFiles = await this._discoverKbFiles() + if (!knownFiles.includes(source)) { + return { + success: false, + message: 'File is not a tracked knowledge-base source.', + } + } + } + + const { EmbedFileJob } = await import('#jobs/embed_file_job') + const { QueueService } = await import('#services/queue_service') + const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue) + const inflight = await queue.getJobs(['waiting', 'active', 'delayed', 'paused']) + if (inflight.some((j) => j.data?.filePath === source)) { + return { + success: false, + message: 'A job for this file is already in progress. Wait for it to finish before re-queuing.', + } + } + + if (force) { + try { + await this._deletePointsBySource(source) + } catch (err) { + logger.error(`[RAG] Failed to delete prior points for ${source}; aborting re-embed:`, err) + return { + success: false, + message: 'Failed to clear prior embeddings before re-embed.', + } + } + } + + const result = await this._dispatchEmbedJobsFor([source], { force }) + if (result.failedPaths.length > 0) { + return { + success: false, + message: 'Failed to dispatch embed job for this file.', + } + } + return { + success: true, + message: force ? 'Re-embed queued for this file.' : 'Indexing queued for this file.', + } + } + /** * Delete all Qdrant points whose `source` payload matches the given path. * Unlike deleteFileBySource(), this does NOT touch the file on disk — used diff --git a/admin/app/validators/rag.ts b/admin/app/validators/rag.ts index 8326f44..5bef836 100644 --- a/admin/app/validators/rag.ts +++ b/admin/app/validators/rag.ts @@ -12,6 +12,13 @@ export const deleteFileSchema = vine.compile( }) ) +export const embedFileSchema = vine.compile( + vine.object({ + source: vine.string().minLength(1), + force: vine.boolean().optional(), + }) +) + export const estimateBatchSchema = vine.compile( vine.object({ files: vine diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index dadfb3f..301bf67 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -2,6 +2,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { useEffect, useRef, useState } from 'react' import FileUploader from '~/components/file-uploader' import StyledButton from '~/components/StyledButton' +import type { DynamicIconName } from '~/lib/icons' import StyledSectionHeader from '~/components/StyledSectionHeader' import StyledTable from '~/components/StyledTable' import { useNotifications } from '~/context/NotificationContext' @@ -10,6 +11,7 @@ import { groupAndSortKbFiles, type KbFileGroup, } from '~/lib/kb_file_grouping' +import type { KbIngestStateValue } from '../../../types/kb_ingest_state' import { IconX } from '@tabler/icons-react' import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' @@ -21,11 +23,82 @@ interface KnowledgeBaseModalProps { onClose: () => void } +/** + * Compact label for the per-row ingestion state. Files that exist in Qdrant + * with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883 + * installs whose chunks are real, so we display them as "Indexed" rather than + * surfacing the absent-row detail. Admin-docs group has no pill (the "Managed + * by NOMAD" message in the action column carries the same signal). + */ +function renderStatePill(record: KbFileGroup): React.ReactNode { + if (record.bucket === 'admin_docs') return null + const effective: KbIngestStateValue = record.state ?? 'indexed' + + const base = 'inline-flex items-center text-xs font-medium rounded px-2 py-0.5 border' + switch (effective) { + case 'indexed': + return ( + + Indexed + + ) + case 'pending_decision': + case 'browse_only': + return ( + + Not Indexed + + ) + case 'failed': + return ( + + Failed + + ) + case 'stalled': + return ( + + Stalled + + ) + } +} + +type RowAction = + | { kind: 'index'; label: string; force: boolean; variant: 'primary'; icon: DynamicIconName } + | { kind: 'reembed'; label: string; force: true; variant: 'secondary'; icon: DynamicIconName } + +/** + * Pick the single adaptive per-row action button. Returns null when no action + * makes sense for the current state (e.g. healthy indexed file with no + * warnings — bulk Re-embed All covers that case). `hasWarnings` lets us + * surface a Re-embed affordance specifically when a file *looks* indexed but + * has zero chunks or a stalled-mid-ingestion warning attached. + */ +function pickRowAction(record: KbFileGroup, hasWarnings: boolean): RowAction | null { + if (record.bucket === 'admin_docs') return null + const effective: KbIngestStateValue = record.state ?? 'indexed' + switch (effective) { + case 'indexed': + return hasWarnings + ? { kind: 'reembed', label: 'Re-embed', force: true, variant: 'secondary', icon: 'IconRefreshAlert' } + : null + case 'pending_decision': + return { kind: 'index', label: 'Index', force: false, variant: 'primary', icon: 'IconDownload' } + case 'browse_only': + return { kind: 'index', label: 'Index', force: true, variant: 'primary', icon: 'IconDownload' } + case 'failed': + case 'stalled': + return { kind: 'index', label: 'Retry', force: true, variant: 'primary', icon: 'IconRefresh' } + } +} + export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", onClose }: KnowledgeBaseModalProps) { const { addNotification } = useNotifications() const [files, setFiles] = useState([]) const [isUploading, setIsUploading] = useState(false) const [confirmDeleteSource, setConfirmDeleteSource] = useState(null) + const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null) const [bulkMode, setBulkMode] = useState(null) const [resetTyped, setResetTyped] = useState('') const fileUploaderRef = useRef>(null) @@ -111,6 +184,25 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }) + const embedMutation = useMutation({ + mutationFn: ({ source, force }: { source: string; force: boolean }) => + api.embedSingleRAGFile(source, force), + onSuccess: (data) => { + addNotification({ + type: 'success', + message: data?.message || 'File queued for embedding.', + }) + setConfirmReembed(null) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + queryClient.invalidateQueries({ queryKey: ['kbFileWarnings'] }) + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to queue file.' }) + setConfirmReembed(null) + }, + }) + const cleanupFailedMutation = useMutation({ mutationFn: () => api.cleanupFailedEmbedJobs(), onSuccess: (data) => { @@ -466,32 +558,38 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o title: 'File Name', render(record) { const warnings = fileWarnings[record.source] ?? [] + const pill = renderStatePill(record) return ( -
+
{record.displayName} - {warnings.map((w, i) => ( - - - {w.kind === 'zero_chunks' && ( - - Embedded 0 chunks — this file has no text content. - AI Assistant cannot reference it. + {(pill || warnings.length > 0) && ( +
+ {pill} + {warnings.map((w, i) => ( + + + {w.kind === 'zero_chunks' && ( + + Embedded 0 chunks — this file has no text content. + AI Assistant cannot reference it. + + )} + {w.kind === 'partial_stall' && ( + + Only {w.chunksEmbedded.toLocaleString()} of est.{' '} + {w.chunksExpected.toLocaleString()} chunks embedded — + ingestion may have stalled. + + )} - )} - {w.kind === 'partial_stall' && ( - - Only {w.chunksEmbedded.toLocaleString()} of est.{' '} - {w.chunksExpected.toLocaleString()} chunks embedded — - ingestion may have stalled. - - )} - - ))} + ))} +
+ )}
) }, @@ -538,14 +636,38 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
) } + + const warnings = fileWarnings[record.source] ?? [] + const action = pickRowAction(record, warnings.length > 0) + const actionPendingForThisRow = + embedMutation.isPending && embedMutation.variables?.source === record.source + return ( -
+
+ {action && ( + { + if (action.kind === 'reembed') { + setConfirmReembed({ source: record.source, displayName: record.displayName }) + } else { + embedMutation.mutate({ source: record.source, force: action.force }) + } + }} + disabled={qdrantOffline || deleteMutation.isPending || embedMutation.isPending} + loading={actionPendingForThisRow} + > + {action.label} + + )} setConfirmDeleteSource(record.source)} - disabled={deleteMutation.isPending} + disabled={deleteMutation.isPending || embedMutation.isPending} loading={deleteMutation.isPending && confirmDeleteSource === record.source} >Delete
@@ -654,6 +776,37 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
)} + + {confirmReembed && ( + + embedMutation.mutate({ source: confirmReembed.source, force: true }) + } + onCancel={() => setConfirmReembed(null)} + > +
+

+ This will delete the existing embeddings for{' '} + {confirmReembed.displayName} and queue + a fresh embedding job. The file on disk is not touched. +

+
+

Heads up

+
    +
  • For large ZIM archives this can take a long time, especially on CPU-only systems.
    • +
  • Search results that referenced this file will be incomplete until the new embedding finishes.
    • +
  • If a job for this file is already running, the re-embed will be refused — wait for it to finish first.
    • +
+
+
+ + )} ) } diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 3d92995..cd7eba8 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -5,7 +5,7 @@ import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' -import { EmbedJobWithProgress, FileWarningsResult } from '../../types/rag' +import { EmbedJobWithProgress, FileWarningsResult, StoredFileInfo } from '../../types/rag' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' @@ -470,11 +470,18 @@ class API { async getStoredRAGFiles() { return catchInternal(async () => { - const response = await this.client.get<{ files: string[] }>('/rag/files') + const response = await this.client.get<{ files: StoredFileInfo[] }>('/rag/files') return response.data.files })() } + async embedSingleRAGFile(source: string, force: boolean = false) { + return catchInternal(async () => { + const response = await this.client.post<{ message: string }>('/rag/files/embed', { source, force }) + return response.data + })() + } + async getKbFileWarnings() { return catchInternal(async () => { const response = await this.client.get('/rag/file-warnings') diff --git a/admin/inertia/lib/kb_file_grouping.ts b/admin/inertia/lib/kb_file_grouping.ts index 9f474c6..3d2ae04 100644 --- a/admin/inertia/lib/kb_file_grouping.ts +++ b/admin/inertia/lib/kb_file_grouping.ts @@ -1,9 +1,12 @@ +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' +import type { StoredFileInfo } from '../../types/rag.js' + /** - * Knowledge-base files come back as a flat list of source paths from - * `/api/rag/files`. The UI groups them so the user sees the categories that - * matter to them — ZIMs, uploaded documents, and a single rolled-up entry for - * Project NOMAD's bundled docs (rather than the 12+ individual markdown files - * those break into). + * Knowledge-base files come back as a list of `{source, state, chunksEmbedded}` + * objects from `/api/rag/files`. The UI groups them so the user sees the + * categories that matter to them — ZIMs, uploaded documents, and a single + * rolled-up entry for Project NOMAD's bundled docs (rather than the 12+ + * individual markdown files those break into). * * Bucket assignment is purely by path prefix; matching is done on `/` so the * server-emitted absolute paths work regardless of which Linux mount the admin @@ -43,27 +46,33 @@ export interface KbFileGroup { count: number /** All member source paths — populated for collapsed groups, empty otherwise. */ members: string[] + /** Per-file ingestion state. `null` for the collapsed admin_docs group and + * for any source that exists in Qdrant but has no state row yet. */ + state: KbIngestStateValue | null + /** Chunks currently embedded for this source; 0 for state-row-less or + * zero-chunk files. Always 0 for the collapsed admin_docs group. */ + chunksEmbedded: number } const BUCKET_SORT_ORDER: KbFileBucket[] = ['zim', 'upload', 'admin_docs', 'other'] /** - * Group raw source paths into rows for the Stored Files table. + * Group stored-file rows into table rows for the Stored Files panel. * * - Admin docs (`/app/docs/*`, README) collapse into a single * "Project NOMAD documentation · N files" row. * - ZIMs, uploads, and others stay as individual rows, sorted by bucket then * alphabetically by filename so related items cluster naturally. */ -export function groupAndSortKbFiles(sources: string[]): KbFileGroup[] { - const buckets: Record = { +export function groupAndSortKbFiles(files: StoredFileInfo[]): KbFileGroup[] { + const buckets: Record = { zim: [], upload: [], admin_docs: [], other: [], } - for (const source of sources) { - buckets[classifyKbFile(source)].push(source) + for (const file of files) { + buckets[classifyKbFile(file.source)].push(file) } const groups: KbFileGroup[] = [] @@ -78,20 +87,24 @@ export function groupAndSortKbFiles(sources: string[]): KbFileGroup[] { source: '__admin_docs_group__', displayName: `Project NOMAD documentation · ${members.length} file${members.length === 1 ? '' : 's'}`, count: members.length, - members, + members: members.map((m) => m.source), + state: null, + chunksEmbedded: 0, }) continue } - for (const source of members.sort((a, b) => - sourceToDisplayName(a).localeCompare(sourceToDisplayName(b)) + for (const file of members.sort((a, b) => + sourceToDisplayName(a.source).localeCompare(sourceToDisplayName(b.source)) )) { groups.push({ bucket, - source, - displayName: sourceToDisplayName(source), + source: file.source, + displayName: sourceToDisplayName(file.source), count: 1, members: [], + state: file.state, + chunksEmbedded: file.chunksEmbedded, }) } } diff --git a/admin/start/routes.ts b/admin/start/routes.ts index d117463..1868b0e 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -143,6 +143,7 @@ router router.get('/files', [RagController, 'getStoredFiles']) router.get('/file-warnings', [RagController, 'getFileWarnings']) router.delete('/files', [RagController, 'deleteFile']) + router.post('/files/embed', [RagController, 'embedFile']) router.get('/active-jobs', [RagController, 'getActiveJobs']) router.get('/failed-jobs', [RagController, 'getFailedJobs']) router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs']) diff --git a/admin/tests/unit/kb_file_grouping.spec.ts b/admin/tests/unit/kb_file_grouping.spec.ts index caa1da7..2dbab98 100644 --- a/admin/tests/unit/kb_file_grouping.spec.ts +++ b/admin/tests/unit/kb_file_grouping.spec.ts @@ -6,6 +6,14 @@ import { groupAndSortKbFiles, sourceToDisplayName, } from '../../inertia/lib/kb_file_grouping.js' +import type { StoredFileInfo } from '../../types/rag.js' + +/** Wrap source paths into the minimal StoredFileInfo shape that + * `groupAndSortKbFiles` now expects. State + chunk count are irrelevant to + * grouping/sorting behavior; the per-file state-pill rendering is exercised + * separately in the modal's component tests (added in the follow-up PR). */ +const asInfos = (sources: string[]): StoredFileInfo[] => + sources.map((source) => ({ source, state: null, chunksEmbedded: 0 })) test('classifyKbFile distinguishes ZIM, upload, admin_docs, and other', () => { assert.equal( @@ -34,12 +42,12 @@ test('sourceToDisplayName returns the basename', () => { }) test('groupAndSortKbFiles collapses all admin docs into a single row', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/docs/release-notes.md', '/app/docs/getting-started.md', '/app/docs/maps.md', '/app/README.md', - ]) + ])) assert.equal(groups.length, 1) assert.equal(groups[0].bucket, 'admin_docs') @@ -54,12 +62,12 @@ test('groupAndSortKbFiles collapses all admin docs into a single row', () => { }) test('groupAndSortKbFiles orders buckets ZIM → upload → admin_docs → other', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/docs/release-notes.md', '/unexpected/foo.txt', '/app/storage/kb_uploads/upload.pdf', '/app/storage/zim/devdocs.zim', - ]) + ])) assert.deepEqual( groups.map((g) => g.bucket), @@ -68,11 +76,11 @@ test('groupAndSortKbFiles orders buckets ZIM → upload → admin_docs → other }) test('groupAndSortKbFiles alphabetizes within a bucket', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/storage/zim/wikipedia.zim', '/app/storage/zim/devdocs.zim', '/app/storage/zim/ifixit.zim', - ]) + ])) assert.deepEqual( groups.map((g) => g.displayName), @@ -81,7 +89,7 @@ test('groupAndSortKbFiles alphabetizes within a bucket', () => { }) test('groupAndSortKbFiles uses singular noun when only one admin doc exists', () => { - const groups = groupAndSortKbFiles(['/app/docs/release-notes.md']) + const groups = groupAndSortKbFiles(asInfos(['/app/docs/release-notes.md'])) assert.equal(groups[0].displayName, 'Project NOMAD documentation · 1 file') }) @@ -90,10 +98,10 @@ test('groupAndSortKbFiles handles empty input', () => { }) test('groupAndSortKbFiles preserves a stable synthetic key for the admin docs group', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/docs/release-notes.md', '/app/docs/maps.md', - ]) + ])) // The admin-docs row uses a synthetic source key (not a real path) so it // can be used as a React key without colliding with any real file row. assert.equal(groups[0].source, '__admin_docs_group__') diff --git a/admin/types/rag.ts b/admin/types/rag.ts index aa14127..0b9c6c8 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -46,6 +46,18 @@ export type FileWarning = | { kind: 'zero_chunks'; fileSizeBytes: number } | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } +/** + * Row returned by `GET /api/rag/files`. `state` is null for sources that exist + * in Qdrant but have no `kb_ingest_state` row (pre-RFC-883 installs where the + * scanner hasn't yet backfilled). `chunksEmbedded` mirrors the state-machine + * field; 0 for state-row-less or zero-chunk files. + */ +export type StoredFileInfo = { + source: string + state: import('./kb_ingest_state.js').KbIngestStateValue | null + chunksEmbedded: number +} + /** * Result of computing per-file warnings. `ok: false` means the computation * itself failed (Qdrant unreachable, DB outage, FS read error) — distinct from