From d850cb958844f4f3d64158921ae0ff27a74b39c2 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Sun, 17 May 2026 21:42:54 -0700 Subject: [PATCH] =?UTF-8?q?feat(KB):=20per-file=20ingest=20action=20+=20st?= =?UTF-8?q?ate=20indicator=20on=20Stored=20Files=20(RFC=20#883=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the Manual-mode UX dead-end: after toggling 'Auto-index new content for AI?' to Manual, a freshly-downloaded ZIM (or any pending_decision file) had no UI path to opt in for embedding short of the global Sync Storage / Re-embed All bulk actions. Per RFC #883 §5, each Stored Files row now carries a state pill and an adaptive single-button action. State pill (left of any existing warning chips): - 'Indexed' — green; row had chunks in Qdrant or state row is 'indexed' - 'Not Indexed' — neutral; state is pending_decision or browse_only - 'Failed' — red - 'Stalled' — amber - admin_docs collapsed row has no pill ('Managed by NOMAD' carries it) Adaptive action button (paired with the existing Delete button per row): - pending_decision → 'Index' (force=false) - browse_only → 'Index' (force=true) - failed / stalled → 'Retry' (force=true) - indexed + warning chip → 'Re-embed' (force=true; confirm modal first) - indexed healthy / null → no action button (bulk Re-embed All covers it) Backend: GET /api/rag/files now returns { files: Array<{ source, state, chunksEmbedded }> } instead of a flat string[]. State + chunk-count come from a single KbIngestState query unioned into the existing Qdrant-derived source list (no new round trips). New POST /api/rag/files/embed validates the source is known, refuses if any inflight job already targets the same filePath (prevents double-click duplicate-chunk hazard), pre-deletes Qdrant points when force=true, then dispatches via the existing _dispatchEmbedJobsFor helper used by reembedAll. Per-file Re-embed (force=true on an already-indexed file) routes through a StyledModal confirmation since it deletes existing vectors before queueing a fresh job — same destructive-action weight as Delete's inline confirm but heavier since it affects search until the rebuild finishes. Folds in PR #907's blank-screen fix because my new render needs the same generic restored: `>` and `record.displayName` (instead of the unresolved `sourceToDisplayName(record.source)` that ships in rc.5 and ReferenceErrors on modal open). PR #907 also adds title tooltips on the three bulk-action buttons; those tooltips are NOT included here — let PR #907 land first or independently for that part. Multi-select bulk-opt-in deferred per discussion: most Manual-mode users ingest 1-2 files at a time, the existing global toggle covers the bulk case, and checkboxes would expand scope past what rc.6 should hold. Will file a follow-up issue for an 'Index N pending files' single-click button once this lands. Tests-in-PR scope was limited to keeping `kb_file_grouping.spec.ts` green after the StoredFileInfo[] signature change (added asInfos() wrapper). Dedicated unit tests for embedSingleFile (unknown source / inflight refused / force=true delete-then-dispatch) and the new state-pill rendering will land in a follow-up PR alongside Playwright coverage of the row actions. Verification path: NOMAD3 currently runs project-nomad-admin:integration- rc6-preview (PRs #907 + #908 atop rc.5). After this branch is built into a new integration tag, I'll re-run targeted Playwright UAT on the KB modal covering: state pill rendering per state, Index click on pending_decision opts in cleanly, Retry on failed re-dispatches successfully, Re-embed confirmation modal copy + delete-then-dispatch on the military-medicine partial-stall row, and Delete flow untouched. Co-Authored-By: Claude Opus 4.7 (1M context) --- admin/app/controllers/rag_controller.ts | 11 +- admin/app/services/rag_service.ts | 85 +++++++- admin/app/validators/rag.ts | 7 + .../components/chat/KnowledgeBaseModal.tsx | 199 ++++++++++++++++-- admin/inertia/lib/api.ts | 11 +- admin/inertia/lib/kb_file_grouping.ts | 43 ++-- admin/start/routes.ts | 1 + admin/tests/unit/kb_file_grouping.spec.ts | 26 ++- admin/types/rag.ts | 12 ++ 9 files changed, 341 insertions(+), 54 deletions(-) diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 7ca7b6d..da17101 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app' import { randomBytes } from 'node:crypto' import { sanitizeFilename } from '../utils/fs.js' import { basename } from 'node:path' -import { deleteFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag' +import { deleteFileSchema, embedFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag' import logger from '@adonisjs/core/services/logger' @inject() @@ -82,6 +82,15 @@ export default class RagController { return response.status(200).json({ message: result.message }) } + public async embedFile({ request, response }: HttpContext) { + const { source, force } = await request.validateUsing(embedFileSchema) + const result = await this.ragService.embedSingleFile(source, force ?? false) + if (!result.success) { + return response.status(409).json({ error: result.message }) + } + return response.status(202).json({ message: result.message }) + } + public async getFailedJobs({ response }: HttpContext) { const jobs = await EmbedFileJob.listFailedJobs() return response.status(200).json(jobs) diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 21e302f..20a418a 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -20,7 +20,8 @@ import KbIngestState from '#models/kb_ingest_state' import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js' import KbRatioRegistry from '#models/kb_ratio_registry' import { decideWarnings } from '../utils/kb_warning_decision.js' -import type { FileWarning, FileWarningsResult } from '../../types/rag.js' +import type { FileWarning, FileWarningsResult, StoredFileInfo } from '../../types/rag.js' +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' @@ -1051,7 +1052,7 @@ export class RagService { } } - public async getStoredFiles(): Promise { + public async getStoredFiles(): Promise { try { await this._ensureCollection( RagService.CONTENT_COLLECTION_NAME, @@ -1090,10 +1091,15 @@ export class RagService { // in particular) have no row to attach to. The state machine is the // authoritative "what's on disk?" view; Qdrant is "what made it into // the vector store?". Both are needed to render the KB UI honestly. + const stateByPath = new Map() try { - const stateRows = await KbIngestState.query().select('file_path') + const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded') for (const row of stateRows) { sources.add(row.file_path) + stateByPath.set(row.file_path, { + state: row.state, + chunks_embedded: row.chunks_embedded, + }) } } catch (error) { // Non-fatal: if the state machine query fails for any reason we'd @@ -1104,7 +1110,14 @@ export class RagService { ) } - return Array.from(sources) + return Array.from(sources).map((source) => { + const row = stateByPath.get(source) + return { + source, + state: row?.state ?? null, + chunksEmbedded: row?.chunks_embedded ?? 0, + } + }) } catch (error) { logger.error('Error retrieving stored files:', error) return [] @@ -1401,6 +1414,70 @@ export class RagService { return { queuedCount, dedupedCount, failedPaths } } + /** + * Dispatch an embed job for a single stored file. Wraps `_dispatchEmbedJobsFor` + * with the safety checks needed for a user-triggered per-row action: + * 1. The source must be known to the scanner OR have a state row — prevents + * arbitrary path dispatch from the public API. + * 2. We refuse if any inflight job (waiting/active/delayed/paused) already + * targets this filePath. Otherwise a double-click or a rapid retry could + * enqueue duplicate jobs, producing duplicate chunks. + * 3. When `force` is true (Re-embed of an already-indexed file), we + * pre-delete the prior Qdrant points so the new run doesn't stack on + * top of the old ones. For force=false (Index of a never-embedded file), + * there's nothing to clear. + */ + public async embedSingleFile( + source: string, + force: boolean = false + ): Promise<{ success: boolean; message: string }> { + const stateRow = await KbIngestState.query().where('file_path', source).first() + if (!stateRow) { + const knownFiles = await this._discoverKbFiles() + if (!knownFiles.includes(source)) { + return { + success: false, + message: 'File is not a tracked knowledge-base source.', + } + } + } + + const { EmbedFileJob } = await import('#jobs/embed_file_job') + const { QueueService } = await import('#services/queue_service') + const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue) + const inflight = await queue.getJobs(['waiting', 'active', 'delayed', 'paused']) + if (inflight.some((j) => j.data?.filePath === source)) { + return { + success: false, + message: 'A job for this file is already in progress. Wait for it to finish before re-queuing.', + } + } + + if (force) { + try { + await this._deletePointsBySource(source) + } catch (err) { + logger.error(`[RAG] Failed to delete prior points for ${source}; aborting re-embed:`, err) + return { + success: false, + message: 'Failed to clear prior embeddings before re-embed.', + } + } + } + + const result = await this._dispatchEmbedJobsFor([source], { force }) + if (result.failedPaths.length > 0) { + return { + success: false, + message: 'Failed to dispatch embed job for this file.', + } + } + return { + success: true, + message: force ? 'Re-embed queued for this file.' : 'Indexing queued for this file.', + } + } + /** * Delete all Qdrant points whose `source` payload matches the given path. * Unlike deleteFileBySource(), this does NOT touch the file on disk — used diff --git a/admin/app/validators/rag.ts b/admin/app/validators/rag.ts index 8326f44..5bef836 100644 --- a/admin/app/validators/rag.ts +++ b/admin/app/validators/rag.ts @@ -12,6 +12,13 @@ export const deleteFileSchema = vine.compile( }) ) +export const embedFileSchema = vine.compile( + vine.object({ + source: vine.string().minLength(1), + force: vine.boolean().optional(), + }) +) + export const estimateBatchSchema = vine.compile( vine.object({ files: vine diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index dadfb3f..301bf67 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -2,6 +2,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { useEffect, useRef, useState } from 'react' import FileUploader from '~/components/file-uploader' import StyledButton from '~/components/StyledButton' +import type { DynamicIconName } from '~/lib/icons' import StyledSectionHeader from '~/components/StyledSectionHeader' import StyledTable from '~/components/StyledTable' import { useNotifications } from '~/context/NotificationContext' @@ -10,6 +11,7 @@ import { groupAndSortKbFiles, type KbFileGroup, } from '~/lib/kb_file_grouping' +import type { KbIngestStateValue } from '../../../types/kb_ingest_state' import { IconX } from '@tabler/icons-react' import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' @@ -21,11 +23,82 @@ interface KnowledgeBaseModalProps { onClose: () => void } +/** + * Compact label for the per-row ingestion state. Files that exist in Qdrant + * with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883 + * installs whose chunks are real, so we display them as "Indexed" rather than + * surfacing the absent-row detail. Admin-docs group has no pill (the "Managed + * by NOMAD" message in the action column carries the same signal). + */ +function renderStatePill(record: KbFileGroup): React.ReactNode { + if (record.bucket === 'admin_docs') return null + const effective: KbIngestStateValue = record.state ?? 'indexed' + + const base = 'inline-flex items-center text-xs font-medium rounded px-2 py-0.5 border' + switch (effective) { + case 'indexed': + return ( + + Indexed + + ) + case 'pending_decision': + case 'browse_only': + return ( + + Not Indexed + + ) + case 'failed': + return ( + + Failed + + ) + case 'stalled': + return ( + + Stalled + + ) + } +} + +type RowAction = + | { kind: 'index'; label: string; force: boolean; variant: 'primary'; icon: DynamicIconName } + | { kind: 'reembed'; label: string; force: true; variant: 'secondary'; icon: DynamicIconName } + +/** + * Pick the single adaptive per-row action button. Returns null when no action + * makes sense for the current state (e.g. healthy indexed file with no + * warnings — bulk Re-embed All covers that case). `hasWarnings` lets us + * surface a Re-embed affordance specifically when a file *looks* indexed but + * has zero chunks or a stalled-mid-ingestion warning attached. + */ +function pickRowAction(record: KbFileGroup, hasWarnings: boolean): RowAction | null { + if (record.bucket === 'admin_docs') return null + const effective: KbIngestStateValue = record.state ?? 'indexed' + switch (effective) { + case 'indexed': + return hasWarnings + ? { kind: 'reembed', label: 'Re-embed', force: true, variant: 'secondary', icon: 'IconRefreshAlert' } + : null + case 'pending_decision': + return { kind: 'index', label: 'Index', force: false, variant: 'primary', icon: 'IconDownload' } + case 'browse_only': + return { kind: 'index', label: 'Index', force: true, variant: 'primary', icon: 'IconDownload' } + case 'failed': + case 'stalled': + return { kind: 'index', label: 'Retry', force: true, variant: 'primary', icon: 'IconRefresh' } + } +} + export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", onClose }: KnowledgeBaseModalProps) { const { addNotification } = useNotifications() const [files, setFiles] = useState([]) const [isUploading, setIsUploading] = useState(false) const [confirmDeleteSource, setConfirmDeleteSource] = useState(null) + const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null) const [bulkMode, setBulkMode] = useState(null) const [resetTyped, setResetTyped] = useState('') const fileUploaderRef = useRef>(null) @@ -111,6 +184,25 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }) + const embedMutation = useMutation({ + mutationFn: ({ source, force }: { source: string; force: boolean }) => + api.embedSingleRAGFile(source, force), + onSuccess: (data) => { + addNotification({ + type: 'success', + message: data?.message || 'File queued for embedding.', + }) + setConfirmReembed(null) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + queryClient.invalidateQueries({ queryKey: ['kbFileWarnings'] }) + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to queue file.' }) + setConfirmReembed(null) + }, + }) + const cleanupFailedMutation = useMutation({ mutationFn: () => api.cleanupFailedEmbedJobs(), onSuccess: (data) => { @@ -466,32 +558,38 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o title: 'File Name', render(record) { const warnings = fileWarnings[record.source] ?? [] + const pill = renderStatePill(record) return ( -
+
{record.displayName} - {warnings.map((w, i) => ( - - - {w.kind === 'zero_chunks' && ( - - Embedded 0 chunks — this file has no text content. - AI Assistant cannot reference it. + {(pill || warnings.length > 0) && ( +
+ {pill} + {warnings.map((w, i) => ( + + + {w.kind === 'zero_chunks' && ( + + Embedded 0 chunks — this file has no text content. + AI Assistant cannot reference it. + + )} + {w.kind === 'partial_stall' && ( + + Only {w.chunksEmbedded.toLocaleString()} of est.{' '} + {w.chunksExpected.toLocaleString()} chunks embedded — + ingestion may have stalled. + + )} - )} - {w.kind === 'partial_stall' && ( - - Only {w.chunksEmbedded.toLocaleString()} of est.{' '} - {w.chunksExpected.toLocaleString()} chunks embedded — - ingestion may have stalled. - - )} - - ))} + ))} +
+ )}
) }, @@ -538,14 +636,38 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
) } + + const warnings = fileWarnings[record.source] ?? [] + const action = pickRowAction(record, warnings.length > 0) + const actionPendingForThisRow = + embedMutation.isPending && embedMutation.variables?.source === record.source + return ( -
+
+ {action && ( + { + if (action.kind === 'reembed') { + setConfirmReembed({ source: record.source, displayName: record.displayName }) + } else { + embedMutation.mutate({ source: record.source, force: action.force }) + } + }} + disabled={qdrantOffline || deleteMutation.isPending || embedMutation.isPending} + loading={actionPendingForThisRow} + > + {action.label} + + )} setConfirmDeleteSource(record.source)} - disabled={deleteMutation.isPending} + disabled={deleteMutation.isPending || embedMutation.isPending} loading={deleteMutation.isPending && confirmDeleteSource === record.source} >Delete
@@ -654,6 +776,37 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
)} + + {confirmReembed && ( + + embedMutation.mutate({ source: confirmReembed.source, force: true }) + } + onCancel={() => setConfirmReembed(null)} + > +
+

+ This will delete the existing embeddings for{' '} + {confirmReembed.displayName} and queue + a fresh embedding job. The file on disk is not touched. +

+
+

Heads up

+
    +
  • For large ZIM archives this can take a long time, especially on CPU-only systems.
    • +
  • Search results that referenced this file will be incomplete until the new embedding finishes.
    • +
  • If a job for this file is already running, the re-embed will be refused — wait for it to finish first.
    • +
+
+
+ + )} ) } diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 3d92995..cd7eba8 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -5,7 +5,7 @@ import { FileEntry } from '../../types/files' import { CheckLatestVersionResult, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' import { DownloadJobWithProgress, WikipediaState } from '../../types/downloads' import type { Country, CountryCode, CountryGroup, MapExtractPreflight } from '../../types/maps' -import { EmbedJobWithProgress, FileWarningsResult } from '../../types/rag' +import { EmbedJobWithProgress, FileWarningsResult, StoredFileInfo } from '../../types/rag' import type { CategoryWithStatus, CollectionWithStatus, ContentUpdateCheckResult, ResourceUpdateInfo } from '../../types/collections' import { catchInternal } from './util' import { NomadChatResponse, NomadInstalledModel, NomadOllamaModel, OllamaChatRequest } from '../../types/ollama' @@ -470,11 +470,18 @@ class API { async getStoredRAGFiles() { return catchInternal(async () => { - const response = await this.client.get<{ files: string[] }>('/rag/files') + const response = await this.client.get<{ files: StoredFileInfo[] }>('/rag/files') return response.data.files })() } + async embedSingleRAGFile(source: string, force: boolean = false) { + return catchInternal(async () => { + const response = await this.client.post<{ message: string }>('/rag/files/embed', { source, force }) + return response.data + })() + } + async getKbFileWarnings() { return catchInternal(async () => { const response = await this.client.get('/rag/file-warnings') diff --git a/admin/inertia/lib/kb_file_grouping.ts b/admin/inertia/lib/kb_file_grouping.ts index 9f474c6..3d2ae04 100644 --- a/admin/inertia/lib/kb_file_grouping.ts +++ b/admin/inertia/lib/kb_file_grouping.ts @@ -1,9 +1,12 @@ +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' +import type { StoredFileInfo } from '../../types/rag.js' + /** - * Knowledge-base files come back as a flat list of source paths from - * `/api/rag/files`. The UI groups them so the user sees the categories that - * matter to them — ZIMs, uploaded documents, and a single rolled-up entry for - * Project NOMAD's bundled docs (rather than the 12+ individual markdown files - * those break into). + * Knowledge-base files come back as a list of `{source, state, chunksEmbedded}` + * objects from `/api/rag/files`. The UI groups them so the user sees the + * categories that matter to them — ZIMs, uploaded documents, and a single + * rolled-up entry for Project NOMAD's bundled docs (rather than the 12+ + * individual markdown files those break into). * * Bucket assignment is purely by path prefix; matching is done on `/` so the * server-emitted absolute paths work regardless of which Linux mount the admin @@ -43,27 +46,33 @@ export interface KbFileGroup { count: number /** All member source paths — populated for collapsed groups, empty otherwise. */ members: string[] + /** Per-file ingestion state. `null` for the collapsed admin_docs group and + * for any source that exists in Qdrant but has no state row yet. */ + state: KbIngestStateValue | null + /** Chunks currently embedded for this source; 0 for state-row-less or + * zero-chunk files. Always 0 for the collapsed admin_docs group. */ + chunksEmbedded: number } const BUCKET_SORT_ORDER: KbFileBucket[] = ['zim', 'upload', 'admin_docs', 'other'] /** - * Group raw source paths into rows for the Stored Files table. + * Group stored-file rows into table rows for the Stored Files panel. * * - Admin docs (`/app/docs/*`, README) collapse into a single * "Project NOMAD documentation · N files" row. * - ZIMs, uploads, and others stay as individual rows, sorted by bucket then * alphabetically by filename so related items cluster naturally. */ -export function groupAndSortKbFiles(sources: string[]): KbFileGroup[] { - const buckets: Record = { +export function groupAndSortKbFiles(files: StoredFileInfo[]): KbFileGroup[] { + const buckets: Record = { zim: [], upload: [], admin_docs: [], other: [], } - for (const source of sources) { - buckets[classifyKbFile(source)].push(source) + for (const file of files) { + buckets[classifyKbFile(file.source)].push(file) } const groups: KbFileGroup[] = [] @@ -78,20 +87,24 @@ export function groupAndSortKbFiles(sources: string[]): KbFileGroup[] { source: '__admin_docs_group__', displayName: `Project NOMAD documentation · ${members.length} file${members.length === 1 ? '' : 's'}`, count: members.length, - members, + members: members.map((m) => m.source), + state: null, + chunksEmbedded: 0, }) continue } - for (const source of members.sort((a, b) => - sourceToDisplayName(a).localeCompare(sourceToDisplayName(b)) + for (const file of members.sort((a, b) => + sourceToDisplayName(a.source).localeCompare(sourceToDisplayName(b.source)) )) { groups.push({ bucket, - source, - displayName: sourceToDisplayName(source), + source: file.source, + displayName: sourceToDisplayName(file.source), count: 1, members: [], + state: file.state, + chunksEmbedded: file.chunksEmbedded, }) } } diff --git a/admin/start/routes.ts b/admin/start/routes.ts index d117463..1868b0e 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -143,6 +143,7 @@ router router.get('/files', [RagController, 'getStoredFiles']) router.get('/file-warnings', [RagController, 'getFileWarnings']) router.delete('/files', [RagController, 'deleteFile']) + router.post('/files/embed', [RagController, 'embedFile']) router.get('/active-jobs', [RagController, 'getActiveJobs']) router.get('/failed-jobs', [RagController, 'getFailedJobs']) router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs']) diff --git a/admin/tests/unit/kb_file_grouping.spec.ts b/admin/tests/unit/kb_file_grouping.spec.ts index caa1da7..2dbab98 100644 --- a/admin/tests/unit/kb_file_grouping.spec.ts +++ b/admin/tests/unit/kb_file_grouping.spec.ts @@ -6,6 +6,14 @@ import { groupAndSortKbFiles, sourceToDisplayName, } from '../../inertia/lib/kb_file_grouping.js' +import type { StoredFileInfo } from '../../types/rag.js' + +/** Wrap source paths into the minimal StoredFileInfo shape that + * `groupAndSortKbFiles` now expects. State + chunk count are irrelevant to + * grouping/sorting behavior; the per-file state-pill rendering is exercised + * separately in the modal's component tests (added in the follow-up PR). */ +const asInfos = (sources: string[]): StoredFileInfo[] => + sources.map((source) => ({ source, state: null, chunksEmbedded: 0 })) test('classifyKbFile distinguishes ZIM, upload, admin_docs, and other', () => { assert.equal( @@ -34,12 +42,12 @@ test('sourceToDisplayName returns the basename', () => { }) test('groupAndSortKbFiles collapses all admin docs into a single row', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/docs/release-notes.md', '/app/docs/getting-started.md', '/app/docs/maps.md', '/app/README.md', - ]) + ])) assert.equal(groups.length, 1) assert.equal(groups[0].bucket, 'admin_docs') @@ -54,12 +62,12 @@ test('groupAndSortKbFiles collapses all admin docs into a single row', () => { }) test('groupAndSortKbFiles orders buckets ZIM → upload → admin_docs → other', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/docs/release-notes.md', '/unexpected/foo.txt', '/app/storage/kb_uploads/upload.pdf', '/app/storage/zim/devdocs.zim', - ]) + ])) assert.deepEqual( groups.map((g) => g.bucket), @@ -68,11 +76,11 @@ test('groupAndSortKbFiles orders buckets ZIM → upload → admin_docs → other }) test('groupAndSortKbFiles alphabetizes within a bucket', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/storage/zim/wikipedia.zim', '/app/storage/zim/devdocs.zim', '/app/storage/zim/ifixit.zim', - ]) + ])) assert.deepEqual( groups.map((g) => g.displayName), @@ -81,7 +89,7 @@ test('groupAndSortKbFiles alphabetizes within a bucket', () => { }) test('groupAndSortKbFiles uses singular noun when only one admin doc exists', () => { - const groups = groupAndSortKbFiles(['/app/docs/release-notes.md']) + const groups = groupAndSortKbFiles(asInfos(['/app/docs/release-notes.md'])) assert.equal(groups[0].displayName, 'Project NOMAD documentation · 1 file') }) @@ -90,10 +98,10 @@ test('groupAndSortKbFiles handles empty input', () => { }) test('groupAndSortKbFiles preserves a stable synthetic key for the admin docs group', () => { - const groups = groupAndSortKbFiles([ + const groups = groupAndSortKbFiles(asInfos([ '/app/docs/release-notes.md', '/app/docs/maps.md', - ]) + ])) // The admin-docs row uses a synthetic source key (not a real path) so it // can be used as a React key without colliding with any real file row. assert.equal(groups[0].source, '__admin_docs_group__') diff --git a/admin/types/rag.ts b/admin/types/rag.ts index aa14127..0b9c6c8 100644 --- a/admin/types/rag.ts +++ b/admin/types/rag.ts @@ -46,6 +46,18 @@ export type FileWarning = | { kind: 'zero_chunks'; fileSizeBytes: number } | { kind: 'partial_stall'; chunksEmbedded: number; chunksExpected: number } +/** + * Row returned by `GET /api/rag/files`. `state` is null for sources that exist + * in Qdrant but have no `kb_ingest_state` row (pre-RFC-883 installs where the + * scanner hasn't yet backfilled). `chunksEmbedded` mirrors the state-machine + * field; 0 for state-row-less or zero-chunk files. + */ +export type StoredFileInfo = { + source: string + state: import('./kb_ingest_state.js').KbIngestStateValue | null + chunksEmbedded: number +} + /** * Result of computing per-file warnings. `ok: false` means the computation * itself failed (Qdrant unreachable, DB outage, FS read error) — distinct from