From 4f4cc5bc659ee7132a7ec7a2b091449d1ec9747b Mon Sep 17 00:00:00 2001 From: just-jbc Date: Mon, 20 Jul 2026 15:09:11 -0700 Subject: [PATCH] feat(rag): add subject/collection organization to knowledge base (#1063) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(rag): add subject/collection organization to knowledge base - Add nullable collection field to KbIngestState, propagated through the embed job, RAG service, and Qdrant point payloads (indexed for filtering) - Add upload-time category selection and per-file collection reassignment in the Knowledge Base modal, with a filterable Stored Files table - Add a 'Search in' collection filter to the chat interface, threaded through to searchSimilarDocuments as an optional Qdrant filter - Fix .docx extraction: previously routed through raw-text extraction (garbage output for a ZIP-based XML format); adds a proper mammoth-based extractor and a dedicated 'docx' file-type case * feat(rag): support dynamic KB collection creation, rename, and removal Extends collection organization with a Manage Collections UI: collections are created on the fly when a file is assigned to a new name, can be renamed (bulk-updates every tagged file and Qdrant point), and can be removed (reassigns tagged files back to Uncategorized rather than deleting anything). * fix(rag): use dynamic collections query in chat search filter chat/index.tsx still imported the static KB_COLLECTIONS constant for its 'Search in' dropdown, inconsistent with KnowledgeBaseModal.tsx which already uses the live getKnowledgeCollections() query. Renamed/added collections via the new Manage Collections UI weren't reflected in the chat filter. * feat(rag): broaden preset tags and add creatable collection combobox Replaces the survival-specific preset list with general-purpose starter tags (recipes, diy, health, technology, finance, travel, hobbies, reference, survival, energy) so the Knowledge Base reads well for home-lab/reference use, not just prepping. Adds sanitizeCollectionName() (trim, lowercase, length cap) applied on every write path server-side, and a dependency-free CollectionCombobox component replacing the plain { + setQuery(e.target.value) + setIsOpen(true) + }} + onFocus={() => setIsOpen(true)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault() + const trimmed = query.trim() + if (!trimmed) { + if (allowUncategorized) commit('') + return + } + commit(exactMatch ?? trimmed.toLowerCase()) + } else if (e.key === 'Escape') { + setIsOpen(false) + setQuery(value) + } + }} + placeholder={placeholder} + className="w-full rounded border border-border-subtle bg-surface-primary px-2 py-1 text-sm text-text-primary disabled:opacity-50" + /> + {isOpen && !disabled && ( +
+ {allowUncategorized && ( + + )} + {filtered.map((opt) => ( + + ))} + {showCreateOption && ( + + )} + {filtered.length === 0 && !showCreateOption && ( +
No matches
+ )} +
+ )} + + ) +} diff --git a/admin/inertia/components/chat/CollectionsManager.tsx b/admin/inertia/components/chat/CollectionsManager.tsx new file mode 100644 index 0000000..d1015c6 --- /dev/null +++ b/admin/inertia/components/chat/CollectionsManager.tsx @@ -0,0 +1,148 @@ +import { useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import StyledModal from '../StyledModal' +import StyledButton from '~/components/StyledButton' +import { useNotifications } from '~/context/NotificationContext' +import api from '~/lib/api' + +interface CollectionsManagerProps { + onClose: () => void +} + +export default function CollectionsManager({ onClose }: CollectionsManagerProps) { + const { addNotification } = useNotifications() + const queryClient = useQueryClient() + const [editingName, setEditingName] = useState(null) + const [editValue, setEditValue] = useState('') + const [confirmDelete, setConfirmDelete] = useState(null) + + const { data: collections = [], isLoading } = useQuery({ + queryKey: ['kbCollections'], + queryFn: () => api.getKnowledgeCollections(), + select: (data) => data?.collections ?? [], + }) + + const invalidateAll = () => { + queryClient.invalidateQueries({ queryKey: ['kbCollections'] }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + } + + const renameMutation = useMutation({ + mutationFn: ({ oldName, newName }: { oldName: string; newName: string }) => + api.renameCollection(oldName, newName), + onSuccess: (data) => { + addNotification({ type: 'success', message: data?.message || 'Collection renamed.' }) + setEditingName(null) + invalidateAll() + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to rename collection.' }) + }, + }) + + const deleteMutation = useMutation({ + mutationFn: (name: string) => api.deleteCollection(name), + onSuccess: (data) => { + addNotification({ type: 'success', message: data?.message || 'Collection removed.' }) + setConfirmDelete(null) + invalidateAll() + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to remove collection.' }) + }, + }) + + return ( + +
+

+ Rename or remove collections. Removing a collection doesn't delete any files — + they're simply moved back to Uncategorized so you can re-sort them. +

+ + {isLoading &&

Loading…

} + {!isLoading && collections.length === 0 && ( +

+ No collections yet. Assign a file to a collection from the Knowledge Base table to create one. +

+ )} + +
    + {collections.map((name) => ( +
  • + {editingName === name ? ( + <> + setEditValue(e.target.value)} + className="flex-1 rounded border border-border-subtle bg-surface-primary px-2 py-1 text-sm text-text-primary" + /> + + renameMutation.mutate({ oldName: name, newName: editValue.trim() }) + } + > + Save + + setEditingName(null)}> + Cancel + + + ) : confirmDelete === name ? ( + <> + + Remove "{name}"? Files move to Uncategorized. + + deleteMutation.mutate(name)} + > + Confirm + + setConfirmDelete(null)}> + Cancel + + + ) : ( + <> + {name} + { + setEditingName(name) + setEditValue(name) + }} + > + Rename + + setConfirmDelete(name)} + > + Remove + + + )} +
  • + ))} +
+
+
+ ) +} diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index cd47d04..942fb0a 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -1,5 +1,5 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useEffect, useRef, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import FileUploader from '~/components/file-uploader' import StyledButton from '~/components/StyledButton' import type { DynamicIconName } from '~/lib/icons' @@ -27,6 +27,9 @@ import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' import ActiveEmbedJobs from '~/components/ActiveEmbedJobs' import { SERVICE_NAMES } from '../../../constants/service_names' +import CollectionsManager from './CollectionsManager' +import { KB_COLLECTIONS } from '../../../constants/kb_collections' +import CollectionCombobox from './CollectionCombobox' interface KnowledgeBaseModalProps { aiAssistantName?: string @@ -34,7 +37,7 @@ interface KnowledgeBaseModalProps { } // File extensions the in-browser viewer can render. Must stay in sync with -// `RagService.VIEWABLE_TEXT_EXTENSIONS` — anything outside this set falls back +// `RagService.VIEWABLE_TEXT_EXTENSIONS` -- anything outside this set falls back // to Download. const VIEWABLE_EXTENSIONS = new Set(['md', 'txt', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'html']) @@ -117,7 +120,7 @@ type RowAction = /** * Pick the single adaptive per-row action button. Returns null when no action * makes sense for the current state (e.g. healthy indexed file with no - * warnings — bulk Re-embed All covers that case). `hasWarnings` lets us + * warnings -- bulk Re-embed All covers that case). `hasWarnings` lets us * surface a Re-embed affordance specifically when a file *looks* indexed but * has zero chunks or a stalled-mid-ingestion warning attached. */ @@ -143,6 +146,9 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o const { addNotification } = useNotifications() const [files, setFiles] = useState([]) const [isUploading, setIsUploading] = useState(false) + const [uploadCollection, setUploadCollection] = useState('') + const [collectionFilter, setCollectionFilter] = useState('All') + const [manageCollectionsOpen, setManageCollectionsOpen] = useState(false) const [confirmDeleteSource, setConfirmDeleteSource] = useState(null) const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null) const [bulkMode, setBulkMode] = useState(null) @@ -172,8 +178,18 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o select: (data) => data || [], }) - // Per-file conditional warnings (RFC #883 §6). `ok: false` means the - // computation itself failed (Qdrant/DB/FS) — distinct from `ok: true` with + const { data: knownCollections = [] } = useQuery({ + queryKey: ['kbCollections'], + queryFn: () => api.getKnowledgeCollections(), + select: (data) => data?.collections ?? [], + }) + + const comboboxOptions = useMemo(() => { + return Array.from(new Set([...KB_COLLECTIONS, ...knownCollections])).sort() + }, [knownCollections]) + + // Per-file conditional warnings (RFC #883 section 6). `ok: false` means the + // computation itself failed (Qdrant/DB/FS) -- distinct from `ok: true` with // an empty map, which means everything is healthy. We surface the failure // explicitly so a silent backend failure doesn't masquerade as health. const { data: warningsResult } = useQuery({ @@ -216,7 +232,20 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }) const uploadMutation = useMutation({ - mutationFn: (file: File) => api.uploadDocument(file), + mutationFn: (file: File) => api.uploadDocument(file, uploadCollection || undefined), + }) + + const updateCollectionMutation = useMutation({ + mutationFn: ({ source, collection }: { source: string; collection: string }) => + api.updateFileCollection(source, collection || null), + onSuccess: (data) => { + addNotification({ type: 'success', message: data?.message || 'Collection updated.' }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['kbCollections'] }) + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to update collection.' }) + }, }) const deleteMutation = useMutation({ @@ -461,7 +490,16 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o setFiles(Array.from(uploadedFiles)) }} /> -
+
+ Clean Up Failed - {/* Not gated on qdrantOffline: clearing stuck jobs must work during - a Qdrant/Ollama outage, which is exactly when they wedge. */}
+ + setManageCollectionsOpen(true)} + > + Manage Collections + — } @@ -725,13 +780,32 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o ) }, }, + { + accessor: 'collection', + title: 'Collection', + className: 'whitespace-nowrap', + render(record) { + if (record.bucket === 'admin_docs') { + return + } + const isSaving = + updateCollectionMutation.isPending && + updateCollectionMutation.variables?.source === record.source + return ( + updateCollectionMutation.mutate({ source: record.source, collection: val })} + options={comboboxOptions} + disabled={isSaving} + className="w-40" + /> + ) + }, + }, { accessor: 'source', title: '', render(record) { - // Admin docs are auto-discovered and managed by NOMAD itself — - // deleting one would just be re-embedded on the next sync, so - // we surface them as informational only and hide Delete. if (record.bucket === 'admin_docs') { return (
@@ -827,7 +901,12 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }, ]} - data={groupAndSortKbFiles(storedFiles, sort)} + data={groupAndSortKbFiles( + collectionFilter === 'All' + ? storedFiles + : storedFiles.filter((f) => f.collection === collectionFilter), + sort + )} loading={isLoadingFiles} />
@@ -966,6 +1045,10 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o onClose={() => setViewerSource(null)} /> )} + + {manageCollectionsOpen && ( + setManageCollectionsOpen(false)} /> + )}
) } @@ -982,7 +1065,7 @@ function FileViewerModal({ source, onClose }: { source: string; onClose: () => v const fallbackName = source.split(/[/\\]/).at(-1) ?? source const title = data?.fileName ?? fallbackName // `catchInternal` swallows errors and resolves to undefined, surfacing a - // toast — so the "couldn't load" branch is gated on a finished-but-empty + // toast -- so the "couldn't load" branch is gated on a finished-but-empty // fetch rather than on react-query's `isError`. const showError = isFetched && !data diff --git a/admin/inertia/components/chat/index.tsx b/admin/inertia/components/chat/index.tsx index 9be60d4..928db00 100644 --- a/admin/inertia/components/chat/index.tsx +++ b/admin/inertia/components/chat/index.tsx @@ -33,6 +33,7 @@ export default function Chat({ const [activeSessionId, setActiveSessionId] = useState(null) const [messages, setMessages] = useState([]) const [selectedModel, setSelectedModel] = useState('') + const [collectionFilter, setCollectionFilter] = useState('') const [pendingModelSwitch, setPendingModelSwitch] = useState(null) const pageLoadNormalizedRef = useRef(false) const [isStreamingResponse, setIsStreamingResponse] = useState(false) @@ -72,6 +73,12 @@ export default function Chat({ select: (data) => data || [], }) + const { data: knownCollections = [] } = useQuery({ + queryKey: ['kbCollections'], + queryFn: () => api.getKnowledgeCollections(), + select: (data) => data?.collections ?? [], + }) + const { data: chatSuggestions, isLoading: chatSuggestionsLoading } = useQuery({ queryKey: ['chatSuggestions'], queryFn: async ({ signal }) => { @@ -102,6 +109,7 @@ export default function Chat({ model: string messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> sessionId?: number + collection?: string }) => api.sendChatMessage({ ...request, stream: false }), onSuccess: async (data) => { if (!data || !activeSessionId) { @@ -323,7 +331,13 @@ export default function Chat({ try { await api.streamChatMessage( - { model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, sessionId: sessionId ? Number(sessionId) : undefined }, + { + model: selectedModel || 'llama3.2', + messages: chatMessages, + stream: true, + sessionId: sessionId ? Number(sessionId) : undefined, + collection: collectionFilter || undefined, + }, (chunkContent, chunkThinking, done) => { if (chunkThinking.length > 0 && thinkingStartTime === null) { thinkingStartTime = Date.now() @@ -414,10 +428,11 @@ export default function Chat({ model: selectedModel || 'llama3.2', messages: chatMessages, sessionId: sessionId ? Number(sessionId) : undefined, + collection: collectionFilter || undefined, }) } }, - [activeSessionId, messages, selectedModel, chatMutation, queryClient, streamingEnabled] + [activeSessionId, messages, selectedModel, collectionFilter, chatMutation, queryClient, streamingEnabled] ) return ( @@ -471,6 +486,22 @@ export default function Chat({ {remoteStatus?.connected === false ? 'Remote Disconnected' : 'Remote Connected'} )} +
+ + +