diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 9174616..e886165 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -77,10 +77,12 @@ export default class OllamaController { logger.debug(`[OllamaController] Rewritten query for RAG: "${rewrittenQuery}"`) if (rewrittenQuery) { + const collectionFilter: string | null = request.input('collection', null) const relevantDocs = await this.ragService.searchSimilarDocuments( rewrittenQuery, 5, // Top 5 most relevant chunks - 0.3 // Minimum similarity score of 0.3 + 0.3, // Minimum similarity score of 0.3 + collectionFilter ?? undefined ) logger.debug(`[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${rewrittenQuery}"`) @@ -456,3 +458,4 @@ export default class OllamaController { } } } + diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index 7738d0f..22106cf 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -9,6 +9,7 @@ import { sanitizeFilename } from '../utils/fs.js' import { basename } from 'node:path' import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag' import logger from '@adonisjs/core/services/logger' +import { sanitizeCollectionName } from '../../constants/kb_collections.js' @inject() export default class RagController { @@ -20,6 +21,8 @@ export default class RagController { return response.status(400).json({ error: 'No file uploaded' }) } + const collection = sanitizeCollectionName(request.input('collection', null)) + const randomSuffix = randomBytes(6).toString('hex') const sanitizedName = sanitizeFilename(uploadedFile.clientName) @@ -34,6 +37,7 @@ export default class RagController { const result = await EmbedFileJob.dispatch({ filePath: fullPath, fileName, + ...(collection ? { collection } : {}), }) return response.status(202).json({ @@ -42,9 +46,9 @@ export default class RagController { fileName, filePath: `/${RagService.UPLOADS_STORAGE_PATH}/${fileName}`, alreadyProcessing: !result.created, + ...(collection ? { collection } : {}), }) } - public async getActiveJobs({ response }: HttpContext) { const jobs = await EmbedFileJob.listActiveJobs() return response.status(200).json(jobs) @@ -68,6 +72,57 @@ export default class RagController { return response.status(200).json({ files }) } + public async getKnowledgeCollections({ response }: HttpContext) { + const collections = await this.ragService.getKnowledgeCollections() + return response.status(200).json({ collections }) + } + + public async updateFileCollection({ request, response }: HttpContext) { + const source: string | null = request.input('source', null) + // sanitizeCollectionName trims/lowercases/caps length, and returns null + // for empty input — which doubles as "clear back to Uncategorized". + const collection = sanitizeCollectionName(request.input('collection', null)) + + if (!source) { + return response.status(400).json({ error: 'source is required.' }) + } + + const result = await this.ragService.updateFileCollection(source, collection) + if (!result.success) { + return response.status(500).json({ error: result.message }) + } + return response.status(200).json({ message: result.message }) + } + + public async renameKnowledgeCollection({ request, response }: HttpContext) { + const oldName = sanitizeCollectionName(request.input('oldName', null)) + const newName = sanitizeCollectionName(request.input('newName', null)) + + if (!oldName || !newName) { + return response.status(400).json({ error: 'oldName and newName are required.' }) + } + + const result = await this.ragService.renameKnowledgeCollection(oldName, newName) + if (!result.success) { + return response.status(500).json({ error: result.message }) + } + return response.status(200).json({ message: result.message }) + } + + public async deleteKnowledgeCollection({ request, response }: HttpContext) { + const name = sanitizeCollectionName(request.input('name', null)) + + if (!name) { + return response.status(400).json({ error: 'name is required.' }) + } + + const result = await this.ragService.deleteKnowledgeCollection(name) + if (!result.success) { + return response.status(500).json({ error: result.message }) + } + return response.status(200).json({ message: result.message }) + } + public async getFileWarnings({ response }: HttpContext) { const result = await this.ragService.computeFileWarnings() return response.status(200).json(result) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index ba52ca8..cdb775b 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -20,9 +20,10 @@ export interface EmbedFileJobParams { isFinalBatch?: boolean // Whether this is the last batch (prevents premature deletion) // Running total of chunks embedded across prior batches in this dispatch chain. // Carried forward so the final batch can persist an accurate `chunks_embedded` - // count via KbIngestState.markIndexed (see #933 — without this, only the last + // count via KbIngestState.markIndexed (see #933 -- without this, only the last // batch's chunk count was stored while Qdrant held the full set). chunksSoFar?: number + collection?: string } export class EmbedFileJob { @@ -56,7 +57,7 @@ export class EmbedFileJob { } async handle(job: Job) { - const { filePath, fileName, batchOffset, totalArticles } = job.data as EmbedFileJobParams + const { filePath, fileName, batchOffset, totalArticles, collection } = job.data as EmbedFileJobParams const isZimBatch = batchOffset !== undefined const batchInfo = isZimBatch ? ` (batch offset: ${batchOffset})` : '' @@ -136,7 +137,8 @@ export class EmbedFileJob { filePath, allowDeletion, batchOffset, - onProgress + onProgress, + collection ) if (!result.success) { @@ -242,7 +244,7 @@ export class EmbedFileJob { // BullMQ's :completed retention (50 jobs) ages out, so the state row is // the only durable record of "this file finished embedding". try { - await KbIngestState.markIndexed(filePath, totalChunks) + await KbIngestState.markIndexed(filePath, totalChunks, collection) } catch (stateErr) { logger.warn( `[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`, diff --git a/admin/app/models/kb_ingest_state.ts b/admin/app/models/kb_ingest_state.ts index 2add407..dac1028 100644 --- a/admin/app/models/kb_ingest_state.ts +++ b/admin/app/models/kb_ingest_state.ts @@ -28,6 +28,9 @@ export default class KbIngestState extends BaseModel { @column() declare chunks_embedded: number + @column() + declare collection: string | null + @column() declare last_error: string | null @@ -37,18 +40,19 @@ export default class KbIngestState extends BaseModel { @column.dateTime({ autoCreate: true, autoUpdate: true }) declare updated_at: DateTime - static async getOrCreate(filePath: string): Promise { + static async getOrCreate(filePath: string, collection?: string): Promise { return this.firstOrCreate( { file_path: filePath }, - { file_path: filePath, state: 'pending_decision', chunks_embedded: 0 } + { file_path: filePath, state: 'pending_decision', chunks_embedded: 0, collection: collection ?? null } ) } - static async markIndexed(filePath: string, chunksEmbedded: number): Promise { - const row = await this.getOrCreate(filePath) + static async markIndexed(filePath: string, chunksEmbedded: number, collection?: string): Promise { + const row = await this.getOrCreate(filePath, collection) row.state = 'indexed' row.chunks_embedded = chunksEmbedded row.last_error = null + if (collection) row.collection = collection await row.save() } diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index 29b65cf..3fc5493 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -133,6 +133,10 @@ export class RagService { field_name: 'content_type', field_schema: 'keyword', }) + await this.qdrant!.createPayloadIndex(collectionName, { + field_name: 'collection', + field_schema: 'keyword', + }) } catch (error) { logger.error('Error ensuring Qdrant collection:', error) throw error @@ -695,14 +699,16 @@ export class RagService { extractedText: string, filepath: string, deleteAfterEmbedding: boolean = false, - onProgress?: (percent: number) => Promise + onProgress?: (percent: number) => Promise, + collection?: string ): Promise<{ success: boolean; message: string; chunks?: number }> { if (!extractedText || extractedText.trim().length === 0) { return { success: false, message: 'Process completed succesfully, but no text was found to embed.' } } const embedResult = await this.embedAndStoreText(extractedText, { - source: filepath + source: filepath, + ...(collection ? { collection } : {}) }, onProgress) if (!embedResult) { @@ -732,7 +738,8 @@ export class RagService { filepath: string, deleteAfterEmbedding: boolean = false, batchOffset?: number, - onProgress?: (percent: number) => Promise + onProgress?: (percent: number) => Promise, + collection?: string ): Promise { try { const fileType = determineFileType(filepath) @@ -781,7 +788,7 @@ export class RagService { : undefined // Embed extracted text and cleanup - return await this.embedTextAndCleanup(extractedText, filepath, deleteAfterEmbedding, scaledProgress) + return await this.embedTextAndCleanup(extractedText, filepath, deleteAfterEmbedding, scaledProgress, collection) } catch (error) { logger.error('[RAG] Error processing and embedding file:', error) return { success: false, message: 'Error processing and embedding file.' } @@ -800,7 +807,8 @@ export class RagService { public async searchSimilarDocuments( query: string, limit: number = 5, - scoreThreshold: number = 0.3 // Lower default threshold - was 0.7, now 0.3 + scoreThreshold: number = 0.3, // Lower default threshold - was 0.7, now 0.3 + collection?: string ): Promise }>> { try { logger.debug(`[RAG] Starting similarity search for query: "${query}"`) @@ -873,6 +881,7 @@ export class RagService { limit: searchLimit, score_threshold: scoreThreshold, with_payload: true, + ...(collection ? { filter: { must: [{ key: 'collection', match: { value: collection } }] } } : {}), }) logger.debug(`[RAG] Found ${searchResults.length} results above threshold ${scoreThreshold}`) @@ -1121,14 +1130,15 @@ export class RagService { // in particular) have no row to attach to. The state machine is the // authoritative "what's on disk?" view; Qdrant is "what made it into // the vector store?". Both are needed to render the KB UI honestly. - const stateByPath = new Map() + const stateByPath = new Map() try { - const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded') + const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded', 'collection') for (const row of stateRows) { sources.add(row.file_path) stateByPath.set(row.file_path, { state: row.state, chunks_embedded: row.chunks_embedded, + collection: row.collection, }) } } catch (error) { @@ -1155,6 +1165,7 @@ export class RagService { size: stats?.size ?? null, uploadedAt: stats?.modifiedTime.toISOString() ?? null, isUserUpload, + collection: row?.collection ?? null, } }) ) @@ -1164,6 +1175,114 @@ export class RagService { } } + /** + * Enumerate distinct `collection` values currently in the knowledge base, + * for populating a subject-picker in the upload/chat UI. Mirrors the + * `source` facet pattern used elsewhere in this file (see getStoredFiles). + */ + public async getKnowledgeCollections(): Promise { + await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION) + const facetResult = await this.qdrant!.facet(RagService.CONTENT_COLLECTION_NAME, { + key: 'collection', + limit: RagService.FACET_SOURCE_LIMIT, + exact: true, + }) + const collections = new Set() + for (const hit of facetResult.hits) { + if (typeof hit.value === 'string') collections.add(hit.value) + } + return Array.from(collections).sort() + } + + /** + * Reassign a stored file's collection after the fact. Updates the `collection` + * payload field on every existing Qdrant point for this source in place (no + * re-chunking or re-embedding needed), then mirrors the change onto the + * KbIngestState row so getStoredFiles() reflects it immediately. + */ + public async updateFileCollection( + source: string, + collection: string | null + ): Promise<{ success: boolean; message: string }> { + try { + await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION) + + await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, { + payload: { collection }, + filter: { must: [{ key: 'source', match: { value: source } }] }, + }) + + const row = await KbIngestState.query().where('file_path', source).first() + if (row) { + row.collection = collection + await row.save() + } + + return { success: true, message: collection ? `Moved to "${collection}".` : 'Moved to Uncategorized.' } + } catch (error) { + logger.error('[RAG] Error updating file collection:', error) + return { success: false, message: 'Error updating file collection.' } + } + } + + /** + * Rename a knowledge-base collection everywhere it's referenced: updates every + * Qdrant point tagged with the old name in place, and mirrors the change onto + * any matching KbIngestState rows so getStoredFiles() reflects it immediately. + */ + public async renameKnowledgeCollection( + oldName: string, + newName: string + ): Promise<{ success: boolean; message: string }> { + try { + if (!oldName || !newName || oldName === newName) { + return { success: false, message: 'Invalid collection names.' } + } + await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION) + + await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, { + payload: { collection: newName }, + filter: { must: [{ key: 'collection', match: { value: oldName } }] }, + }) + + await KbIngestState.query().where('collection', oldName).update({ collection: newName }) + + return { success: true, message: `Renamed "${oldName}" to "${newName}".` } + } catch (error) { + logger.error('[RAG] Error renaming knowledge collection:', error) + return { success: false, message: 'Error renaming collection.' } + } + } + + /** + * Remove a collection by reassigning every file tagged with it back to + * Uncategorized (collection: null). Non-destructive — no files or embeddings + * are deleted, only the grouping label is cleared so items can be + * recategorized later. + */ + public async deleteKnowledgeCollection( + name: string + ): Promise<{ success: boolean; message: string }> { + try { + if (!name) { + return { success: false, message: 'Invalid collection name.' } + } + await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION) + + await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, { + payload: { collection: null }, + filter: { must: [{ key: 'collection', match: { value: name } }] }, + }) + + await KbIngestState.query().where('collection', name).update({ collection: null }) + + return { success: true, message: `"${name}" removed. Files moved to Uncategorized.` } + } catch (error) { + logger.error('[RAG] Error deleting knowledge collection:', error) + return { success: false, message: 'Error deleting collection.' } + } + } + /** * Resolve a stored-file `source` to an absolute disk path, but only if the * path lives under the uploads directory. Mirrors the docs_service traversal diff --git a/admin/constants/kb_collections.ts b/admin/constants/kb_collections.ts new file mode 100644 index 0000000..a0b3a1e --- /dev/null +++ b/admin/constants/kb_collections.ts @@ -0,0 +1,40 @@ +/** + * Curated starter tags shown in the collection picker. These are just + * suggested defaults — the actual set of usable tags is open-ended, since + * `collection` is a free-form string and getKnowledgeCollections() returns + * whatever's actually in use (see RagService). Kept general-purpose rather + * than survival-specific so NOMAD's Knowledge Base reads well for home-lab, + * reference, and everyday use too. + */ +export const KB_COLLECTIONS = [ + 'recipes', + 'diy', + 'health', + 'technology', + 'finance', + 'travel', + 'hobbies', + 'reference', + 'survival', + 'energy', +] as const + +export type KbCollection = (typeof KB_COLLECTIONS)[number] + +/** Hard cap on a user-created tag's length, enforced client- and server-side. */ +export const KB_COLLECTION_NAME_MAX_LENGTH = 40 + +/** + * Normalize a user-entered collection name: trim whitespace, lowercase, cap + * length. Returns null for empty/whitespace-only input, meaning + * "uncategorized". Lowercasing is what makes de-dupe work — "Medical" and + * "medical" normalize to the same tag rather than forking into two, so this + * must run on every write path (upload, reassignment, rename) both + * client-side for instant feedback and server-side as the actual guarantee. + */ +export function sanitizeCollectionName(raw: string | null | undefined): string | null { + if (!raw) return null + const trimmed = raw.trim().toLowerCase() + if (!trimmed) return null + return trimmed.slice(0, KB_COLLECTION_NAME_MAX_LENGTH) +} diff --git a/admin/database/migrations/1776400000001_add_collection_to_kb_ingest_state.ts b/admin/database/migrations/1776400000001_add_collection_to_kb_ingest_state.ts new file mode 100644 index 0000000..ed49ac5 --- /dev/null +++ b/admin/database/migrations/1776400000001_add_collection_to_kb_ingest_state.ts @@ -0,0 +1,17 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'kb_ingest_state' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.string('collection').nullable().index() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('collection') + }) + } +} diff --git a/admin/inertia/components/chat/CollectionCombobox.tsx b/admin/inertia/components/chat/CollectionCombobox.tsx new file mode 100644 index 0000000..a1b0c48 --- /dev/null +++ b/admin/inertia/components/chat/CollectionCombobox.tsx @@ -0,0 +1,129 @@ +import { useEffect, useRef, useState } from 'react' + +interface CollectionComboboxProps { + /** Current value. Empty string means "Uncategorized". */ + value: string + onChange: (value: string) => void + /** Presets + known-in-use tags, already merged and deduped by the caller. */ + options: string[] + placeholder?: string + allowUncategorized?: boolean + className?: string + disabled?: boolean +} + +/** + * Dependency-free creatable combobox for collection tags. Lets you pick an + * existing option (preset or already-in-use) or type a brand new one — the + * "+ Create" row only appears when the (normalized) input doesn't already + * match something. Actual normalization (trim/lowercase/length cap) happens + * server-side via sanitizeCollectionName; this just lowercases for the + * match-check and display so duplicates-by-case don't look like real options. + */ +export default function CollectionCombobox({ + value, + onChange, + options, + placeholder = 'Uncategorized', + allowUncategorized = true, + className = '', + disabled = false, +}: CollectionComboboxProps) { + const [query, setQuery] = useState(value) + const [isOpen, setIsOpen] = useState(false) + const containerRef = useRef(null) + + useEffect(() => { + setQuery(value) + }, [value]) + + useEffect(() => { + function handleClickOutside(e: MouseEvent) { + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { + setIsOpen(false) + setQuery(value) + } + } + document.addEventListener('mousedown', handleClickOutside) + return () => document.removeEventListener('mousedown', handleClickOutside) + }, [value]) + + const normalizedQuery = query.trim().toLowerCase() + const filtered = normalizedQuery + ? options.filter((o) => o.toLowerCase().includes(normalizedQuery)) + : options + const exactMatch = options.find((o) => o.toLowerCase() === normalizedQuery) + const showCreateOption = normalizedQuery.length > 0 && !exactMatch + + const commit = (val: string) => { + onChange(val) + setQuery(val) + setIsOpen(false) + } + + return ( +
+ { + setQuery(e.target.value) + setIsOpen(true) + }} + onFocus={() => setIsOpen(true)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault() + const trimmed = query.trim() + if (!trimmed) { + if (allowUncategorized) commit('') + return + } + commit(exactMatch ?? trimmed.toLowerCase()) + } else if (e.key === 'Escape') { + setIsOpen(false) + setQuery(value) + } + }} + placeholder={placeholder} + className="w-full rounded border border-border-subtle bg-surface-primary px-2 py-1 text-sm text-text-primary disabled:opacity-50" + /> + {isOpen && !disabled && ( +
+ {allowUncategorized && ( + + )} + {filtered.map((opt) => ( + + ))} + {showCreateOption && ( + + )} + {filtered.length === 0 && !showCreateOption && ( +
No matches
+ )} +
+ )} +
+ ) +} diff --git a/admin/inertia/components/chat/CollectionsManager.tsx b/admin/inertia/components/chat/CollectionsManager.tsx new file mode 100644 index 0000000..d1015c6 --- /dev/null +++ b/admin/inertia/components/chat/CollectionsManager.tsx @@ -0,0 +1,148 @@ +import { useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import StyledModal from '../StyledModal' +import StyledButton from '~/components/StyledButton' +import { useNotifications } from '~/context/NotificationContext' +import api from '~/lib/api' + +interface CollectionsManagerProps { + onClose: () => void +} + +export default function CollectionsManager({ onClose }: CollectionsManagerProps) { + const { addNotification } = useNotifications() + const queryClient = useQueryClient() + const [editingName, setEditingName] = useState(null) + const [editValue, setEditValue] = useState('') + const [confirmDelete, setConfirmDelete] = useState(null) + + const { data: collections = [], isLoading } = useQuery({ + queryKey: ['kbCollections'], + queryFn: () => api.getKnowledgeCollections(), + select: (data) => data?.collections ?? [], + }) + + const invalidateAll = () => { + queryClient.invalidateQueries({ queryKey: ['kbCollections'] }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + } + + const renameMutation = useMutation({ + mutationFn: ({ oldName, newName }: { oldName: string; newName: string }) => + api.renameCollection(oldName, newName), + onSuccess: (data) => { + addNotification({ type: 'success', message: data?.message || 'Collection renamed.' }) + setEditingName(null) + invalidateAll() + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to rename collection.' }) + }, + }) + + const deleteMutation = useMutation({ + mutationFn: (name: string) => api.deleteCollection(name), + onSuccess: (data) => { + addNotification({ type: 'success', message: data?.message || 'Collection removed.' }) + setConfirmDelete(null) + invalidateAll() + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to remove collection.' }) + }, + }) + + return ( + +
+

+ Rename or remove collections. Removing a collection doesn't delete any files — + they're simply moved back to Uncategorized so you can re-sort them. +

+ + {isLoading &&

Loading…

} + {!isLoading && collections.length === 0 && ( +

+ No collections yet. Assign a file to a collection from the Knowledge Base table to create one. +

+ )} + +
    + {collections.map((name) => ( +
  • + {editingName === name ? ( + <> + setEditValue(e.target.value)} + className="flex-1 rounded border border-border-subtle bg-surface-primary px-2 py-1 text-sm text-text-primary" + /> + + renameMutation.mutate({ oldName: name, newName: editValue.trim() }) + } + > + Save + + setEditingName(null)}> + Cancel + + + ) : confirmDelete === name ? ( + <> + + Remove "{name}"? Files move to Uncategorized. + + deleteMutation.mutate(name)} + > + Confirm + + setConfirmDelete(null)}> + Cancel + + + ) : ( + <> + {name} + { + setEditingName(name) + setEditValue(name) + }} + > + Rename + + setConfirmDelete(name)} + > + Remove + + + )} +
    • + ))} +
+
+ + ) +} diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index cd47d04..942fb0a 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -1,5 +1,5 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useEffect, useRef, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import FileUploader from '~/components/file-uploader' import StyledButton from '~/components/StyledButton' import type { DynamicIconName } from '~/lib/icons' @@ -27,6 +27,9 @@ import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' import ActiveEmbedJobs from '~/components/ActiveEmbedJobs' import { SERVICE_NAMES } from '../../../constants/service_names' +import CollectionsManager from './CollectionsManager' +import { KB_COLLECTIONS } from '../../../constants/kb_collections' +import CollectionCombobox from './CollectionCombobox' interface KnowledgeBaseModalProps { aiAssistantName?: string @@ -34,7 +37,7 @@ interface KnowledgeBaseModalProps { } // File extensions the in-browser viewer can render. Must stay in sync with -// `RagService.VIEWABLE_TEXT_EXTENSIONS` — anything outside this set falls back +// `RagService.VIEWABLE_TEXT_EXTENSIONS` -- anything outside this set falls back // to Download. const VIEWABLE_EXTENSIONS = new Set(['md', 'txt', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'html']) @@ -117,7 +120,7 @@ type RowAction = /** * Pick the single adaptive per-row action button. Returns null when no action * makes sense for the current state (e.g. healthy indexed file with no - * warnings — bulk Re-embed All covers that case). `hasWarnings` lets us + * warnings -- bulk Re-embed All covers that case). `hasWarnings` lets us * surface a Re-embed affordance specifically when a file *looks* indexed but * has zero chunks or a stalled-mid-ingestion warning attached. */ @@ -143,6 +146,9 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o const { addNotification } = useNotifications() const [files, setFiles] = useState([]) const [isUploading, setIsUploading] = useState(false) + const [uploadCollection, setUploadCollection] = useState('') + const [collectionFilter, setCollectionFilter] = useState('All') + const [manageCollectionsOpen, setManageCollectionsOpen] = useState(false) const [confirmDeleteSource, setConfirmDeleteSource] = useState(null) const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null) const [bulkMode, setBulkMode] = useState(null) @@ -172,8 +178,18 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o select: (data) => data || [], }) - // Per-file conditional warnings (RFC #883 §6). `ok: false` means the - // computation itself failed (Qdrant/DB/FS) — distinct from `ok: true` with + const { data: knownCollections = [] } = useQuery({ + queryKey: ['kbCollections'], + queryFn: () => api.getKnowledgeCollections(), + select: (data) => data?.collections ?? [], + }) + + const comboboxOptions = useMemo(() => { + return Array.from(new Set([...KB_COLLECTIONS, ...knownCollections])).sort() + }, [knownCollections]) + + // Per-file conditional warnings (RFC #883 section 6). `ok: false` means the + // computation itself failed (Qdrant/DB/FS) -- distinct from `ok: true` with // an empty map, which means everything is healthy. We surface the failure // explicitly so a silent backend failure doesn't masquerade as health. const { data: warningsResult } = useQuery({ @@ -216,7 +232,20 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }) const uploadMutation = useMutation({ - mutationFn: (file: File) => api.uploadDocument(file), + mutationFn: (file: File) => api.uploadDocument(file, uploadCollection || undefined), + }) + + const updateCollectionMutation = useMutation({ + mutationFn: ({ source, collection }: { source: string; collection: string }) => + api.updateFileCollection(source, collection || null), + onSuccess: (data) => { + addNotification({ type: 'success', message: data?.message || 'Collection updated.' }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['kbCollections'] }) + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to update collection.' }) + }, }) const deleteMutation = useMutation({ @@ -461,7 +490,16 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o setFiles(Array.from(uploadedFiles)) }} /> -
+
+ Clean Up Failed - {/* Not gated on qdrantOffline: clearing stuck jobs must work during - a Qdrant/Ollama outage, which is exactly when they wedge. */}
+ + setManageCollectionsOpen(true)} + > + Manage Collections + — } @@ -725,13 +780,32 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o ) }, }, + { + accessor: 'collection', + title: 'Collection', + className: 'whitespace-nowrap', + render(record) { + if (record.bucket === 'admin_docs') { + return — + } + const isSaving = + updateCollectionMutation.isPending && + updateCollectionMutation.variables?.source === record.source + return ( + updateCollectionMutation.mutate({ source: record.source, collection: val })} + options={comboboxOptions} + disabled={isSaving} + className="w-40" + /> + ) + }, + }, { accessor: 'source', title: '', render(record) { - // Admin docs are auto-discovered and managed by NOMAD itself — - // deleting one would just be re-embedded on the next sync, so - // we surface them as informational only and hide Delete. if (record.bucket === 'admin_docs') { return (
@@ -827,7 +901,12 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }, ]} - data={groupAndSortKbFiles(storedFiles, sort)} + data={groupAndSortKbFiles( + collectionFilter === 'All' + ? storedFiles + : storedFiles.filter((f) => f.collection === collectionFilter), + sort + )} loading={isLoadingFiles} />
@@ -966,6 +1045,10 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o onClose={() => setViewerSource(null)} /> )} + + {manageCollectionsOpen && ( + setManageCollectionsOpen(false)} /> + )}
) } @@ -982,7 +1065,7 @@ function FileViewerModal({ source, onClose }: { source: string; onClose: () => v const fallbackName = source.split(/[/\\]/).at(-1) ?? source const title = data?.fileName ?? fallbackName // `catchInternal` swallows errors and resolves to undefined, surfacing a - // toast — so the "couldn't load" branch is gated on a finished-but-empty + // toast -- so the "couldn't load" branch is gated on a finished-but-empty // fetch rather than on react-query's `isError`. const showError = isFetched && !data diff --git a/admin/inertia/components/chat/index.tsx b/admin/inertia/components/chat/index.tsx index 9be60d4..928db00 100644 --- a/admin/inertia/components/chat/index.tsx +++ b/admin/inertia/components/chat/index.tsx @@ -33,6 +33,7 @@ export default function Chat({ const [activeSessionId, setActiveSessionId] = useState(null) const [messages, setMessages] = useState([]) const [selectedModel, setSelectedModel] = useState('') + const [collectionFilter, setCollectionFilter] = useState('') const [pendingModelSwitch, setPendingModelSwitch] = useState(null) const pageLoadNormalizedRef = useRef(false) const [isStreamingResponse, setIsStreamingResponse] = useState(false) @@ -72,6 +73,12 @@ export default function Chat({ select: (data) => data || [], }) + const { data: knownCollections = [] } = useQuery({ + queryKey: ['kbCollections'], + queryFn: () => api.getKnowledgeCollections(), + select: (data) => data?.collections ?? [], + }) + const { data: chatSuggestions, isLoading: chatSuggestionsLoading } = useQuery({ queryKey: ['chatSuggestions'], queryFn: async ({ signal }) => { @@ -102,6 +109,7 @@ export default function Chat({ model: string messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> sessionId?: number + collection?: string }) => api.sendChatMessage({ ...request, stream: false }), onSuccess: async (data) => { if (!data || !activeSessionId) { @@ -323,7 +331,13 @@ export default function Chat({ try { await api.streamChatMessage( - { model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, sessionId: sessionId ? Number(sessionId) : undefined }, + { + model: selectedModel || 'llama3.2', + messages: chatMessages, + stream: true, + sessionId: sessionId ? Number(sessionId) : undefined, + collection: collectionFilter || undefined, + }, (chunkContent, chunkThinking, done) => { if (chunkThinking.length > 0 && thinkingStartTime === null) { thinkingStartTime = Date.now() @@ -414,10 +428,11 @@ export default function Chat({ model: selectedModel || 'llama3.2', messages: chatMessages, sessionId: sessionId ? Number(sessionId) : undefined, + collection: collectionFilter || undefined, }) } }, - [activeSessionId, messages, selectedModel, chatMutation, queryClient, streamingEnabled] + [activeSessionId, messages, selectedModel, collectionFilter, chatMutation, queryClient, streamingEnabled] ) return ( @@ -471,6 +486,22 @@ export default function Chat({ {remoteStatus?.connected === false ? 'Remote Disconnected' : 'Remote Connected'} )} +
+ + +