RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(rag): add subject/collection organization to knowledge base 

- Add nullable collection field to KbIngestState, propagated through the
  embed job, RAG service, and Qdrant point payloads (indexed for filtering)
- Add upload-time category selection and per-file collection reassignment
  in the Knowledge Base modal, with a filterable Stored Files table
- Add a 'Search in' collection filter to the chat interface, threaded
  through to searchSimilarDocuments as an optional Qdrant filter
- Fix .docx extraction: previously routed through raw-text extraction
  (garbage output for a ZIP-based XML format); adds a proper mammoth-based
  extractor and a dedicated 'docx' file-type case
This commit is contained in:
John Cortright 2026-07-03 14:16:01 -07:00
parent 6a4f02dd46
commit cae3dde8b4
19 changed files with 457 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -41,3 +41,5 @@ admin/public/assets
# Admin specific development files
admin/storage
nomad_admin_config.json
nomad_admin_hostconfig.json

5
admin/app/controllers/ollama_controller.ts
View File

@ -77,10 +77,12 @@ export default class OllamaController {
logger.debug(`[OllamaController] Rewritten query for RAG: "${rewrittenQuery}"`)
if (rewrittenQuery) {
const collectionFilter: string | null = request.input('collection', null)
const relevantDocs = await this.ragService.searchSimilarDocuments(
rewrittenQuery,
5, // Top 5 most relevant chunks
0.3 // Minimum similarity score of 0.3
0.3, // Minimum similarity score of 0.3
collectionFilter ?? undefined
)
logger.debug(`[RAG] Retrieved ${relevantDocs.length} relevant documents for query: "${rewrittenQuery}"`)
@ -456,3 +458,4 @@ export default class OllamaController {
}
}
}

28
admin/app/controllers/rag_controller.ts
View File

@ -20,6 +20,8 @@ export default class RagController {
return response.status(400).json({ error: 'No file uploaded' })
}
const collection: string | null = request.input('collection', null)
const randomSuffix = randomBytes(6).toString('hex')
const sanitizedName = sanitizeFilename(uploadedFile.clientName)
@ -34,6 +36,7 @@ export default class RagController {
const result = await EmbedFileJob.dispatch({
filePath: fullPath,
fileName,
...(collection ? { collection } : {}),
})
return response.status(202).json({
@ -42,9 +45,9 @@ export default class RagController {
fileName,
filePath: `/${RagService.UPLOADS_STORAGE_PATH}/${fileName}`,
alreadyProcessing: !result.created,
...(collection ? { collection } : {}),
})
}
public async getActiveJobs({ response }: HttpContext) {
const jobs = await EmbedFileJob.listActiveJobs()
return response.status(200).json(jobs)
@ -68,6 +71,29 @@ export default class RagController {
return response.status(200).json({ files })
}
public async getKnowledgeCollections({ response }: HttpContext) {
const collections = await this.ragService.getKnowledgeCollections()
return response.status(200).json({ collections })
}
public async updateFileCollection({ request, response }: HttpContext) {
const source: string | null = request.input('source', null)
// Empty string means "clear back to Uncategorized" — coerce to null rather
// than rejecting, so the frontend can offer an Uncategorized option.
const rawCollection: string | null = request.input('collection', null)
const collection = rawCollection && rawCollection.trim() !== '' ? rawCollection : null
if (!source) {
return response.status(400).json({ error: 'source is required.' })
}
const result = await this.ragService.updateFileCollection(source, collection)
if (!result.success) {
return response.status(500).json({ error: result.message })
}
return response.status(200).json({ message: result.message })
}
public async getFileWarnings({ response }: HttpContext) {
const result = await this.ragService.computeFileWarnings()
return response.status(200).json(result)

12
admin/app/jobs/embed_file_job.ts
View File

@ -18,11 +18,8 @@ export interface EmbedFileJobParams {
batchOffset?: number // Current batch offset (for ZIM files)
totalArticles?: number // Total articles in ZIM (for progress tracking)
isFinalBatch?: boolean // Whether this is the last batch (prevents premature deletion)
// Running total of chunks embedded across prior batches in this dispatch chain.
// Carried forward so the final batch can persist an accurate `chunks_embedded`
// count via KbIngestState.markIndexed (see #933 — without this, only the last
// batch's chunk count was stored while Qdrant held the full set).
chunksSoFar?: number
collection?: string
}
export class EmbedFileJob {
@ -56,7 +53,7 @@ export class EmbedFileJob {
}
async handle(job: Job) {
const { filePath, fileName, batchOffset, totalArticles } = job.data as EmbedFileJobParams
const { filePath, fileName, batchOffset, totalArticles, collection } = job.data as EmbedFileJobParams
const isZimBatch = batchOffset !== undefined
const batchInfo = isZimBatch ? ` (batch offset: ${batchOffset})` : ''
@ -136,7 +133,8 @@ export class EmbedFileJob {
filePath,
allowDeletion,
batchOffset,
onProgress
onProgress,
collection
)
if (!result.success) {
@ -242,7 +240,7 @@ export class EmbedFileJob {
// BullMQ's :completed retention (50 jobs) ages out, so the state row is
// the only durable record of "this file finished embedding".
try {
await KbIngestState.markIndexed(filePath, totalChunks)
await KbIngestState.markIndexed(filePath, totalChunks, collection)
} catch (stateErr) {
logger.warn(
`[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`,

30
admin/app/models/kb_ingest_state.ts
View File

@ -28,6 +28,9 @@ export default class KbIngestState extends BaseModel {
@column()
declare chunks_embedded: number
@column()
declare collection: string | null
@column()
declare last_error: string | null
@ -37,20 +40,21 @@ export default class KbIngestState extends BaseModel {
@column.dateTime({ autoCreate: true, autoUpdate: true })
declare updated_at: DateTime
static async getOrCreate(filePath: string): Promise<KbIngestState> {
return this.firstOrCreate(
{ file_path: filePath },
{ file_path: filePath, state: 'pending_decision', chunks_embedded: 0 }
)
}
static async getOrCreate(filePath: string, collection?: string): Promise<KbIngestState> {
return this.firstOrCreate(
{ file_path: filePath },
{ file_path: filePath, state: 'pending_decision', chunks_embedded: 0, collection: collection ?? null }
)
}
static async markIndexed(filePath: string, chunksEmbedded: number): Promise<void> {
const row = await this.getOrCreate(filePath)
row.state = 'indexed'
row.chunks_embedded = chunksEmbedded
row.last_error = null
await row.save()
}
static async markIndexed(filePath: string, chunksEmbedded: number, collection?: string): Promise<void> {
const row = await this.getOrCreate(filePath, collection)
row.state = 'indexed'
row.chunks_embedded = chunksEmbedded
row.last_error = null
if (collection) row.collection = collection
await row.save()
}
static async markFailed(filePath: string, errorMessage: string): Promise<void> {
const row = await this.getOrCreate(filePath)

89
admin/app/services/rag_service.ts
View File

@ -10,6 +10,7 @@ import { createWorker } from 'tesseract.js'
import { fromBuffer } from 'pdf2pic'
import JSZip from 'jszip'
import * as cheerio from 'cheerio'
import mammoth from 'mammoth'
import { OllamaService } from './ollama_service.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { removeStopwords } from 'stopword'
@ -133,6 +134,10 @@ export class RagService {
field_name: 'content_type',
field_schema: 'keyword',
})
await this.qdrant!.createPayloadIndex(collectionName, {
field_name: 'collection',
field_schema: 'keyword',
})
} catch (error) {
logger.error('Error ensuring Qdrant collection:', error)
throw error
@ -611,6 +616,16 @@ export class RagService {
return await this.extractTXTText(fileBuffer)
}
/**
* Extract text content from a DOCX file using mammoth. DOCX is a ZIP-based
* XML format, so raw-text extraction (extractTXTText) would return garbage —
* this parses the document XML properly and returns clean plain text.
*/
private async processDocxFile(fileBuffer: Buffer): Promise<string> {
const { value: text } = await mammoth.extractRawText({ buffer: fileBuffer })
return text
}
/**
* Extract text content from an EPUB file.
* EPUBs are ZIP archives containing XHTML content files.
@ -695,14 +710,16 @@ export class RagService {
extractedText: string,
filepath: string,
deleteAfterEmbedding: boolean = false,
onProgress?: (percent: number) => Promise<void>
onProgress?: (percent: number) => Promise<void>,
collection?: string
): Promise<{ success: boolean; message: string; chunks?: number }> {
if (!extractedText || extractedText.trim().length === 0) {
return { success: false, message: 'Process completed succesfully, but no text was found to embed.' }
}
const embedResult = await this.embedAndStoreText(extractedText, {
source: filepath
source: filepath,
...(collection ? { collection } : {})
}, onProgress)
if (!embedResult) {
@ -732,7 +749,8 @@ export class RagService {
filepath: string,
deleteAfterEmbedding: boolean = false,
batchOffset?: number,
onProgress?: (percent: number) => Promise<void>
onProgress?: (percent: number) => Promise<void>,
collection?: string
): Promise<ProcessAndEmbedFileResponse> {
try {
const fileType = determineFileType(filepath)
@ -765,6 +783,9 @@ export class RagService {
case 'pdf':
extractedText = await this.processPDFFile(fileBuffer!)
break
case 'docx':
extractedText = await this.processDocxFile(fileBuffer!)
break
case 'epub':
extractedText = await this.processEPUBFile(fileBuffer!)
break
@ -781,7 +802,7 @@ export class RagService {
: undefined
// Embed extracted text and cleanup
return await this.embedTextAndCleanup(extractedText, filepath, deleteAfterEmbedding, scaledProgress)
return await this.embedTextAndCleanup(extractedText, filepath, deleteAfterEmbedding, scaledProgress, collection)
} catch (error) {
logger.error('[RAG] Error processing and embedding file:', error)
return { success: false, message: 'Error processing and embedding file.' }
@ -800,7 +821,8 @@ export class RagService {
public async searchSimilarDocuments(
query: string,
limit: number = 5,
scoreThreshold: number = 0.3 // Lower default threshold - was 0.7, now 0.3
scoreThreshold: number = 0.3, // Lower default threshold - was 0.7, now 0.3
collection?: string
): Promise<Array<{ text: string; score: number; metadata?: Record<string, any> }>> {
try {
logger.debug(`[RAG] Starting similarity search for query: "${query}"`)
@ -873,6 +895,7 @@ export class RagService {
limit: searchLimit,
score_threshold: scoreThreshold,
with_payload: true,
...(collection ? { filter: { must: [{ key: 'collection', match: { value: collection } }] } } : {}),
})
logger.debug(`[RAG] Found ${searchResults.length} results above threshold ${scoreThreshold}`)
@ -1121,14 +1144,15 @@ export class RagService {
// in particular) have no row to attach to. The state machine is the
// authoritative "what's on disk?" view; Qdrant is "what made it into
// the vector store?". Both are needed to render the KB UI honestly.
const stateByPath = new Map<string, { state: KbIngestStateValue; chunks_embedded: number }>()
const stateByPath = new Map<string, { state: KbIngestStateValue; chunks_embedded: number; collection: string | null }>()
try {
const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded')
const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded', 'collection')
for (const row of stateRows) {
sources.add(row.file_path)
stateByPath.set(row.file_path, {
state: row.state,
chunks_embedded: row.chunks_embedded,
collection: row.collection,
})
}
} catch (error) {
@ -1155,6 +1179,7 @@ export class RagService {
size: stats?.size ?? null,
uploadedAt: stats?.modifiedTime.toISOString() ?? null,
isUserUpload,
collection: row?.collection ?? null,
}
})
)
@ -1164,6 +1189,56 @@ export class RagService {
}
}
/**
* Enumerate distinct `collection` values currently in the knowledge base,
* for populating a subject-picker in the upload/chat UI. Mirrors the
* `source` facet pattern used elsewhere in this file (see getStoredFiles).
*/
public async getKnowledgeCollections(): Promise<string[]> {
await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
const facetResult = await this.qdrant!.facet(RagService.CONTENT_COLLECTION_NAME, {
key: 'collection',
limit: RagService.FACET_SOURCE_LIMIT,
exact: true,
})
const collections = new Set<string>()
for (const hit of facetResult.hits) {
if (typeof hit.value === 'string') collections.add(hit.value)
}
return Array.from(collections).sort()
}
/**
* Reassign a stored file's collection after the fact. Updates the `collection`
* payload field on every existing Qdrant point for this source in place (no
* re-chunking or re-embedding needed), then mirrors the change onto the
* KbIngestState row so getStoredFiles() reflects it immediately.
*/
public async updateFileCollection(
source: string,
collection: string | null
): Promise<{ success: boolean; message: string }> {
try {
await this._ensureCollection(RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION)
await this.qdrant!.setPayload(RagService.CONTENT_COLLECTION_NAME, {
payload: { collection },
filter: { must: [{ key: 'source', match: { value: source } }] },
})
const row = await KbIngestState.query().where('file_path', source).first()
if (row) {
row.collection = collection
await row.save()
}
return { success: true, message: collection ? `Moved to "${collection}".` : 'Moved to Uncategorized.' }
} catch (error) {
logger.error('[RAG] Error updating file collection:', error)
return { success: false, message: 'Error updating file collection.' }
}
}
/**
* Resolve a stored-file `source` to an absolute disk path, but only if the
* path lives under the uploads directory. Mirrors the docs_service traversal

6
admin/app/utils/fs.ts
View File

@ -190,13 +190,15 @@ export function matchesDevice(fsPath: string, deviceName: string): boolean {
return false
}
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'epub' | 'zim' | 'unknown' {
export function determineFileType(filename: string): 'image' | 'pdf' | 'text' | 'docx' | 'epub' | 'zim' | 'unknown' {
const ext = path.extname(filename).toLowerCase()
if (['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'].includes(ext)) {
return 'image'
} else if (ext === '.pdf') {
return 'pdf'
} else if (['.txt', '.md', '.docx', '.rtf'].includes(ext)) {
} else if (ext === '.docx') {
return 'docx'
} else if (['.txt', '.md', '.rtf'].includes(ext)) {
return 'text'
} else if (ext === '.epub') {
return 'epub'

13
admin/constants/kb_collections.ts Normal file
View File

@ -0,0 +1,13 @@
export const KB_COLLECTIONS = [
'food',
'water',
'shelter',
'electricity',
'survival',
'farming',
'computer',
'medical',
'boating',
] as const
export type KbCollection = (typeof KB_COLLECTIONS)[number]

17
admin/database/migrations/1776400000001_add_collection_to_kb_ingest_state.ts Normal file
View File

@ -0,0 +1,17 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'kb_ingest_state'
async up() {
this.schema.alterTable(this.tableName, (table) => {
table.string('collection').nullable().index()
})
}
async down() {
this.schema.alterTable(this.tableName, (table) => {
table.dropColumn('collection')
})
}
}

123
admin/inertia/components/chat/KnowledgeBaseModal.tsx
View File

@ -27,15 +27,13 @@ import { useModals } from '~/context/ModalContext'
import StyledModal from '../StyledModal'
import ActiveEmbedJobs from '~/components/ActiveEmbedJobs'
import { SERVICE_NAMES } from '../../../constants/service_names'
import { KB_COLLECTIONS } from '../../../constants/kb_collections'
interface KnowledgeBaseModalProps {
aiAssistantName?: string
onClose: () => void
}
// File extensions the in-browser viewer can render. Must stay in sync with
// `RagService.VIEWABLE_TEXT_EXTENSIONS` — anything outside this set falls back
// to Download.
const VIEWABLE_EXTENSIONS = new Set(['md', 'txt', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'html'])
function isViewableExtension(filename: string): boolean {
@ -69,13 +67,6 @@ function renderSortHeader(
)
}
/**
* Compact label for the per-row ingestion state. Files that exist in Qdrant
* with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883
* installs whose chunks are real, so we display them as "Indexed" rather than
* surfacing the absent-row detail. Admin-docs group has no pill (the "Managed
* by NOMAD" message in the action column carries the same signal).
*/
function renderStatePill(record: KbFileGroup): React.ReactNode {
if (record.bucket === 'admin_docs') return null
const effective: KbIngestStateValue = record.state ?? 'indexed'
@ -114,13 +105,6 @@ type RowAction =
| { kind: 'index'; label: string; force: boolean; variant: 'primary'; icon: DynamicIconName }
| { kind: 'reembed'; label: string; force: true; variant: 'secondary'; icon: DynamicIconName }
/**
* Pick the single adaptive per-row action button. Returns null when no action
* makes sense for the current state (e.g. healthy indexed file with no
* warnings — bulk Re-embed All covers that case). `hasWarnings` lets us
* surface a Re-embed affordance specifically when a file *looks* indexed but
* has zero chunks or a stalled-mid-ingestion warning attached.
*/
function pickRowAction(record: KbFileGroup, hasWarnings: boolean): RowAction | null {
if (record.bucket === 'admin_docs') return null
const effective: KbIngestStateValue = record.state ?? 'indexed'
@ -143,6 +127,8 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
const { addNotification } = useNotifications()
const [files, setFiles] = useState<File[]>([])
const [isUploading, setIsUploading] = useState(false)
const [uploadCollection, setUploadCollection] = useState<string>('')
const [collectionFilter, setCollectionFilter] = useState<string>('All')
const [confirmDeleteSource, setConfirmDeleteSource] = useState<string | null>(null)
const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null)
const [bulkMode, setBulkMode] = useState<null | 'reembed' | 'reset'>(null)
@ -172,10 +158,12 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
select: (data) => data || [],
})
// Per-file conditional warnings (RFC #883 §6). `ok: false` means the
// computation itself failed (Qdrant/DB/FS) — distinct from `ok: true` with
// an empty map, which means everything is healthy. We surface the failure
// explicitly so a silent backend failure doesn't masquerade as health.
const { data: knownCollections = [] } = useQuery({
queryKey: ['kbCollections'],
queryFn: () => api.getKnowledgeCollections(),
select: (data) => data?.collections ?? [],
})
const { data: warningsResult } = useQuery({
queryKey: ['kbFileWarnings'],
queryFn: () => api.getKbFileWarnings(),
@ -184,9 +172,6 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
const fileWarnings = warningsResult?.warnings ?? {}
const warningsUnavailable = warningsResult !== undefined && warningsResult.ok === false
// Global auto-index policy. KVStore returns `null` for an unset key, which
// we treat as 'Always' for backward compatibility with installs that predate
// this UI. The user can opt into Manual mode from the toggle below.
const { data: ingestPolicySetting } = useQuery({
queryKey: ['ingestPolicy'],
queryFn: () => api.getSetting('rag.defaultIngestPolicy'),
@ -216,7 +201,20 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
})
const uploadMutation = useMutation({
mutationFn: (file: File) => api.uploadDocument(file),
mutationFn: (file: File) => api.uploadDocument(file, uploadCollection || undefined),
})
const updateCollectionMutation = useMutation({
mutationFn: ({ source, collection }: { source: string; collection: string }) =>
api.updateFileCollection(source, collection || null),
onSuccess: (data) => {
addNotification({ type: 'success', message: data?.message || 'Collection updated.' })
queryClient.invalidateQueries({ queryKey: ['storedFiles'] })
queryClient.invalidateQueries({ queryKey: ['kbCollections'] })
},
onError: (error: any) => {
addNotification({ type: 'error', message: error?.message || 'Failed to update collection.' })
},
})
const deleteMutation = useMutation({
@ -461,7 +459,20 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
setFiles(Array.from(uploadedFiles))
}}
/>
<div className="flex justify-center gap-4 my-6">
<div className="flex justify-center items-center gap-4 my-6">
<label className="flex items-center gap-2 text-sm text-text-secondary">
Collection:
<select
value={uploadCollection}
onChange={(e) => setUploadCollection(e.target.value)}
className="rounded border border-border-subtle bg-surface-primary px-3 py-2 text-text-primary"
>
<option value="">Uncategorized</option>
{KB_COLLECTIONS.map((c) => (
<option key={c} value={c}>{c}</option>
))}
</select>
</label>
<StyledButton
variant="primary"
size="lg"
@ -585,8 +596,6 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
>
Clean Up Failed
</StyledButton>
{/* Not gated on qdrantOffline: clearing stuck jobs must work during
a Qdrant/Ollama outage, which is exactly when they wedge. */}
<StyledButton
variant="danger"
size="md"
@ -607,6 +616,19 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
<div className='flex items-center justify-between mb-6 gap-2 flex-wrap'>
<StyledSectionHeader title="Stored Knowledge Base Files" className='!mb-0' />
<div className="flex items-center gap-2 flex-wrap">
<label className="flex items-center gap-2 text-sm text-text-secondary">
Search in:
<select
value={collectionFilter}
onChange={(e) => setCollectionFilter(e.target.value)}
className="rounded border border-border-subtle bg-surface-primary px-3 py-2 text-text-primary"
>
<option value="All">All</option>
{KB_COLLECTIONS.map((c) => (
<option key={c} value={c}>{c}</option>
))}
</select>
</label>
<StyledButton
variant="danger"
size="md"
@ -701,8 +723,6 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
title: renderSortHeader('Size', 'size', sort, setSort),
className: 'whitespace-nowrap',
render(record) {
// The collapsed admin_docs group has no single size — leave blank
// rather than misleadingly summing across N files.
if (record.bucket === 'admin_docs' || record.size === null) {
return <span className="text-text-muted">—</span>
}
@ -725,13 +745,38 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
)
},
},
{
accessor: 'collection',
title: 'Collection',
className: 'whitespace-nowrap',
render(record) {
if (record.bucket === 'admin_docs') {
return <span className="text-text-muted">—</span>
}
const isSaving =
updateCollectionMutation.isPending &&
updateCollectionMutation.variables?.source === record.source
return (
<select
value={record.collection ?? ''}
disabled={isSaving}
onChange={(e) =>
updateCollectionMutation.mutate({ source: record.source, collection: e.target.value })
}
className="rounded border border-border-subtle bg-surface-primary px-2 py-1 text-xs text-text-primary"
>
<option value="">Uncategorized</option>
{KB_COLLECTIONS.map((c) => (
<option key={c} value={c}>{c}</option>
))}
</select>
)
},
},
{
accessor: 'source',
title: '',
render(record) {
// Admin docs are auto-discovered and managed by NOMAD itself —
// deleting one would just be re-embedded on the next sync, so
// we surface them as informational only and hide Delete.
if (record.bucket === 'admin_docs') {
return (
<div className="flex justify-end">
@ -827,7 +872,12 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
},
},
]}
data={groupAndSortKbFiles(storedFiles, sort)}
data={groupAndSortKbFiles(
collectionFilter === 'All'
? storedFiles
: storedFiles.filter((f) => f.collection === collectionFilter),
sort
)}
loading={isLoadingFiles}
/>
</div>
@ -977,13 +1027,8 @@ function FileViewerModal({ source, onClose }: { source: string; onClose: () => v
staleTime: 60_000,
})
// Title falls back to the trailing path segment so the modal still has a
// useful header while the fetch is in-flight or if it failed.
const fallbackName = source.split(/[/\\]/).at(-1) ?? source
const title = data?.fileName ?? fallbackName
// `catchInternal` swallows errors and resolves to undefined, surfacing a
// toast — so the "couldn't load" branch is gated on a finished-but-empty
// fetch rather than on react-query's `isError`.
const showError = isFetched && !data
return (

30
admin/inertia/components/chat/index.tsx
View File

@ -11,6 +11,7 @@ import { ChatMessage } from '../../../types/chat'
import classNames from '~/lib/classNames'
import { IconX } from '@tabler/icons-react'
import { DEFAULT_QUERY_REWRITE_MODEL } from '../../../constants/ollama'
import { KB_COLLECTIONS } from '../../../constants/kb_collections'
import { useSystemSetting } from '~/hooks/useSystemSetting'
interface ChatProps {
@ -33,6 +34,7 @@ export default function Chat({
const [activeSessionId, setActiveSessionId] = useState<string | null>(null)
const [messages, setMessages] = useState<ChatMessage[]>([])
const [selectedModel, setSelectedModel] = useState<string>('')
const [collectionFilter, setCollectionFilter] = useState<string>('')
const [pendingModelSwitch, setPendingModelSwitch] = useState<string | null>(null)
const pageLoadNormalizedRef = useRef(false)
const [isStreamingResponse, setIsStreamingResponse] = useState(false)
@ -102,6 +104,7 @@ export default function Chat({
model: string
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>
sessionId?: number
collection?: string
}) => api.sendChatMessage({ ...request, stream: false }),
onSuccess: async (data) => {
if (!data || !activeSessionId) {
@ -323,7 +326,13 @@ export default function Chat({
try {
await api.streamChatMessage(
{ model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, sessionId: sessionId ? Number(sessionId) : undefined },
{
model: selectedModel || 'llama3.2',
messages: chatMessages,
stream: true,
sessionId: sessionId ? Number(sessionId) : undefined,
collection: collectionFilter || undefined,
},
(chunkContent, chunkThinking, done) => {
if (chunkThinking.length > 0 && thinkingStartTime === null) {
thinkingStartTime = Date.now()
@ -414,10 +423,11 @@ export default function Chat({
model: selectedModel || 'llama3.2',
messages: chatMessages,
sessionId: sessionId ? Number(sessionId) : undefined,
collection: collectionFilter || undefined,
})
}
},
[activeSessionId, messages, selectedModel, chatMutation, queryClient, streamingEnabled]
[activeSessionId, messages, selectedModel, collectionFilter, chatMutation, queryClient, streamingEnabled]
)
return (
@ -471,6 +481,22 @@ export default function Chat({
{remoteStatus?.connected === false ? 'Remote Disconnected' : 'Remote Connected'}
</span>
)}
<div className="flex items-center gap-2">
<label htmlFor="collection-select" className="text-sm text-text-secondary">
Search in:
</label>
<select
id="collection-select"
value={collectionFilter}
onChange={(e) => setCollectionFilter(e.target.value)}
className="px-3 py-1.5 border border-border-default rounded-lg text-sm focus:outline-none focus:ring-2 focus:ring-desert-green focus:border-transparent bg-surface-primary"
>
<option value="">All</option>
{KB_COLLECTIONS.map((c) => (
<option key={c} value={c}>{c}</option>
))}
</select>
</div>
<div className="flex items-center gap-2">
<label htmlFor="model-select" className="text-sm text-text-secondary">
Model:

70
admin/inertia/lib/api.ts
View File

@ -124,9 +124,10 @@ class API {
async downloadRemoteMapRegionPreflight(url: string) {
return catchInternal(async () => {
const response = await this.client.post<
{ filename: string; size: number } | { message: string }
>('/maps/download-remote-preflight', { url })
const response = await this.client.post<{ filename: string; size: number } | { message: string }>(
'/maps/download-remote-preflight',
{ url }
)
return response.data
})()
}
@ -272,14 +273,6 @@ class API {
})()
}
/**
* Ask the backend to send Ollama `keep_alive: 0` to every currently-loaded
* chat model except `targetModel` (and the embedding model, which is always
* exempt server-side). Fire-and-forget — the chat UI doesn't await this
* before creating a new session, since unload is housekeeping.
*
* Pass `null` to unload every chat model.
*/
async unloadChatModels(targetModel: string | null) {
return catchInternal(async () => {
const response = await this.client.post<{ unloaded: string[] }>(
@ -314,7 +307,6 @@ class API {
onChunk: (content: string, thinking: string, done: boolean) => void,
signal?: AbortSignal
): Promise<void> {
// Axios doesn't support ReadableStream in browser, so need to use fetch
const response = await fetch('/api/ollama/chat', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@ -344,7 +336,7 @@ class API {
let data: any
try {
data = JSON.parse(line.slice(6))
} catch { continue /* skip malformed chunks */ }
} catch { continue }
if (data.error) throw new Error('The model encountered an error. Please try again.')
@ -376,15 +368,13 @@ class API {
async getChatSessions() {
return catchInternal(async () => {
const response = await this.client.get<
Array<{
id: string
title: string
model: string | null
timestamp: string
lastMessage: string | null
}>
>('/chat/sessions')
const response = await this.client.get<Array<{
id: string
title: string
model: string | null
timestamp: string
lastMessage: string | null
}>>('/chat/sessions')
return response.data
})()
}
@ -713,27 +703,21 @@ class API {
async listMapMarkers() {
return catchInternal(async () => {
const response = await this.client.get<
Array<{ id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }>
>('/maps/markers')
const response = await this.client.get<Array<{ id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }>>('/maps/markers')
return response.data
})()
}
async createMapMarker(data: { name: string; longitude: number; latitude: number; color?: string }) {
return catchInternal(async () => {
const response = await this.client.post<
{ id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }
>('/maps/markers', data)
const response = await this.client.post<{ id: number; name: string; longitude: number; latitude: number; color: string; notes: string | null; created_at: string }>('/maps/markers', data)
return response.data
})()
}
async updateMapMarker(id: number, data: { name?: string; color?: string }) {
return catchInternal(async () => {
const response = await this.client.patch<
{ id: number; name: string; longitude: number; latitude: number; color: string }
>(`/maps/markers/${id}`, data)
const response = await this.client.patch<{ id: number; name: string; longitude: number; latitude: number; color: string }>(`/maps/markers/${id}`, data)
return response.data
})()
}
@ -872,13 +856,11 @@ class API {
const response = await this.client.post<SubmitBenchmarkResponse>('/benchmark/submit', { benchmark_id, anonymous })
return response.data
} catch (error: any) {
// For 409 Conflict errors, throw a specific error that the UI can handle
if (error.response?.status === 409) {
const err = new Error(error.response?.data?.error || 'This benchmark has already been submitted to the repository')
; (err as any).status = 409
throw err
}
// For other errors, extract the message and throw
const errorMessage = error.response?.data?.error || error.message || 'Failed to submit benchmark'
throw new Error(errorMessage)
}
@ -952,8 +934,6 @@ class API {
})()
}
// Wikipedia selector methods
async getWikipediaState(): Promise<WikipediaState | undefined> {
return catchInternal(async () => {
const response = await this.client.get<WikipediaState>('/zim/wikipedia')
@ -984,10 +964,11 @@ class API {
})()
}
async uploadDocument(file: File) {
async uploadDocument(file: File, collection?: string) {
return catchInternal(async () => {
const formData = new FormData()
formData.append('file', file)
if (collection) formData.append('collection', collection)
const response = await this.client.post<{ message: string; file_path: string }>(
'/rag/upload',
formData,
@ -1001,6 +982,23 @@ class API {
})()
}
async getKnowledgeCollections() {
return catchInternal(async () => {
const response = await this.client.get<{ collections: string[] }>('/rag/collections')
return response.data
})()
}
async updateFileCollection(source: string, collection: string | null) {
return catchInternal(async () => {
const response = await this.client.post<{ message: string }>('/rag/update-collection', {
source,
collection,
})
return response.data
})()
}
async getSetting(key: string) {
return catchInternal(async () => {
const response = await this.client.get<{ key: string; value: any }>(

5
admin/inertia/lib/kb_file_grouping.ts
View File

@ -61,6 +61,9 @@ export interface KbFileGroup {
/** True when the row corresponds to a user upload — drives whether the
* view/download buttons render. False for the collapsed admin_docs group. */
isUserUpload: boolean
/** Subject/category tag, or null if uncategorized. Always null for the
* collapsed admin_docs group. */
collection: string | null
}
const BUCKET_SORT_ORDER: KbFileBucket[] = ['zim', 'upload', 'admin_docs', 'other']
@ -136,6 +139,7 @@ export function groupAndSortKbFiles(
size: null,
uploadedAt: null,
isUserUpload: false,
collection: null,
})
continue
}
@ -152,6 +156,7 @@ export function groupAndSortKbFiles(
size: file.size,
uploadedAt: file.uploadedAt,
isUserUpload: file.isUserUpload,
collection: file.collection,
})
}
}

259
admin/package-lock.json generated
View File

@ -52,6 +52,7 @@
"ipaddr.js": "2.4.0",
"jszip": "3.10.1",
"luxon": "3.7.2",
"mammoth": "^1.12.0",
"maplibre-gl": "4.7.1",
"mysql2": "3.22.5",
"ollama": "0.6.3",
@ -2230,9 +2231,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2249,9 +2247,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2268,9 +2263,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2287,9 +2279,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2306,9 +2295,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2325,9 +2311,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2344,9 +2327,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2363,9 +2343,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@ -2382,9 +2359,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2407,9 +2381,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2432,9 +2403,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2457,9 +2425,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2482,9 +2447,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2507,9 +2469,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2532,9 +2491,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -2557,9 +2513,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@ -3435,9 +3388,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -3454,9 +3404,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -3473,9 +3420,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -3492,9 +3436,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -3511,9 +3452,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4199,9 +4137,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4215,9 +4150,6 @@
"cpu": [
"arm"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4231,9 +4163,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4247,9 +4176,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4263,9 +4189,6 @@
"cpu": [
"loong64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4279,9 +4202,6 @@
"cpu": [
"loong64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4295,9 +4215,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4311,9 +4228,6 @@
"cpu": [
"ppc64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4327,9 +4241,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4343,9 +4254,6 @@
"cpu": [
"riscv64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4359,9 +4267,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4375,9 +4280,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4391,9 +4293,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4590,6 +4489,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4606,6 +4506,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4622,6 +4523,7 @@
"cpu": [
"arm"
],
"dev": true,
"license": "Apache-2.0",
"optional": true,
"os": [
@ -4638,9 +4540,7 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4657,9 +4557,7 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4676,9 +4574,7 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4695,9 +4591,7 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4714,6 +4608,7 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4730,6 +4625,7 @@
"cpu": [
"ia32"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4746,6 +4642,7 @@
"cpu": [
"x64"
],
"dev": true,
"license": "Apache-2.0 AND MIT",
"optional": true,
"os": [
@ -4938,9 +4835,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4957,9 +4851,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -4976,9 +4867,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@ -4995,9 +4883,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@ -6347,6 +6232,15 @@
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
}
},
"node_modules/@xmldom/xmldom": {
"version": "0.8.13",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz",
"integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/abbrev": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz",
@ -6806,6 +6700,12 @@
"readable-stream": "^3.4.0"
}
},
"node_modules/bluebird": {
"version": "3.4.7",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
"integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==",
"license": "MIT"
},
"node_modules/bmp-js": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
@ -8069,6 +7969,12 @@
"node": ">=0.3.1"
}
},
"node_modules/dingbat-to-unicode": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz",
"integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==",
"license": "BSD-2-Clause"
},
"node_modules/dlv": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
@ -8175,6 +8081,15 @@
"url": "https://dotenvx.com"
}
},
"node_modules/duck": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz",
"integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==",
"license": "BSD",
"dependencies": {
"underscore": "^1.13.1"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@ -11240,9 +11155,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11263,9 +11175,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11286,9 +11195,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11309,9 +11215,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MPL-2.0",
"optional": true,
"os": [
@ -11491,6 +11394,17 @@
"loose-envify": "cli.js"
}
},
"node_modules/lop": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz",
"integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==",
"license": "BSD-2-Clause",
"dependencies": {
"duck": "^0.1.12",
"option": "~0.2.1",
"underscore": "^1.13.1"
}
},
"node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
@ -11570,6 +11484,39 @@
"node": ">= 0.6"
}
},
"node_modules/mammoth": {
"version": "1.12.0",
"resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz",
"integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==",
"license": "BSD-2-Clause",
"dependencies": {
"@xmldom/xmldom": "^0.8.6",
"argparse": "~1.0.3",
"base64-js": "^1.5.1",
"bluebird": "~3.4.0",
"dingbat-to-unicode": "^1.0.1",
"jszip": "^3.7.1",
"lop": "^0.4.2",
"path-is-absolute": "^1.0.0",
"underscore": "^1.13.1",
"xmlbuilder": "^10.0.0"
},
"bin": {
"mammoth": "bin/mammoth"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/mammoth/node_modules/argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"license": "MIT",
"dependencies": {
"sprintf-js": "~1.0.2"
}
},
"node_modules/maplibre-gl": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz",
@ -13284,6 +13231,12 @@
"opencollective-postinstall": "index.js"
}
},
"node_modules/option": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz",
"integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==",
"license": "BSD-2-Clause"
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@ -13628,6 +13581,15 @@
"node": ">=14.0.0"
}
},
"node_modules/path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@ -15641,6 +15603,12 @@
"node": ">= 10.x"
}
},
"node_modules/sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
"license": "BSD-3-Clause"
},
"node_modules/sql-escaper": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
@ -16627,6 +16595,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/underscore": {
"version": "1.13.8",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz",
"integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==",
"license": "MIT"
},
"node_modules/undici": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-6.26.0.tgz",
@ -17258,6 +17232,15 @@
"node": ">=16.0.0"
}
},
"node_modules/xmlbuilder": {
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz",
"integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==",
"license": "MIT",
"engines": {
"node": ">=4.0"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

1
admin/package.json
View File

@ -105,6 +105,7 @@
"ipaddr.js": "2.4.0",
"jszip": "3.10.1",
"luxon": "3.7.2",
"mammoth": "^1.12.0",
"maplibre-gl": "4.7.1",
"mysql2": "3.22.5",
"ollama": "0.6.3",

2
admin/start/routes.ts
View File

@ -161,6 +161,8 @@ router
router.post('/estimate-batch', [RagController, 'estimateBatch'])
router.get('/policy-prompt-state', [RagController, 'policyPromptState'])
router.get('/health', [RagController, 'health'])
router.get('/collections', [RagController, 'getKnowledgeCollections'])
router.post('/update-collection', [RagController, 'updateFileCollection'])
})
.prefix('/api/rag')

15
admin/tests/unit/kb_file_grouping.spec.ts
View File

@ -23,6 +23,7 @@ const asInfos = (sources: string[]): StoredFileInfo[] =>
size: null,
uploadedAt: null,
isUserUpload: classifyKbFile(source) === 'upload',
collection: null,
}))
test('classifyKbFile distinguishes ZIM, upload, admin_docs, and other', () => {
@ -121,9 +122,9 @@ test('groupAndSortKbFiles preserves a stable synthetic key for the admin docs gr
* three sort keys (name, size, uploadedAt) produce visibly different orders —
* if a test passed for name it had better fail for size. */
const sized: StoredFileInfo[] = [
{ source: '/app/storage/kb_uploads/charlie.txt', state: null, chunksEmbedded: 0, fileName: 'charlie.txt', size: 100, uploadedAt: '2026-01-01T00:00:00Z', isUserUpload: true },
{ source: '/app/storage/kb_uploads/alpha.txt', state: null, chunksEmbedded: 0, fileName: 'alpha.txt', size: 300, uploadedAt: '2026-03-01T00:00:00Z', isUserUpload: true },
{ source: '/app/storage/kb_uploads/bravo.txt', state: null, chunksEmbedded: 0, fileName: 'bravo.txt', size: 200, uploadedAt: '2026-02-01T00:00:00Z', isUserUpload: true },
{ source: '/app/storage/kb_uploads/charlie.txt', state: null, chunksEmbedded: 0, fileName: 'charlie.txt', size: 100, uploadedAt: '2026-01-01T00:00:00Z', isUserUpload: true, collection: null },
{ source: '/app/storage/kb_uploads/alpha.txt', state: null, chunksEmbedded: 0, fileName: 'alpha.txt', size: 300, uploadedAt: '2026-03-01T00:00:00Z', isUserUpload: true, collection: null },
{ source: '/app/storage/kb_uploads/bravo.txt', state: null, chunksEmbedded: 0, fileName: 'bravo.txt', size: 200, uploadedAt: '2026-02-01T00:00:00Z', isUserUpload: true, collection: null },
]
test('groupAndSortKbFiles sorts by size ascending', () => {
@ -149,7 +150,7 @@ test('groupAndSortKbFiles sorts by uploadedAt descending', () => {
test('groupAndSortKbFiles parks files with null size at the end of size sort', () => {
const withMissing: StoredFileInfo[] = [
...sized,
{ source: '/app/storage/kb_uploads/zzz_missing.txt', state: null, chunksEmbedded: 0, fileName: 'zzz_missing.txt', size: null, uploadedAt: null, isUserUpload: true },
{ source: '/app/storage/kb_uploads/zzz_missing.txt', state: null, chunksEmbedded: 0, fileName: 'zzz_missing.txt', size: null, uploadedAt: null, isUserUpload: true, collection: null },
]
// Missing-size files sort last regardless of direction so the "real" data
// owns the top of the view either way.
@ -161,9 +162,9 @@ test('groupAndSortKbFiles parks files with null size at the end of size sort', (
test('groupAndSortKbFiles preserves bucket order across all sort modes', () => {
const mixed: StoredFileInfo[] = [
{ source: '/app/storage/zim/big.zim', state: null, chunksEmbedded: 0, fileName: 'big.zim', size: 999, uploadedAt: '2026-01-01T00:00:00Z', isUserUpload: false },
{ source: '/app/storage/kb_uploads/small.txt', state: null, chunksEmbedded: 0, fileName: 'small.txt', size: 1, uploadedAt: '2026-09-01T00:00:00Z', isUserUpload: true },
{ source: '/app/docs/release-notes.md', state: null, chunksEmbedded: 0, fileName: 'release-notes.md', size: 50, uploadedAt: '2026-05-01T00:00:00Z', isUserUpload: false },
{ source: '/app/storage/zim/big.zim', state: null, chunksEmbedded: 0, fileName: 'big.zim', size: 999, uploadedAt: '2026-01-01T00:00:00Z', isUserUpload: false, collection: null },
{ source: '/app/storage/kb_uploads/small.txt', state: null, chunksEmbedded: 0, fileName: 'small.txt', size: 1, uploadedAt: '2026-09-01T00:00:00Z', isUserUpload: true, collection: null },
{ source: '/app/docs/release-notes.md', state: null, chunksEmbedded: 0, fileName: 'release-notes.md', size: 50, uploadedAt: '2026-05-01T00:00:00Z', isUserUpload: false, collection: null },
]
// Even if size-desc would put zim first naturally, sort runs *within* a
// bucket — buckets themselves stay in the canonical zim → upload → admin_docs

1
admin/types/ollama.ts
View File

@ -33,6 +33,7 @@ export type OllamaChatRequest = {
messages: OllamaChatMessage[]
stream?: boolean
sessionId?: number
collection?: string
}
export type OllamaChatResponse = {

2
admin/types/rag.ts
View File

@ -65,6 +65,8 @@ export type StoredFileInfo = {
/** True when `source` lives under the user-uploads directory. Drives which
* rows offer view/download in the UI. */
isUserUpload: boolean
/** Subject/category tag, or null if uncategorized. */
collection: string | null
}
/**