feat(KnowledgeBase): add document viewer, download, metadata, and sorting (#721)
Rebuilt on top of dev's RFC #883 state-machine UI rather than the now-defunct StoredFile shape: - Extend StoredFileInfo with fileName/size/uploadedAt/isUserUpload - Populate metadata from on-disk stats in RagService.getStoredFiles - Add fileSourceSchema validator + getFileContent/downloadFile endpoints scoped to the uploads directory only (tighter than the original PR — matches docs_service traversal pattern) - KnowledgeBaseModal: sortable Size and Uploaded columns; View/Download buttons on upload-bucket rows; new FileViewerModal for in-browser text preview. Bucket grouping preserved — sort applies within each bucket. - Use formatBytes from ~/lib/util rather than redefining
This commit is contained in:
parent
fad9e30ddc
commit
5181637926
|
|
@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app'
|
||||||
import { randomBytes } from 'node:crypto'
|
import { randomBytes } from 'node:crypto'
|
||||||
import { sanitizeFilename } from '../utils/fs.js'
|
import { sanitizeFilename } from '../utils/fs.js'
|
||||||
import { basename } from 'node:path'
|
import { basename } from 'node:path'
|
||||||
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag'
|
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag'
|
||||||
import logger from '@adonisjs/core/services/logger'
|
import logger from '@adonisjs/core/services/logger'
|
||||||
|
|
||||||
@inject()
|
@inject()
|
||||||
|
|
@ -162,4 +162,23 @@ export default class RagController {
|
||||||
const result = await KbRatioRegistry.estimateBatch(normalized)
|
const result = await KbRatioRegistry.estimateBatch(normalized)
|
||||||
return response.status(200).json(result)
|
return response.status(200).json(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async getFileContent({ request, response }: HttpContext) {
|
||||||
|
const { source } = await request.validateUsing(fileSourceSchema)
|
||||||
|
const result = await this.ragService.readFileContent(source)
|
||||||
|
if (!result) {
|
||||||
|
return response.status(404).json({ error: 'File not found or not viewable' })
|
||||||
|
}
|
||||||
|
return response.status(200).json(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
public async downloadFile({ request, response }: HttpContext) {
|
||||||
|
const { source } = await request.validateUsing(fileSourceSchema)
|
||||||
|
const filePath = await this.ragService.resolveDownloadPath(source)
|
||||||
|
if (!filePath) {
|
||||||
|
return response.status(404).json({ error: 'File not found' })
|
||||||
|
}
|
||||||
|
const fileName = filePath.split(/[/\\]/).at(-1) ?? 'download'
|
||||||
|
return response.attachment(filePath, fileName)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1140,20 +1140,90 @@ export class RagService {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return Array.from(sources).map((source) => {
|
const uploadsAbsPath = resolve(join(process.cwd(), RagService.UPLOADS_STORAGE_PATH))
|
||||||
const row = stateByPath.get(source)
|
return await Promise.all(
|
||||||
return {
|
Array.from(sources).map(async (source) => {
|
||||||
source,
|
const row = stateByPath.get(source)
|
||||||
state: row?.state ?? null,
|
const fileName = source.split(/[/\\]/).at(-1) ?? source
|
||||||
chunksEmbedded: row?.chunks_embedded ?? 0,
|
const isUserUpload = resolve(source).startsWith(uploadsAbsPath + sep)
|
||||||
}
|
const stats = await getFileStatsIfExists(source)
|
||||||
})
|
return {
|
||||||
|
source,
|
||||||
|
state: row?.state ?? null,
|
||||||
|
chunksEmbedded: row?.chunks_embedded ?? 0,
|
||||||
|
fileName,
|
||||||
|
size: stats?.size ?? null,
|
||||||
|
uploadedAt: stats?.modifiedTime.toISOString() ?? null,
|
||||||
|
isUserUpload,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Error retrieving stored files:', error)
|
logger.error('Error retrieving stored files:', error)
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve a stored-file `source` to an absolute disk path, but only if the
|
||||||
|
* path lives under the uploads directory. Mirrors the docs_service traversal
|
||||||
|
* guard: resolve, then require the resolved path to be strictly inside the
|
||||||
|
* base + path separator (so siblings of `kb_uploads` can't slip through).
|
||||||
|
* Returns null for anything else — ZIMs, admin docs, README, or paths
|
||||||
|
* outside the app entirely. The viewer/download endpoints lean on this so
|
||||||
|
* they don't need to repeat the check.
|
||||||
|
*/
|
||||||
|
private resolveUploadPath(source: string): string | null {
|
||||||
|
const uploadsAbsPath = resolve(join(process.cwd(), RagService.UPLOADS_STORAGE_PATH))
|
||||||
|
const resolved = resolve(source)
|
||||||
|
if (!resolved.startsWith(uploadsAbsPath + sep)) return null
|
||||||
|
return resolved
|
||||||
|
}
|
||||||
|
|
||||||
|
private static readonly VIEWABLE_TEXT_EXTENSIONS: ReadonlySet<string> = new Set([
|
||||||
|
'md', 'txt', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'html',
|
||||||
|
])
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the text content of a user-uploaded file for in-browser viewing.
|
||||||
|
* Returns null when the source is outside uploads, missing, or not a
|
||||||
|
* recognized text extension. The extension allowlist is intentionally narrow
|
||||||
|
* — PDFs/EPUBs/ZIMs round-trip through Download, not the viewer.
|
||||||
|
*/
|
||||||
|
public async readFileContent(
|
||||||
|
source: string
|
||||||
|
): Promise<{ content: string; extension: string; fileName: string } | null> {
|
||||||
|
const resolved = this.resolveUploadPath(source)
|
||||||
|
if (!resolved) return null
|
||||||
|
|
||||||
|
const extension = resolved.split('.').at(-1)?.toLowerCase() ?? ''
|
||||||
|
if (!RagService.VIEWABLE_TEXT_EXTENSIONS.has(extension)) return null
|
||||||
|
|
||||||
|
const stats = await getFileStatsIfExists(resolved)
|
||||||
|
if (!stats) return null
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { readFile } = await import('node:fs/promises')
|
||||||
|
const content = await readFile(resolved, 'utf-8')
|
||||||
|
const fileName = resolved.split(/[/\\]/).at(-1) ?? resolved
|
||||||
|
return { content, extension, fileName }
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn({ err: error, source }, '[RagService.readFileContent] read failed')
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve a download-target path for a stored upload. Returns null if the
|
||||||
|
* source isn't an upload or the file is missing on disk.
|
||||||
|
*/
|
||||||
|
public async resolveDownloadPath(source: string): Promise<string | null> {
|
||||||
|
const resolved = this.resolveUploadPath(source)
|
||||||
|
if (!resolved) return null
|
||||||
|
const stats = await getFileStatsIfExists(resolved)
|
||||||
|
return stats ? resolved : null
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute whether the first-chat JIT prompt should fire and surface the file
|
* Compute whether the first-chat JIT prompt should fire and surface the file
|
||||||
* count the banner uses in its copy ("Index your N existing files?"). The
|
* count the banner uses in its copy ("Index your N existing files?"). The
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,12 @@ export const embedFileSchema = vine.compile(
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
|
|
||||||
|
export const fileSourceSchema = vine.compile(
|
||||||
|
vine.object({
|
||||||
|
source: vine.string().minLength(1),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
export const estimateBatchSchema = vine.compile(
|
export const estimateBatchSchema = vine.compile(
|
||||||
vine.object({
|
vine.object({
|
||||||
files: vine
|
files: vine
|
||||||
|
|
|
||||||
|
|
@ -10,9 +10,19 @@ import api from '~/lib/api'
|
||||||
import {
|
import {
|
||||||
groupAndSortKbFiles,
|
groupAndSortKbFiles,
|
||||||
type KbFileGroup,
|
type KbFileGroup,
|
||||||
|
type KbFileSort,
|
||||||
|
type KbFileSortKey,
|
||||||
} from '~/lib/kb_file_grouping'
|
} from '~/lib/kb_file_grouping'
|
||||||
import type { KbIngestStateValue } from '../../../types/kb_ingest_state'
|
import type { KbIngestStateValue } from '../../../types/kb_ingest_state'
|
||||||
import { IconX } from '@tabler/icons-react'
|
import { formatBytes } from '~/lib/util'
|
||||||
|
import {
|
||||||
|
IconArrowsSort,
|
||||||
|
IconDownload,
|
||||||
|
IconEye,
|
||||||
|
IconSortAscending,
|
||||||
|
IconSortDescending,
|
||||||
|
IconX,
|
||||||
|
} from '@tabler/icons-react'
|
||||||
import { useModals } from '~/context/ModalContext'
|
import { useModals } from '~/context/ModalContext'
|
||||||
import StyledModal from '../StyledModal'
|
import StyledModal from '../StyledModal'
|
||||||
import ActiveEmbedJobs from '~/components/ActiveEmbedJobs'
|
import ActiveEmbedJobs from '~/components/ActiveEmbedJobs'
|
||||||
|
|
@ -23,6 +33,42 @@ interface KnowledgeBaseModalProps {
|
||||||
onClose: () => void
|
onClose: () => void
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// File extensions the in-browser viewer can render. Must stay in sync with
|
||||||
|
// `RagService.VIEWABLE_TEXT_EXTENSIONS` — anything outside this set falls back
|
||||||
|
// to Download.
|
||||||
|
const VIEWABLE_EXTENSIONS = new Set(['md', 'txt', 'csv', 'json', 'yaml', 'yml', 'toml', 'xml', 'html'])
|
||||||
|
|
||||||
|
function isViewableExtension(filename: string): boolean {
|
||||||
|
const ext = filename.split('.').at(-1)?.toLowerCase() ?? ''
|
||||||
|
return VIEWABLE_EXTENSIONS.has(ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderSortHeader(
|
||||||
|
label: string,
|
||||||
|
key: KbFileSortKey,
|
||||||
|
sort: KbFileSort,
|
||||||
|
setSort: (s: KbFileSort) => void
|
||||||
|
): React.ReactNode {
|
||||||
|
const active = sort.key === key
|
||||||
|
const Icon = !active ? IconArrowsSort : sort.direction === 'asc' ? IconSortAscending : IconSortDescending
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="inline-flex items-center gap-1 text-left hover:text-text-primary transition-colors"
|
||||||
|
onClick={() => {
|
||||||
|
if (!active) {
|
||||||
|
setSort({ key, direction: 'asc' })
|
||||||
|
} else {
|
||||||
|
setSort({ key, direction: sort.direction === 'asc' ? 'desc' : 'asc' })
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span>{label}</span>
|
||||||
|
<Icon size={14} className={active ? 'text-text-primary' : 'text-text-muted'} aria-hidden="true" />
|
||||||
|
</button>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compact label for the per-row ingestion state. Files that exist in Qdrant
|
* Compact label for the per-row ingestion state. Files that exist in Qdrant
|
||||||
* with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883
|
* with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883
|
||||||
|
|
@ -101,6 +147,8 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
||||||
const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null)
|
const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null)
|
||||||
const [bulkMode, setBulkMode] = useState<null | 'reembed' | 'reset'>(null)
|
const [bulkMode, setBulkMode] = useState<null | 'reembed' | 'reset'>(null)
|
||||||
const [resetTyped, setResetTyped] = useState('')
|
const [resetTyped, setResetTyped] = useState('')
|
||||||
|
const [sort, setSort] = useState<KbFileSort>({ key: 'name', direction: 'asc' })
|
||||||
|
const [viewerSource, setViewerSource] = useState<string | null>(null)
|
||||||
const fileUploaderRef = useRef<React.ComponentRef<typeof FileUploader>>(null)
|
const fileUploaderRef = useRef<React.ComponentRef<typeof FileUploader>>(null)
|
||||||
const { openModal, closeModal } = useModals()
|
const { openModal, closeModal } = useModals()
|
||||||
const queryClient = useQueryClient()
|
const queryClient = useQueryClient()
|
||||||
|
|
@ -555,7 +603,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
||||||
columns={[
|
columns={[
|
||||||
{
|
{
|
||||||
accessor: 'source',
|
accessor: 'source',
|
||||||
title: 'File Name',
|
title: renderSortHeader('File Name', 'name', sort, setSort),
|
||||||
render(record) {
|
render(record) {
|
||||||
const warnings = fileWarnings[record.source] ?? []
|
const warnings = fileWarnings[record.source] ?? []
|
||||||
const pill = renderStatePill(record)
|
const pill = renderStatePill(record)
|
||||||
|
|
@ -594,6 +642,35 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
accessor: 'size',
|
||||||
|
title: renderSortHeader('Size', 'size', sort, setSort),
|
||||||
|
className: 'whitespace-nowrap',
|
||||||
|
render(record) {
|
||||||
|
// The collapsed admin_docs group has no single size — leave blank
|
||||||
|
// rather than misleadingly summing across N files.
|
||||||
|
if (record.bucket === 'admin_docs' || record.size === null) {
|
||||||
|
return <span className="text-text-muted">—</span>
|
||||||
|
}
|
||||||
|
return <span className="text-text-secondary">{formatBytes(record.size)}</span>
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessor: 'uploadedAt',
|
||||||
|
title: renderSortHeader('Uploaded', 'uploadedAt', sort, setSort),
|
||||||
|
className: 'whitespace-nowrap',
|
||||||
|
render(record) {
|
||||||
|
if (record.bucket === 'admin_docs' || !record.uploadedAt) {
|
||||||
|
return <span className="text-text-muted">—</span>
|
||||||
|
}
|
||||||
|
const d = new Date(record.uploadedAt)
|
||||||
|
return (
|
||||||
|
<span className="text-text-secondary" title={d.toISOString()}>
|
||||||
|
{d.toLocaleDateString()}
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
accessor: 'source',
|
accessor: 'source',
|
||||||
title: '',
|
title: '',
|
||||||
|
|
@ -642,6 +719,9 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
||||||
const actionPendingForThisRow =
|
const actionPendingForThisRow =
|
||||||
embedMutation.isPending && embedMutation.variables?.source === record.source
|
embedMutation.isPending && embedMutation.variables?.source === record.source
|
||||||
|
|
||||||
|
const canView = record.isUserUpload && isViewableExtension(record.displayName) && record.size !== null
|
||||||
|
const canDownload = record.isUserUpload && record.size !== null
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex justify-end items-center gap-2">
|
<div className="flex justify-end items-center gap-2">
|
||||||
{action && (
|
{action && (
|
||||||
|
|
@ -662,6 +742,24 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
||||||
{action.label}
|
{action.label}
|
||||||
</StyledButton>
|
</StyledButton>
|
||||||
)}
|
)}
|
||||||
|
{canView && (
|
||||||
|
<StyledButton
|
||||||
|
variant="ghost"
|
||||||
|
size="sm"
|
||||||
|
icon="IconEye"
|
||||||
|
onClick={() => setViewerSource(record.source)}
|
||||||
|
>View</StyledButton>
|
||||||
|
)}
|
||||||
|
{canDownload && (
|
||||||
|
<StyledButton
|
||||||
|
variant="ghost"
|
||||||
|
size="sm"
|
||||||
|
icon="IconDownload"
|
||||||
|
onClick={() => {
|
||||||
|
window.location.href = `/api/rag/files/download?source=${encodeURIComponent(record.source)}`
|
||||||
|
}}
|
||||||
|
>Download</StyledButton>
|
||||||
|
)}
|
||||||
<StyledButton
|
<StyledButton
|
||||||
variant="danger"
|
variant="danger"
|
||||||
size="sm"
|
size="sm"
|
||||||
|
|
@ -675,7 +773,7 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
]}
|
]}
|
||||||
data={groupAndSortKbFiles(storedFiles)}
|
data={groupAndSortKbFiles(storedFiles, sort)}
|
||||||
loading={isLoadingFiles}
|
loading={isLoadingFiles}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -807,6 +905,57 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
||||||
</div>
|
</div>
|
||||||
</StyledModal>
|
</StyledModal>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{viewerSource && (
|
||||||
|
<FileViewerModal
|
||||||
|
source={viewerSource}
|
||||||
|
onClose={() => setViewerSource(null)}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function FileViewerModal({ source, onClose }: { source: string; onClose: () => void }) {
|
||||||
|
const { data, isLoading, isFetched } = useQuery({
|
||||||
|
queryKey: ['rag', 'file-content', source],
|
||||||
|
queryFn: () => api.getFileContent(source),
|
||||||
|
staleTime: 60_000,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Title falls back to the trailing path segment so the modal still has a
|
||||||
|
// useful header while the fetch is in-flight or if it failed.
|
||||||
|
const fallbackName = source.split(/[/\\]/).at(-1) ?? source
|
||||||
|
const title = data?.fileName ?? fallbackName
|
||||||
|
// `catchInternal` swallows errors and resolves to undefined, surfacing a
|
||||||
|
// toast — so the "couldn't load" branch is gated on a finished-but-empty
|
||||||
|
// fetch rather than on react-query's `isError`.
|
||||||
|
const showError = isFetched && !data
|
||||||
|
|
||||||
|
return (
|
||||||
|
<StyledModal
|
||||||
|
title={title}
|
||||||
|
open={true}
|
||||||
|
onClose={onClose}
|
||||||
|
onCancel={onClose}
|
||||||
|
cancelText="Close"
|
||||||
|
large
|
||||||
|
>
|
||||||
|
<div className="text-left text-sm">
|
||||||
|
{isLoading && (
|
||||||
|
<div className="text-text-secondary">Loading…</div>
|
||||||
|
)}
|
||||||
|
{showError && (
|
||||||
|
<div className="text-amber-700 dark:text-amber-300">
|
||||||
|
Couldn't load file. It may have been moved or its type isn't viewable.
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{data && (
|
||||||
|
<pre className="max-h-[60vh] overflow-auto whitespace-pre-wrap rounded border border-border-subtle bg-surface-secondary p-3 font-mono text-xs text-text-primary">
|
||||||
|
{data.content}
|
||||||
|
</pre>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</StyledModal>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -514,6 +514,17 @@ class API {
|
||||||
})()
|
})()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getFileContent(source: string) {
|
||||||
|
return catchInternal(async () => {
|
||||||
|
const response = await this.client.get<{
|
||||||
|
content: string
|
||||||
|
extension: string
|
||||||
|
fileName: string
|
||||||
|
}>('/rag/files/content', { params: { source } })
|
||||||
|
return response.data
|
||||||
|
})()
|
||||||
|
}
|
||||||
|
|
||||||
async getSystemInfo() {
|
async getSystemInfo() {
|
||||||
return catchInternal(async () => {
|
return catchInternal(async () => {
|
||||||
const response = await this.client.get<SystemInformationResponse>('/system/info')
|
const response = await this.client.get<SystemInformationResponse>('/system/info')
|
||||||
|
|
|
||||||
|
|
@ -52,19 +52,62 @@ export interface KbFileGroup {
|
||||||
/** Chunks currently embedded for this source; 0 for state-row-less or
|
/** Chunks currently embedded for this source; 0 for state-row-less or
|
||||||
* zero-chunk files. Always 0 for the collapsed admin_docs group. */
|
* zero-chunk files. Always 0 for the collapsed admin_docs group. */
|
||||||
chunksEmbedded: number
|
chunksEmbedded: number
|
||||||
|
/** File size in bytes from disk. Null for the collapsed admin_docs group,
|
||||||
|
* and for any file the scanner couldn't stat. */
|
||||||
|
size: number | null
|
||||||
|
/** Last-modified timestamp (ISO 8601). Null for collapsed groups and for
|
||||||
|
* files the scanner couldn't stat. */
|
||||||
|
uploadedAt: string | null
|
||||||
|
/** True when the row corresponds to a user upload — drives whether the
|
||||||
|
* view/download buttons render. False for the collapsed admin_docs group. */
|
||||||
|
isUserUpload: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
const BUCKET_SORT_ORDER: KbFileBucket[] = ['zim', 'upload', 'admin_docs', 'other']
|
const BUCKET_SORT_ORDER: KbFileBucket[] = ['zim', 'upload', 'admin_docs', 'other']
|
||||||
|
|
||||||
|
export type KbFileSortKey = 'name' | 'size' | 'uploadedAt'
|
||||||
|
export type KbFileSortDirection = 'asc' | 'desc'
|
||||||
|
export interface KbFileSort {
|
||||||
|
key: KbFileSortKey
|
||||||
|
direction: KbFileSortDirection
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_SORT: KbFileSort = { key: 'name', direction: 'asc' }
|
||||||
|
|
||||||
|
function compareForSort(a: StoredFileInfo, b: StoredFileInfo, sort: KbFileSort): number {
|
||||||
|
// Files the scanner couldn't stat sort to the end regardless of direction so
|
||||||
|
// they don't pollute the top of size/uploaded-at views.
|
||||||
|
const aMissing = sort.key !== 'name' && (sort.key === 'size' ? a.size === null : a.uploadedAt === null)
|
||||||
|
const bMissing = sort.key !== 'name' && (sort.key === 'size' ? b.size === null : b.uploadedAt === null)
|
||||||
|
if (aMissing && !bMissing) return 1
|
||||||
|
if (!aMissing && bMissing) return -1
|
||||||
|
|
||||||
|
let cmp = 0
|
||||||
|
if (sort.key === 'size') {
|
||||||
|
cmp = (a.size ?? 0) - (b.size ?? 0)
|
||||||
|
} else if (sort.key === 'uploadedAt') {
|
||||||
|
cmp = (a.uploadedAt ?? '').localeCompare(b.uploadedAt ?? '')
|
||||||
|
}
|
||||||
|
if (cmp === 0) {
|
||||||
|
// Tiebreak (and primary key for 'name') is filename — keeps stable order.
|
||||||
|
cmp = sourceToDisplayName(a.source).localeCompare(sourceToDisplayName(b.source))
|
||||||
|
}
|
||||||
|
return sort.direction === 'desc' ? -cmp : cmp
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Group stored-file rows into table rows for the Stored Files panel.
|
* Group stored-file rows into table rows for the Stored Files panel.
|
||||||
*
|
*
|
||||||
* - Admin docs (`/app/docs/*`, README) collapse into a single
|
* - Admin docs (`/app/docs/*`, README) collapse into a single
|
||||||
* "Project NOMAD documentation · N files" row.
|
* "Project NOMAD documentation · N files" row.
|
||||||
* - ZIMs, uploads, and others stay as individual rows, sorted by bucket then
|
* - ZIMs, uploads, and others stay as individual rows, sorted within their
|
||||||
* alphabetically by filename so related items cluster naturally.
|
* bucket by the active sort key. Bucket order itself is fixed — sorting
|
||||||
|
* never flattens or reorders the groups themselves.
|
||||||
*/
|
*/
|
||||||
export function groupAndSortKbFiles(files: StoredFileInfo[]): KbFileGroup[] {
|
export function groupAndSortKbFiles(
|
||||||
|
files: StoredFileInfo[],
|
||||||
|
sort: KbFileSort = DEFAULT_SORT
|
||||||
|
): KbFileGroup[] {
|
||||||
const buckets: Record<KbFileBucket, StoredFileInfo[]> = {
|
const buckets: Record<KbFileBucket, StoredFileInfo[]> = {
|
||||||
zim: [],
|
zim: [],
|
||||||
upload: [],
|
upload: [],
|
||||||
|
|
@ -90,13 +133,14 @@ export function groupAndSortKbFiles(files: StoredFileInfo[]): KbFileGroup[] {
|
||||||
members: members.map((m) => m.source),
|
members: members.map((m) => m.source),
|
||||||
state: null,
|
state: null,
|
||||||
chunksEmbedded: 0,
|
chunksEmbedded: 0,
|
||||||
|
size: null,
|
||||||
|
uploadedAt: null,
|
||||||
|
isUserUpload: false,
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const file of members.sort((a, b) =>
|
for (const file of members.sort((a, b) => compareForSort(a, b, sort))) {
|
||||||
sourceToDisplayName(a.source).localeCompare(sourceToDisplayName(b.source))
|
|
||||||
)) {
|
|
||||||
groups.push({
|
groups.push({
|
||||||
bucket,
|
bucket,
|
||||||
source: file.source,
|
source: file.source,
|
||||||
|
|
@ -105,6 +149,9 @@ export function groupAndSortKbFiles(files: StoredFileInfo[]): KbFileGroup[] {
|
||||||
members: [],
|
members: [],
|
||||||
state: file.state,
|
state: file.state,
|
||||||
chunksEmbedded: file.chunksEmbedded,
|
chunksEmbedded: file.chunksEmbedded,
|
||||||
|
size: file.size,
|
||||||
|
uploadedAt: file.uploadedAt,
|
||||||
|
isUserUpload: file.isUserUpload,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -147,6 +147,8 @@ router
|
||||||
router.get('/file-warnings', [RagController, 'getFileWarnings'])
|
router.get('/file-warnings', [RagController, 'getFileWarnings'])
|
||||||
router.delete('/files', [RagController, 'deleteFile'])
|
router.delete('/files', [RagController, 'deleteFile'])
|
||||||
router.post('/files/embed', [RagController, 'embedFile'])
|
router.post('/files/embed', [RagController, 'embedFile'])
|
||||||
|
router.get('/files/content', [RagController, 'getFileContent'])
|
||||||
|
router.get('/files/download', [RagController, 'downloadFile'])
|
||||||
router.get('/active-jobs', [RagController, 'getActiveJobs'])
|
router.get('/active-jobs', [RagController, 'getActiveJobs'])
|
||||||
router.get('/failed-jobs', [RagController, 'getFailedJobs'])
|
router.get('/failed-jobs', [RagController, 'getFailedJobs'])
|
||||||
router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs'])
|
router.delete('/failed-jobs', [RagController, 'cleanupFailedJobs'])
|
||||||
|
|
|
||||||
|
|
@ -11,9 +11,19 @@ import type { StoredFileInfo } from '../../types/rag.js'
|
||||||
/** Wrap source paths into the minimal StoredFileInfo shape that
|
/** Wrap source paths into the minimal StoredFileInfo shape that
|
||||||
* `groupAndSortKbFiles` now expects. State + chunk count are irrelevant to
|
* `groupAndSortKbFiles` now expects. State + chunk count are irrelevant to
|
||||||
* grouping/sorting behavior; the per-file state-pill rendering is exercised
|
* grouping/sorting behavior; the per-file state-pill rendering is exercised
|
||||||
* separately in the modal's component tests (added in the follow-up PR). */
|
* separately in the modal's component tests (added in the follow-up PR).
|
||||||
|
* Metadata fields default to nullish/false — individual tests that exercise
|
||||||
|
* sort-by-size or sort-by-uploadedAt override as needed. */
|
||||||
const asInfos = (sources: string[]): StoredFileInfo[] =>
|
const asInfos = (sources: string[]): StoredFileInfo[] =>
|
||||||
sources.map((source) => ({ source, state: null, chunksEmbedded: 0 }))
|
sources.map((source) => ({
|
||||||
|
source,
|
||||||
|
state: null,
|
||||||
|
chunksEmbedded: 0,
|
||||||
|
fileName: sourceToDisplayName(source),
|
||||||
|
size: null,
|
||||||
|
uploadedAt: null,
|
||||||
|
isUserUpload: classifyKbFile(source) === 'upload',
|
||||||
|
}))
|
||||||
|
|
||||||
test('classifyKbFile distinguishes ZIM, upload, admin_docs, and other', () => {
|
test('classifyKbFile distinguishes ZIM, upload, admin_docs, and other', () => {
|
||||||
assert.equal(
|
assert.equal(
|
||||||
|
|
@ -106,3 +116,78 @@ test('groupAndSortKbFiles preserves a stable synthetic key for the admin docs gr
|
||||||
// can be used as a React key without colliding with any real file row.
|
// can be used as a React key without colliding with any real file row.
|
||||||
assert.equal(groups[0].source, '__admin_docs_group__')
|
assert.equal(groups[0].source, '__admin_docs_group__')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/** Sized fixtures for the sort tests. `size` and `uploadedAt` are set so the
|
||||||
|
* three sort keys (name, size, uploadedAt) produce visibly different orders —
|
||||||
|
* if a test passed for name it had better fail for size. */
|
||||||
|
const sized: StoredFileInfo[] = [
|
||||||
|
{ source: '/app/storage/kb_uploads/charlie.txt', state: null, chunksEmbedded: 0, fileName: 'charlie.txt', size: 100, uploadedAt: '2026-01-01T00:00:00Z', isUserUpload: true },
|
||||||
|
{ source: '/app/storage/kb_uploads/alpha.txt', state: null, chunksEmbedded: 0, fileName: 'alpha.txt', size: 300, uploadedAt: '2026-03-01T00:00:00Z', isUserUpload: true },
|
||||||
|
{ source: '/app/storage/kb_uploads/bravo.txt', state: null, chunksEmbedded: 0, fileName: 'bravo.txt', size: 200, uploadedAt: '2026-02-01T00:00:00Z', isUserUpload: true },
|
||||||
|
]
|
||||||
|
|
||||||
|
test('groupAndSortKbFiles sorts by size ascending', () => {
|
||||||
|
const groups = groupAndSortKbFiles(sized, { key: 'size', direction: 'asc' })
|
||||||
|
assert.deepEqual(groups.map((g) => g.displayName), ['charlie.txt', 'bravo.txt', 'alpha.txt'])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('groupAndSortKbFiles sorts by size descending', () => {
|
||||||
|
const groups = groupAndSortKbFiles(sized, { key: 'size', direction: 'desc' })
|
||||||
|
assert.deepEqual(groups.map((g) => g.displayName), ['alpha.txt', 'bravo.txt', 'charlie.txt'])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('groupAndSortKbFiles sorts by uploadedAt ascending', () => {
|
||||||
|
const groups = groupAndSortKbFiles(sized, { key: 'uploadedAt', direction: 'asc' })
|
||||||
|
assert.deepEqual(groups.map((g) => g.displayName), ['charlie.txt', 'bravo.txt', 'alpha.txt'])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('groupAndSortKbFiles sorts by uploadedAt descending', () => {
|
||||||
|
const groups = groupAndSortKbFiles(sized, { key: 'uploadedAt', direction: 'desc' })
|
||||||
|
assert.deepEqual(groups.map((g) => g.displayName), ['alpha.txt', 'bravo.txt', 'charlie.txt'])
|
||||||
|
})
|
||||||
|
|
||||||
|
test('groupAndSortKbFiles parks files with null size at the end of size sort', () => {
|
||||||
|
const withMissing: StoredFileInfo[] = [
|
||||||
|
...sized,
|
||||||
|
{ source: '/app/storage/kb_uploads/zzz_missing.txt', state: null, chunksEmbedded: 0, fileName: 'zzz_missing.txt', size: null, uploadedAt: null, isUserUpload: true },
|
||||||
|
]
|
||||||
|
// Missing-size files sort last regardless of direction so the "real" data
|
||||||
|
// owns the top of the view either way.
|
||||||
|
const asc = groupAndSortKbFiles(withMissing, { key: 'size', direction: 'asc' })
|
||||||
|
assert.equal(asc.at(-1)?.displayName, 'zzz_missing.txt')
|
||||||
|
const desc = groupAndSortKbFiles(withMissing, { key: 'size', direction: 'desc' })
|
||||||
|
assert.equal(desc.at(-1)?.displayName, 'zzz_missing.txt')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('groupAndSortKbFiles preserves bucket order across all sort modes', () => {
|
||||||
|
const mixed: StoredFileInfo[] = [
|
||||||
|
{ source: '/app/storage/zim/big.zim', state: null, chunksEmbedded: 0, fileName: 'big.zim', size: 999, uploadedAt: '2026-01-01T00:00:00Z', isUserUpload: false },
|
||||||
|
{ source: '/app/storage/kb_uploads/small.txt', state: null, chunksEmbedded: 0, fileName: 'small.txt', size: 1, uploadedAt: '2026-09-01T00:00:00Z', isUserUpload: true },
|
||||||
|
{ source: '/app/docs/release-notes.md', state: null, chunksEmbedded: 0, fileName: 'release-notes.md', size: 50, uploadedAt: '2026-05-01T00:00:00Z', isUserUpload: false },
|
||||||
|
]
|
||||||
|
// Even if size-desc would put zim first naturally, sort runs *within* a
|
||||||
|
// bucket — buckets themselves stay in the canonical zim → upload → admin_docs
|
||||||
|
// order. This is the invariant that lets the per-bucket grouping stay
|
||||||
|
// legible while still giving the user a sortable view.
|
||||||
|
for (const direction of ['asc', 'desc'] as const) {
|
||||||
|
for (const key of ['name', 'size', 'uploadedAt'] as const) {
|
||||||
|
const groups = groupAndSortKbFiles(mixed, { key, direction })
|
||||||
|
assert.deepEqual(
|
||||||
|
groups.map((g) => g.bucket),
|
||||||
|
['zim', 'upload', 'admin_docs'],
|
||||||
|
`bucket order changed for ${key}/${direction}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test('groupAndSortKbFiles emits null metadata + isUserUpload=false for the admin_docs group', () => {
|
||||||
|
const groups = groupAndSortKbFiles(asInfos([
|
||||||
|
'/app/docs/release-notes.md',
|
||||||
|
'/app/README.md',
|
||||||
|
]))
|
||||||
|
assert.equal(groups[0].bucket, 'admin_docs')
|
||||||
|
assert.equal(groups[0].size, null)
|
||||||
|
assert.equal(groups[0].uploadedAt, null)
|
||||||
|
assert.equal(groups[0].isUserUpload, false)
|
||||||
|
})
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,15 @@ export type StoredFileInfo = {
|
||||||
source: string
|
source: string
|
||||||
state: import('./kb_ingest_state.js').KbIngestStateValue | null
|
state: import('./kb_ingest_state.js').KbIngestStateValue | null
|
||||||
chunksEmbedded: number
|
chunksEmbedded: number
|
||||||
|
/** Filename portion of `source` (last path segment). */
|
||||||
|
fileName: string
|
||||||
|
/** File size in bytes from disk; null if the file is missing or unreadable. */
|
||||||
|
size: number | null
|
||||||
|
/** Last-modified timestamp from disk (ISO 8601); null if unavailable. */
|
||||||
|
uploadedAt: string | null
|
||||||
|
/** True when `source` lives under the user-uploads directory. Drives which
|
||||||
|
* rows offer view/download in the UI. */
|
||||||
|
isUserUpload: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue