diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 63f62ca..ca73a14 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,8 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: Sync tags + run: git fetch --tags --force - name: semantic-release uses: cycjimmy/semantic-release-action@v6 id: semver diff --git a/admin/adonisrc.ts b/admin/adonisrc.ts index 741b160..9b82ee0 100644 --- a/admin/adonisrc.ts +++ b/admin/adonisrc.ts @@ -57,6 +57,7 @@ export default defineConfig({ () => import('#providers/kiwix_migration_provider'), () => import('#providers/qdrant_restart_policy_provider'), () => import('#providers/version_check_provider'), + () => import('#providers/gpu_passthrough_remediation_provider'), ], /* diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 68b6b78..3bf9e73 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -5,7 +5,8 @@ import { RagService } from '#services/rag_service' import Service from '#models/service' import KVStore from '#models/kv_store' import { modelNameSchema } from '#validators/download' -import { chatSchema, getAvailableModelsSchema } from '#validators/ollama' +import { chatSchema, getAvailableModelsSchema, unloadChatModelsSchema } from '#validators/ollama' +import { assertNotCloudMetadataUrl } from '#validators/common' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' import { RAG_CONTEXT_LIMITS, SYSTEM_PROMPTS } from '../../constants/ollama.js' @@ -33,6 +34,19 @@ export default class OllamaController { }) } + /** + * Send Ollama `keep_alive: 0` hints to every currently-loaded chat model + * except the embedding model and (optionally) a target model to preserve. + * Used by the chat UI to enforce the "one chat model at a time" invariant + * on model-switch, session-switch, and page-load. Best-effort: a failure + * here should not block the calling flow. + */ + async unloadChatModels({ request, response }: HttpContext) { + const { targetModel } = await request.validateUsing(unloadChatModelsSchema) + const unloaded = await this.ollamaService.unloadAllChatModelsExcept(targetModel ?? null) + return response.status(200).json({ unloaded }) + } + async chat({ request, response }: HttpContext) { const reqData = await request.validateUsing(chatSchema) @@ -229,11 +243,12 @@ export default class OllamaController { } } - // Validate URL format - if (!remoteUrl.startsWith('http')) { + try { + assertNotCloudMetadataUrl(remoteUrl) + } catch (err) { return response.status(400).send({ success: false, - message: 'Invalid URL. Must start with http:// or https://', + message: err instanceof Error ? err.message : 'Invalid URL.', }) } @@ -383,10 +398,15 @@ export default class OllamaController { // Get recent conversation history (last 6 messages for 3 turns) const recentMessages = messages.slice(-6) - // Skip rewriting for short conversations. Rewriting adds latency with - // little RAG benefit until there is enough context to matter. + // Skip rewriting on the very first turn — with only one user message + // there is no prior context to fold in, so the rewrite would just echo + // the message back at the cost of an extra LLM round-trip. From the + // first follow-up onward we need the rewrite so the RAG query carries + // entities and topics from earlier turns ("the bars" → "Hershey's bars + // chocolate poisoning dog"); without it, embeddings match nothing and + // the assistant loses the thread. const userMessages = recentMessages.filter(msg => msg.role === 'user') - if (userMessages.length <= 2) { + if (userMessages.length < 2) { return lastUserMessage?.content || null } diff --git a/admin/app/controllers/rag_controller.ts b/admin/app/controllers/rag_controller.ts index c836393..16cedcd 100644 --- a/admin/app/controllers/rag_controller.ts +++ b/admin/app/controllers/rag_controller.ts @@ -1,11 +1,13 @@ import { RagService } from '#services/rag_service' import { EmbedFileJob } from '#jobs/embed_file_job' +import KbRatioRegistry from '#models/kb_ratio_registry' import { inject } from '@adonisjs/core' import type { HttpContext } from '@adonisjs/core/http' import app from '@adonisjs/core/services/app' import { randomBytes } from 'node:crypto' import { sanitizeFilename } from '../utils/fs.js' -import { deleteFileSchema, getJobStatusSchema } from '#validators/rag' +import { basename } from 'node:path' +import { deleteFileSchema, embedFileSchema, estimateBatchSchema, getJobStatusSchema } from '#validators/rag' import logger from '@adonisjs/core/services/logger' @inject() @@ -66,6 +68,11 @@ export default class RagController { return response.status(200).json({ files }) } + public async getFileWarnings({ response }: HttpContext) { + const result = await this.ragService.computeFileWarnings() + return response.status(200).json(result) + } + public async deleteFile({ request, response }: HttpContext) { const { source } = await request.validateUsing(deleteFileSchema) const result = await this.ragService.deleteFileBySource(source) @@ -75,6 +82,21 @@ export default class RagController { return response.status(200).json({ message: result.message }) } + public async embedFile({ request, response }: HttpContext) { + const { source, force } = await request.validateUsing(embedFileSchema) + const result = await this.ragService.embedSingleFile(source, force ?? false) + if (!result.success) { + const status = { + not_found: 404, + inflight: 409, + delete_failed: 500, + dispatch_failed: 500, + }[result.code] + return response.status(status).json({ error: result.message, code: result.code }) + } + return response.status(202).json({ message: result.message }) + } + public async getFailedJobs({ response }: HttpContext) { const jobs = await EmbedFileJob.listFailedJobs() return response.status(200).json(jobs) @@ -88,6 +110,11 @@ export default class RagController { }) } + public async policyPromptState({ response }: HttpContext) { + const result = await this.ragService.getPolicyPromptState() + return response.status(200).json(result) + } + public async scanAndSync({ response }: HttpContext) { try { const syncResult = await this.ragService.scanAndSyncStorage() @@ -98,8 +125,41 @@ export default class RagController { } } + public async reembedAll({ response }: HttpContext) { + try { + const result = await this.ragService.reembedAll() + return response.status(200).json(result) + } catch (error) { + logger.error({ err: error }, '[RagController] Error during re-embed all') + return response.status(500).json({ error: 'Error during re-embed all' }) + } + } + + public async resetAndRebuild({ response }: HttpContext) { + try { + const result = await this.ragService.resetAndRebuild() + return response.status(200).json(result) + } catch (error) { + logger.error({ err: error }, '[RagController] Error during reset and rebuild') + return response.status(500).json({ error: 'Error during reset and rebuild' }) + } + } + public async health({ response }: HttpContext) { const result = await this.ragService.checkQdrantHealth() return response.status(200).json(result) } + + public async estimateBatch({ request, response }: HttpContext) { + const { files } = await request.validateUsing(estimateBatchSchema) + // The registry matches on basename prefixes; if a caller passes a full path + // (e.g. /app/storage/zim/wikipedia_en_simple_…), strip directories first so + // patterns like `wikipedia_en_simple_` still match. + const normalized = files.map((f) => ({ + filename: basename(f.filename), + sizeBytes: f.sizeBytes, + })) + const result = await KbRatioRegistry.estimateBatch(normalized) + return response.status(200).json(result) + } } diff --git a/admin/app/jobs/check_service_updates_job.ts b/admin/app/jobs/check_service_updates_job.ts index 6fb7335..58be73c 100644 --- a/admin/app/jobs/check_service_updates_job.ts +++ b/admin/app/jobs/check_service_updates_job.ts @@ -95,7 +95,7 @@ export class CheckServiceUpdatesJob { } static async scheduleNightly() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) await queue.upsertJobScheduler( @@ -114,7 +114,7 @@ export class CheckServiceUpdatesJob { } static async dispatch() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const job = await queue.add( diff --git a/admin/app/jobs/check_update_job.ts b/admin/app/jobs/check_update_job.ts index 046d9c0..aaac08d 100644 --- a/admin/app/jobs/check_update_job.ts +++ b/admin/app/jobs/check_update_job.ts @@ -42,7 +42,7 @@ export class CheckUpdateJob { } static async scheduleNightly() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) await queue.upsertJobScheduler( @@ -61,7 +61,7 @@ export class CheckUpdateJob { } static async dispatch() { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const job = await queue.add(this.key, {}, { diff --git a/admin/app/jobs/download_model_job.ts b/admin/app/jobs/download_model_job.ts index f189021..2ba0080 100644 --- a/admin/app/jobs/download_model_job.ts +++ b/admin/app/jobs/download_model_job.ts @@ -34,7 +34,7 @@ export class DownloadModelJob { /** Signal cancellation via Redis so the worker process can pick it up on its next poll tick */ static async signalCancel(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const client = await queue.client await client.set(this.cancelKey(jobId), '1', 'EX', 300) // 5 min TTL @@ -66,7 +66,7 @@ export class DownloadModelJob { DownloadModelJob.abortControllers.set(job.id!, abortController) // Get Redis client for checking cancel signals from the API process - const queueService = new QueueService() + const queueService = QueueService.getInstance() const cancelRedis = await queueService.getQueue(DownloadModelJob.queue).client // Track whether cancellation was explicitly requested by the user. Only user-initiated @@ -154,14 +154,14 @@ export class DownloadModelJob { } static async getByModelName(modelName: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(modelName) return await queue.getJob(jobId) } static async dispatch(params: DownloadModelJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(params.modelName) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index eef45a8..90ce677 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -4,9 +4,11 @@ import { EmbedJobWithProgress } from '../../types/rag.js' import { RagService } from '#services/rag_service' import { DockerService } from '#services/docker_service' import { OllamaService } from '#services/ollama_service' +import KbIngestState from '#models/kb_ingest_state' import { createHash } from 'crypto' import logger from '@adonisjs/core/services/logger' import fs from 'node:fs/promises' +import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' export interface EmbedFileJobParams { filePath: string @@ -27,6 +29,12 @@ export class EmbedFileJob { return 'embed-file' } + // Delay between continuation batches when embedding runs CPU-only. Gives the OS + // scheduler a brief idle window so sshd / disk-collector / other services don't + // starve during long multi-batch ZIM ingestions. Skipped entirely when the + // embedding model is GPU-offloaded — see OllamaService.isEmbeddingGpuAccelerated(). + static readonly CPU_BATCH_DELAY_MS = 1000 + static getJobId(filePath: string): string { return createHash('sha256').update(filePath).digest('hex').slice(0, 16) } @@ -77,8 +85,16 @@ export class EmbedFileJob { logger.info(`[EmbedFileJob] Services ready. Processing file: ${fileName}`) - // Update progress starting - await this.safeUpdateProgress(job, 5) + // Anchor initial progress to where we are in the overall file. For a + // continuation batch midway through a multi-batch ZIM (e.g. offset 100k of + // 600k), the hardcoded 5 used to make the gauge briefly flash 0→5→real, + // which read as a backward jump. Fall back to 5 for single-batch files + // where totalArticles isn't set. + const initialPercent = + totalArticles && totalArticles > 0 + ? Math.min(99, Math.round(((batchOffset || 0) / totalArticles) * 100)) + : 5 + await this.safeUpdateProgress(job, initialPercent) await job.updateData({ ...job.data, status: 'processing', @@ -87,9 +103,25 @@ export class EmbedFileJob { logger.info(`[EmbedFileJob] Processing file: ${filePath}`) - // Progress callback: maps service-reported 0-100% into the 5-95% job range + // Progress callback. For multi-batch ZIM ingestions, scale the service-reported + // 0-100% (which is % through the current batch's chunks) into the overall-file + // frame so the UI gauge climbs monotonically across the many continuation jobs + // BullMQ creates per file. Without this, every new continuation jobId resets the + // gauge to ~5% and the user sees ingestion progress "jumping around" between + // each batch's local frame and the end-of-batch overall-file overwrite below. + // + // For single-batch files (uploaded PDFs, txts) totalArticles is undefined and + // we fall back to the original 5-95% per-job range, which is what the UI expects + // for a one-shot file with no continuations. const onProgress = async (percent: number) => { - await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9))) + const useOverallFrame = totalArticles && totalArticles > 0 + if (useOverallFrame) { + const articlesDone = (batchOffset || 0) + (percent / 100) * ZIM_BATCH_SIZE + const overallPercent = Math.min(99, Math.round((articlesDone / totalArticles) * 100)) + await this.safeUpdateProgress(job, overallPercent) + } else { + await this.safeUpdateProgress(job, Math.min(95, Math.round(5 + percent * 0.9))) + } } // Process and embed the file @@ -114,6 +146,19 @@ export class EmbedFileJob { `[EmbedFileJob] Batch complete. Dispatching next batch at offset ${nextOffset}` ) + // Pace continuation batches when embedding is CPU-bound. Sustained 100% CPU + // saturation across all cores during multi-batch ZIM ingestion can starve + // other services (sshd has been seen to lose responsiveness hard enough to + // require a power-cycle). When GPU-accelerated, embeddings stream through + // the GPU and CPUs stay free — no pacing needed. + const isGpuAccelerated = await ollamaService.isEmbeddingGpuAccelerated() + if (!isGpuAccelerated) { + logger.info( + `[EmbedFileJob] Embedding is CPU-only — pacing ${EmbedFileJob.CPU_BATCH_DELAY_MS}ms before dispatching next batch` + ) + await new Promise((resolve) => setTimeout(resolve, EmbedFileJob.CPU_BATCH_DELAY_MS)) + } + // Dispatch next batch (not final yet) await EmbedFileJob.dispatch({ filePath, @@ -157,6 +202,18 @@ export class EmbedFileJob { chunks: totalChunks, }) + // Persist the post-job state so scanAndSyncStorage knows this file is done. + // BullMQ's :completed retention (50 jobs) ages out, so the state row is + // the only durable record of "this file finished embedding". + try { + await KbIngestState.markIndexed(filePath, totalChunks) + } catch (stateErr) { + logger.warn( + `[EmbedFileJob] Failed to persist ingest state for ${fileName}: %s`, + stateErr instanceof Error ? stateErr.message : String(stateErr) + ) + } + const batchMsg = isZimBatch ? ` (final batch, total chunks: ${totalChunks})` : '' logger.info( `[EmbedFileJob] Successfully embedded ${result.chunks} chunks from file: ${fileName}${batchMsg}` @@ -179,64 +236,125 @@ export class EmbedFileJob { error: error instanceof Error ? error.message : 'Unknown error', }) + // Only persist `failed` for unrecoverable errors. Retryable errors get + // automatic BullMQ retries (30 attempts); marking state failed on every + // transient blip would suppress the retry-driven recovery path. + if (error instanceof UnrecoverableError) { + try { + await KbIngestState.markFailed( + filePath, + error instanceof Error ? error.message : 'Unknown error' + ) + } catch (stateErr) { + logger.warn( + `[EmbedFileJob] Failed to persist failed state for ${fileName}: %s`, + stateErr instanceof Error ? stateErr.message : String(stateErr) + ) + } + } + throw error } } static async listActiveJobs(): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobs = await queue.getJobs(['waiting', 'active', 'delayed']) - return jobs.map((job) => ({ - jobId: job.id!.toString(), - fileName: (job.data as EmbedFileJobParams).fileName, - filePath: (job.data as EmbedFileJobParams).filePath, - progress: typeof job.progress === 'number' ? job.progress : 0, - status: ((job.data as any).status as string) ?? 'waiting', - })) + return jobs.map((job) => { + const data = job.data as EmbedFileJobParams & { + status?: string + lastBatchAt?: number + startedAt?: number + chunks?: number + } + return { + jobId: job.id!.toString(), + fileName: data.fileName, + filePath: data.filePath, + progress: typeof job.progress === 'number' ? job.progress : 0, + status: data.status ?? 'waiting', + lastBatchAt: data.lastBatchAt, + startedAt: data.startedAt, + chunks: data.chunks, + } + }) } static async getByFilePath(filePath: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(filePath) return await queue.getJob(jobId) } - static async dispatch(params: EmbedFileJobParams) { - const queueService = new QueueService() + static async dispatch(params: EmbedFileJobParams, options?: { force?: boolean }) { + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) - const jobId = this.getJobId(params.filePath) + + // Continuation batches (batchOffset > 0) must NOT reuse the deterministic + // per-file jobId. Two BullMQ dedupe paths would otherwise silently swallow them: + // 1) The parent batch's handle() calls dispatch() before returning, so the + // parent job is still `active` and locked — queue.add() with the same + // jobId returns the locked parent rather than enqueueing the new batch. + // 2) After the parent completes, its entry stays in `completed` (held by + // `removeOnComplete: { count: 50 }`), still tripping jobId dedupe. + // Letting BullMQ auto-generate a unique jobId for continuation batches stacks + // them as independent queue entries that each process via handle(). + // Initial dispatches keep the deterministic jobId so re-triggering an install + // (UI re-click, sync rescan, etc.) is still idempotent. + // `force` skips the deterministic jobId for bulk callers (reembedAll / + // resetAndRebuild) where historical entries in :completed would otherwise + // silently swallow the new dispatch. + const isContinuation = !!(params.batchOffset && params.batchOffset > 0) + const force = !!options?.force + const initialJobId = this.getJobId(params.filePath) + + const jobOptions: Parameters[2] = { + attempts: 30, + backoff: { + type: 'fixed', + delay: 60000, // Check every 60 seconds for service readiness + }, + removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history + removeOnFail: { count: 20 }, // Keep last 20 failed jobs for debugging + } + if (!isContinuation && !force) { + jobOptions.jobId = initialJobId + } try { - const job = await queue.add(this.key, params, { - jobId, - attempts: 30, - backoff: { - type: 'fixed', - delay: 60000, // Check every 60 seconds for service readiness - }, - removeOnComplete: { count: 50 }, // Keep last 50 completed jobs for history - removeOnFail: { count: 20 } // Keep last 20 failed jobs for debugging - }) + const job = await queue.add(this.key, params, jobOptions) - logger.info(`[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}`) + const label = isContinuation + ? ` (continuation @ offset ${params.batchOffset})` + : force + ? ' (forced re-dispatch)' + : '' + logger.info( + `[EmbedFileJob] Dispatched embedding job for file: ${params.fileName}${label}` + ) return { job, created: true, - jobId, + jobId: job.id ?? initialJobId, message: `File queued for embedding: ${params.fileName}`, } } catch (error) { - if (error.message && error.message.includes('job already exists')) { - const existing = await queue.getJob(jobId) + if ( + !isContinuation && + !force && + error.message && + error.message.includes('job already exists') + ) { + const existing = await queue.getJob(initialJobId) logger.info(`[EmbedFileJob] Job already exists for file: ${params.fileName}`) return { job: existing, created: false, - jobId, + jobId: initialJobId, message: `Embedding job already exists for: ${params.fileName}`, } } @@ -245,7 +363,7 @@ export class EmbedFileJob { } static async listFailedJobs(): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) // Jobs that have failed at least once are in 'delayed' (retrying) or terminal 'failed' state. // We identify them by job.data.status === 'failed' set in the catch block of handle(). @@ -264,7 +382,7 @@ export class EmbedFileJob { } static async cleanupFailedJobs(): Promise<{ cleaned: number; filesDeleted: number }> { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const allJobs = await queue.getJobs(['waiting', 'delayed', 'failed']) const failedJobs = allJobs.filter((job) => (job.data as any).status === 'failed') diff --git a/admin/app/jobs/run_benchmark_job.ts b/admin/app/jobs/run_benchmark_job.ts index 0ae41e8..962e663 100644 --- a/admin/app/jobs/run_benchmark_job.ts +++ b/admin/app/jobs/run_benchmark_job.ts @@ -53,7 +53,7 @@ export class RunBenchmarkJob { } static async dispatch(params: RunBenchmarkJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) try { @@ -89,7 +89,7 @@ export class RunBenchmarkJob { } static async getJob(benchmarkId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) return await queue.getJob(benchmarkId) } diff --git a/admin/app/jobs/run_download_job.ts b/admin/app/jobs/run_download_job.ts index 12b3532..8a219e9 100644 --- a/admin/app/jobs/run_download_job.ts +++ b/admin/app/jobs/run_download_job.ts @@ -31,7 +31,7 @@ export class RunDownloadJob { /** Signal cancellation via Redis so the worker process can pick it up */ static async signalCancel(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const client = await queue.client await client.set(this.cancelKey(jobId), '1', 'EX', 300) // 5 min TTL @@ -46,7 +46,7 @@ export class RunDownloadJob { RunDownloadJob.abortControllers.set(job.id!, abortController) // Get Redis client for checking cancel signals from the API process - const queueService = new QueueService() + const queueService = QueueService.getInstance() const cancelRedis = await queueService.getQueue(RunDownloadJob.queue).client let lastKnownProgress: Pick = { @@ -147,13 +147,40 @@ export class RunDownloadJob { // Only dispatch embedding job if AI Assistant (Ollama) is installed const ollamaUrl = await dockerService.getServiceURL('nomad_ollama') if (ollamaUrl) { - try { - await EmbedFileJob.dispatch({ - fileName: url.split('/').pop() || '', - filePath: filepath, - }) - } catch (error) { - console.error(`[RunDownloadJob] Error dispatching EmbedFileJob for URL ${url}:`, error) + // Respect the global ingest policy. Under Manual, record the file + // as pending_decision so the KB panel surfaces the per-file Index + // affordance (PR #909) instead of silently auto-embedding behind + // the user's back. Unset is treated as Always to preserve legacy + // behavior — mirrors rag_service.ts:1587-1588. + const { default: KVStore } = await import('#models/kv_store') + const { default: KbIngestState } = await import('#models/kb_ingest_state') + const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy') + const policy: 'Always' | 'Manual' = policyRaw === 'Manual' ? 'Manual' : 'Always' + + if (policy === 'Manual') { + try { + // firstOrCreate so a re-download doesn't demote an existing + // indexed/failed row — user keeps prior state and can re-index + // explicitly from the KB panel if they want fresh content. + await KbIngestState.firstOrCreate( + { file_path: filepath }, + { file_path: filepath, state: 'pending_decision', chunks_embedded: 0 } + ) + } catch (error) { + console.error( + `[RunDownloadJob] Error recording pending_decision state for ${filepath}:`, + error + ) + } + } else { + try { + await EmbedFileJob.dispatch({ + fileName: url.split('/').pop() || '', + filePath: filepath, + }) + } catch (error) { + console.error(`[RunDownloadJob] Error dispatching EmbedFileJob for URL ${url}:`, error) + } } } } else if (filetype === 'map') { @@ -199,7 +226,7 @@ export class RunDownloadJob { } static async getByUrl(url: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(url) return await queue.getJob(jobId) @@ -229,7 +256,7 @@ export class RunDownloadJob { } static async dispatch(params: RunDownloadJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(params.url) diff --git a/admin/app/jobs/run_extract_pmtiles_job.ts b/admin/app/jobs/run_extract_pmtiles_job.ts index 73c4eed..de7049f 100644 --- a/admin/app/jobs/run_extract_pmtiles_job.ts +++ b/admin/app/jobs/run_extract_pmtiles_job.ts @@ -49,7 +49,7 @@ export class RunExtractPmtilesJob { } static async signalCancel(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const client = await queue.client await client.set(this.cancelKey(jobId), '1', 'EX', 300) @@ -77,7 +77,7 @@ export class RunExtractPmtilesJob { `maxzoom=${maxzoom ?? 'source-max'} out=${outputFilepath}` ) - const queueService = new QueueService() + const queueService = QueueService.getInstance() const cancelRedis = await queueService.getQueue(RunExtractPmtilesJob.queue).client let userCancelled = false @@ -249,13 +249,13 @@ export class RunExtractPmtilesJob { } static async getById(jobId: string): Promise { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) return await queue.getJob(jobId) } static async dispatch(params: RunExtractPmtilesJobParams) { - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue(this.queue) const jobId = this.getJobId(params.sourceUrl, params.regionFilepath, params.maxzoom) diff --git a/admin/app/models/chat_message.ts b/admin/app/models/chat_message.ts index da93e00..f0633ff 100644 --- a/admin/app/models/chat_message.ts +++ b/admin/app/models/chat_message.ts @@ -18,7 +18,7 @@ export default class ChatMessage extends BaseModel { @column() declare content: string - @belongsTo(() => ChatSession, { foreignKey: 'id', localKey: 'session_id' }) + @belongsTo(() => ChatSession, { foreignKey: 'session_id', localKey: 'id' }) declare session: BelongsTo @column.dateTime({ autoCreate: true }) diff --git a/admin/app/models/kb_ingest_state.ts b/admin/app/models/kb_ingest_state.ts new file mode 100644 index 0000000..2add407 --- /dev/null +++ b/admin/app/models/kb_ingest_state.ts @@ -0,0 +1,77 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' + +const LAST_ERROR_MAX_LEN = 1024 + +/** + * Tracks the per-file decision and outcome of AI knowledge-base ingestion. + * + * The row exists for any embeddable file the scanner has seen and is independent + * of `installed_resources` (which only covers curated downloads). Replaces the + * earlier "any chunks in qdrant ⇒ embedded" binary check, which conflated + * partially-stalled ingestions with fully-indexed files. See RFC #883. + */ +export default class KbIngestState extends BaseModel { + static table = 'kb_ingest_state' + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare file_path: string + + @column() + declare state: KbIngestStateValue + + @column() + declare chunks_embedded: number + + @column() + declare last_error: string | null + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime + + static async getOrCreate(filePath: string): Promise { + return this.firstOrCreate( + { file_path: filePath }, + { file_path: filePath, state: 'pending_decision', chunks_embedded: 0 } + ) + } + + static async markIndexed(filePath: string, chunksEmbedded: number): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'indexed' + row.chunks_embedded = chunksEmbedded + row.last_error = null + await row.save() + } + + static async markFailed(filePath: string, errorMessage: string): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'failed' + row.last_error = errorMessage.slice(0, LAST_ERROR_MAX_LEN) + await row.save() + } + + static async markBrowseOnly(filePath: string): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'browse_only' + await row.save() + } + + static async markStalled(filePath: string): Promise { + const row = await this.getOrCreate(filePath) + row.state = 'stalled' + await row.save() + } + + static async remove(filePath: string): Promise { + await this.query().where('file_path', filePath).delete() + } +} diff --git a/admin/app/models/kb_ratio_registry.ts b/admin/app/models/kb_ratio_registry.ts new file mode 100644 index 0000000..13731b8 --- /dev/null +++ b/admin/app/models/kb_ratio_registry.ts @@ -0,0 +1,67 @@ +import { DateTime } from 'luxon' +import { BaseModel, column, SnakeCaseNamingStrategy } from '@adonisjs/lucid/orm' +import { + findChunksPerMb, + estimateChunkCount, + estimateBatch, + type BatchEstimate, + type BatchEstimateInput, +} from '../utils/kb_ratio_lookup.js' + +/** + * Self-calibrating registry of `{filename-prefix → chunks_per_mb}` ratios used + * for disk-footprint and time-to-embed estimates surfaced in the KB panel. + * + * Migration seeds the registry with heuristic defaults from the RFC #883 + * appendix; Phase 4 self-calibration will update rows in place as ZIMs finish + * ingesting and the real ratio becomes known. Lookup is longest-prefix-match + * (see `kb_ratio_lookup.ts`) so a specific entry (`wikipedia_en_simple_`) + * overrides a broader one (`wikipedia_en_`). + */ +export default class KbRatioRegistry extends BaseModel { + static table = 'kb_ratio_registry' + static namingStrategy = new SnakeCaseNamingStrategy() + + @column({ isPrimary: true }) + declare id: number + + @column() + declare pattern: string + + @column() + declare chunks_per_mb: number + + @column() + declare sample_count: number + + @column() + declare notes: string | null + + @column.dateTime({ autoCreate: true }) + declare created_at: DateTime + + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updated_at: DateTime + + /** Look up chunks_per_mb for a filename by longest-prefix match. */ + static async lookup(filename: string): Promise { + const rows = await this.all() + return findChunksPerMb(filename, rows) + } + + /** Estimate total chunks for a file of the given size on disk. */ + static async estimateChunks(filename: string, fileSizeBytes: number): Promise { + const rows = await this.all() + return estimateChunkCount(filename, fileSizeBytes, rows) + } + + /** + * Aggregate an embedding-disk-cost estimate across a batch of files. Used by + * the curated-tier-change UI to show "you're about to add ~X GB of + * embeddings on top of the ZIM downloads" before the user commits. + */ + static async estimateBatch(files: BatchEstimateInput[]): Promise { + const rows = await this.all() + return estimateBatch(files, rows) + } +} diff --git a/admin/app/services/collection_manifest_service.ts b/admin/app/services/collection_manifest_service.ts index bc69368..2479f82 100644 --- a/admin/app/services/collection_manifest_service.ts +++ b/admin/app/services/collection_manifest_service.ts @@ -5,6 +5,8 @@ import { DateTime } from 'luxon' import { join } from 'path' import CollectionManifest from '#models/collection_manifest' import InstalledResource from '#models/installed_resource' +import { QueueService } from './queue_service.js' +import { RunDownloadJob } from '#jobs/run_download_job' import { zimCategoriesSpecSchema, mapsSpecSchema, wikipediaSpecSchema } from '#validators/curated_collections' import { ensureDirectoryExists, @@ -98,10 +100,74 @@ export class CollectionManifestService { const installedResources = await InstalledResource.query().where('resource_type', 'zim') const installedMap = new Map(installedResources.map((r) => [r.resource_id, r])) - return spec.categories.map((category) => ({ - ...category, - installedTierSlug: this.getInstalledTierForCategory(category.tiers, installedMap), - })) + // In-flight ZIM download resource IDs from the BullMQ queue. Used to + // surface the user's tier intent immediately on submit, before any single + // file has finished downloading. Failed jobs are excluded so a stuck + // queue entry doesn't keep claiming the user's pick forever. + const inFlightIds = await this.getInFlightZimResourceIds() + + return spec.categories.map((category) => { + const installedTierSlug = this.getInstalledTierForCategory(category.tiers, installedMap) + const downloadingTierSlug = this.getDownloadingTierForCategory( + category.tiers, + installedMap, + inFlightIds, + installedTierSlug + ) + return { ...category, installedTierSlug, downloadingTierSlug } + }) + } + + private async getInFlightZimResourceIds(): Promise> { + const ids = new Set() + try { + const queue = QueueService.getInstance().getQueue(RunDownloadJob.queue) + const jobs = await queue.getJobs(['waiting', 'active', 'delayed']) + for (const job of jobs) { + if (job.data?.filetype !== 'zim') continue + const resourceId = job.data?.resourceMetadata?.resource_id + if (typeof resourceId === 'string') ids.add(resourceId) + } + } catch (error) { + // Don't fail the whole categories endpoint if the queue is briefly + // unreachable — just report no in-flight downloads. + logger.warn('[CollectionManifestService] Could not read download queue:', error?.message || error) + } + return ids + } + + /** + * Highest tier whose every resource is installed OR has an in-flight + * download. Returns undefined when there are no in-flight downloads for this + * category, or when the result would just duplicate installedTierSlug (i.e. + * everything that's downloading is already installed — nothing new to show). + */ + getDownloadingTierForCategory( + tiers: SpecTier[], + installedMap: Map, + inFlightIds: Set, + installedTierSlug: string | undefined + ): string | undefined { + if (inFlightIds.size === 0) return undefined + + // Cheap pre-check: any of this category's resources actually in flight? + const anyInFlight = tiers.some((tier) => + CollectionManifestService.resolveTierResources(tier, tiers).some((r) => inFlightIds.has(r.id)) + ) + if (!anyInFlight) return undefined + + const reversedTiers = [...tiers].reverse() + for (const tier of reversedTiers) { + const resolved = CollectionManifestService.resolveTierResources(tier, tiers) + if (resolved.length === 0) continue + const allAccountedFor = resolved.every( + (r) => installedMap.has(r.id) || inFlightIds.has(r.id) + ) + if (allAccountedFor) { + return tier.slug === installedTierSlug ? undefined : tier.slug + } + } + return undefined } async getMapCollectionsWithStatus(): Promise { diff --git a/admin/app/services/docker_service.ts b/admin/app/services/docker_service.ts index 7e8fc4e..7f9d4f1 100644 --- a/admin/app/services/docker_service.ts +++ b/admin/app/services/docker_service.ts @@ -291,8 +291,12 @@ export class DockerService { /** * Force reinstall a service by stopping, removing, and recreating its container. - * This method will also clear any associated volumes/data. - * Handles edge cases gracefully (e.g., container not running, container not found). + * + * Volume handling: removes Docker-managed named volumes whose name equals + * `serviceName`, starts with `${serviceName}_`, or carries a `service=${serviceName}` + * label. Host bind mounts are NOT touched — any data living on a bind-mounted + * host path (ZIM stores, model caches, MySQL data dir, etc.) survives the reinstall. + * Anonymous volumes (random hash names) are also not matched. */ async forceReinstall(serviceName: string): Promise<{ success: boolean; message: string }> { try { @@ -365,7 +369,10 @@ export class DockerService { const volumes = await this.docker.listVolumes() const serviceVolumes = volumes.Volumes?.filter( - (v) => v.Name.includes(serviceName) || v.Labels?.service === serviceName + (v) => + v.Name === serviceName || + v.Name.startsWith(`${serviceName}_`) || + v.Labels?.service === serviceName ) || [] for (const vol of serviceVolumes) { @@ -656,7 +663,7 @@ export class DockerService { await KVStore.setValue('chat.suggestionsEnabled', false) logger.info('[DockerService] Ollama installation complete. Triggering Nomad docs discovery...') - + // Need to use dynamic imports here to avoid circular dependency const ollamaService = new (await import('./ollama_service.js')).OllamaService() const ragService = new (await import('./rag_service.js')).RagService(this, ollamaService) @@ -1103,13 +1110,17 @@ export class DockerService { this.activeInstallations.add(serviceName) - // Compute new image string. AMD-on-Ollama overrides this to the rolling :rocm tag - // (set during GPU detection below) since per-version ROCm tags aren't always published. + // newImage = the semver tag we record in the DB after the update (e.g. ollama/ollama:0.23.2). + // runtimeImage = the tag we actually pull and run. For AMD-on-Ollama these diverge: we run + // the rolling :rocm tag because per-version ROCm tags aren't always published, but the DB + // must keep the semver tag so the Apps page shows the actual version (not literally "rocm") + // and the registry update-check parses a valid tag (instead of looping on the same update). const currentImage = service.container_image const imageBase = currentImage.includes(':') ? currentImage.substring(0, currentImage.lastIndexOf(':')) : currentImage - let newImage = `${imageBase}:${targetVersion}` + const newImage = `${imageBase}:${targetVersion}` + let runtimeImage = newImage // GPU detection runs before the pull so AMD updates pull ollama/ollama:rocm rather // than the standard tag. Detection result is reused below when building the new @@ -1137,7 +1148,7 @@ export class DockerService { 'update-gpu-config', `AMD GPU detected. Using ROCm image with /dev/kfd and /dev/dri passthrough...` ) - newImage = 'ollama/ollama:rocm' + runtimeImage = 'ollama/ollama:rocm' updatedAmdDevices = await this._discoverAMDDevices() updatedAmdGpuConfigured = true } else { @@ -1158,9 +1169,9 @@ export class DockerService { } } - // Step 1: Pull new image - this._broadcast(serviceName, 'update-pulling', `Pulling image ${newImage}...`) - const pullStream = await this.docker.pull(newImage) + // Step 1: Pull new image (runtimeImage diverges from newImage for AMD, see above) + this._broadcast(serviceName, 'update-pulling', `Pulling image ${runtimeImage}...`) + const pullStream = await this.docker.pull(runtimeImage) await new Promise((res) => this.docker.modem.followProgress(pullStream, res)) // Step 2: Find and stop existing container @@ -1205,9 +1216,9 @@ export class DockerService { } const newContainerConfig: any = { - Image: newImage, + Image: runtimeImage, name: serviceName, - Env: finalEnv.length > 0 ? finalEnv : undefined, + Env: inspectData.Config?.Env || undefined, Cmd: inspectData.Config?.Cmd || undefined, ExposedPorts: inspectData.Config?.ExposedPorts || undefined, WorkingDir: inspectData.Config?.WorkingDir || undefined, diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index fe0cb1c..526407c 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -3,7 +3,7 @@ import OpenAI from 'openai' import type { ChatCompletionChunk, ChatCompletionMessageParam } from 'openai/resources/chat/completions.js' import type { Stream } from 'openai/streaming.js' import { NomadOllamaModel } from '../../types/ollama.js' -import { FALLBACK_RECOMMENDED_OLLAMA_MODELS } from '../../constants/ollama.js' +import { EMBEDDING_MODEL_NAME, FALLBACK_RECOMMENDED_OLLAMA_MODELS } from '../../constants/ollama.js' import fs from 'node:fs/promises' import path from 'node:path' import logger from '@adonisjs/core/services/logger' @@ -469,6 +469,18 @@ export class OllamaService { } } + /** + * Hard char cap per embed input, applied as a runtime safety net regardless of + * which backend path runs. The chunker in RagService caps at MAX_SAFE_TOKENS=1600 + * (3200 chars at the conservative 2 chars/token estimate), but dense technical + * content has been observed to slip past on multi-batch ZIM ingestion (#881). + * + * 4000 chars ≈ 1000–2000 tokens depending on density, which keeps us comfortably + * under nomic-embed-text:v1.5's default 2048-token context even on the OpenAI-compat + * fallback path (which can't pass `truncate:true`/`num_ctx` to the model). + */ + public static readonly EMBED_MAX_INPUT_CHARS = 4000 + /** * Generate embeddings for the given input strings. * Tries the Ollama native /api/embed endpoint first, falls back to /v1/embeddings. @@ -479,6 +491,28 @@ export class OllamaService { throw new Error('AI service is not initialized.') } + // Runtime safety net (#881). The OpenAI-compat fallback has no equivalent of + // truncate:true, so a chunk that exceeds the model's loaded context_length + // (often 2048 for nomic-embed-text:v1.5) returns 400 and the chunk is silently + // dropped from Qdrant. Pre-capping at the input layer protects both paths. + const safeInput = input.map((s) => + s.length > OllamaService.EMBED_MAX_INPUT_CHARS + ? s.slice(0, OllamaService.EMBED_MAX_INPUT_CHARS) + : s + ) + const truncatedCount = input.reduce( + (n, s) => (s.length > OllamaService.EMBED_MAX_INPUT_CHARS ? n + 1 : n), + 0 + ) + if (truncatedCount > 0) { + logger.debug( + '[OllamaService] embed: pre-capped %d/%d inputs at %d chars', + truncatedCount, + input.length, + OllamaService.EMBED_MAX_INPUT_CHARS + ) + } + try { // Prefer Ollama native endpoint (supports batch input natively). // Pass num_ctx explicitly so we don't depend on the embedding model's @@ -491,7 +525,7 @@ export class OllamaService { `${this.baseUrl}/api/embed`, { model, - input, + input: safeInput, truncate: true, options: { num_ctx: 8192 }, }, @@ -503,16 +537,130 @@ export class OllamaService { throw new Error('Invalid /api/embed response — missing embeddings array') } return { embeddings: response.data.embeddings } - } catch { - // Fall back to OpenAI-compatible /v1/embeddings + } catch (err) { + // Capture the original error so we know *why* we fell back. Earlier bare + // catches here masked recurring "input length exceeds context length" + // failures for months (#369, #670, #881) — without this log we have no + // signal that /api/embed is the broken path vs the fallback. + logger.warn( + '[OllamaService] /api/embed failed, falling back to /v1/embeddings: %s', + err instanceof Error ? err.message : String(err) + ) + // Fall back to OpenAI-compatible /v1/embeddings. // Explicitly request float format — some backends (e.g. LM Studio) don't reliably // implement the base64 encoding the OpenAI SDK requests by default. - logger.info('[OllamaService] /api/embed unavailable, falling back to /v1/embeddings') - const results = await this.openai.embeddings.create({ model, input, encoding_format: 'float' }) + const results = await this.openai.embeddings.create({ + model, + input: safeInput, + encoding_format: 'float', + }) return { embeddings: results.data.map((e) => e.embedding as number[]) } } } + /** + * Returns true if Ollama is currently running an embedding model with non-zero VRAM + * (i.e., GPU-offloaded). Returns false if the model is running CPU-only OR if it's + * not currently loaded OR if /api/ps is unreachable. + * + * Used by EmbedFileJob to pace continuation batches when the embedding model is + * CPU-bound — sustained 100% CPU on a multi-batch ZIM ingestion can starve other + * services (sshd, etc.) hard enough to require a power-cycle. AMD ROCm installs + * hit this today because Ollama's ROCm build doesn't accelerate nomic-bert; on + * NVIDIA, nomic-embed-text runs at 100% GPU and pacing is unnecessary. + * + * Only the Ollama-native endpoint is supported — backends that expose + * `/v1/embeddings` (LM Studio, llama.cpp) don't surface placement info. + */ + public async isEmbeddingGpuAccelerated(): Promise { + await this._ensureDependencies() + if (!this.baseUrl) return false + + try { + const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 }) + const models: Array<{ name?: string; size_vram?: number }> = response.data?.models ?? [] + // Match any loaded model whose name signals it's an embedding model. + // nomic-embed-text, mxbai-embed-large, snowflake-arctic-embed, etc. all follow this convention. + return models.some( + (m) => m.name?.toLowerCase().includes('embed') && (m.size_vram ?? 0) > 0 + ) + } catch (err: any) { + // /api/ps unreachable (Ollama down, non-native backend, etc.) — fail closed: assume CPU, + // which means we'll pace. Better to over-pace than risk box-killing CPU saturation. + logger.warn( + `[OllamaService] Could not check embedding placement via /api/ps: ${err?.message ?? err}` + ) + return false + } + } + + /** + * Enforces the "at most one chat model resident in VRAM" invariant by firing + * `keep_alive: 0` against every currently-loaded model except (a) the + * embedding model (always exempt) and (b) `targetModel` (the one we want + * loaded next — leaving it alone preserves a hot model when the target is + * already loaded). + * + * Best-effort: queries `/api/ps` and POSTs unload hints in parallel. Network + * or Ollama errors are swallowed and logged — neither chat nor page-load + * should fail just because the unload housekeeping didn't go through. + * + * Returns the list of model names that were sent the unload hint, so the + * caller (and tests) can confirm what actually happened. + * + * Pass `targetModel: null` to unload every chat model (used for the future + * "free up VRAM" path; not exposed yet but the helper supports it). + * + * Note that `keep_alive: 0` is a post-completion hint, not a force-kill — + * Ollama defers eviction until the runner is idle, so in-flight inference + * on the same model is never interrupted. See the design doc for the race + * analysis behind this. + */ + public async unloadAllChatModelsExcept(targetModel: string | null): Promise { + await this._ensureDependencies() + if (!this.baseUrl) return [] + + let loadedModels: string[] = [] + try { + const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 }) + loadedModels = (response.data?.models ?? []) + .map((m: { name?: string }) => m.name) + .filter((name: unknown): name is string => typeof name === 'string') + } catch (err: any) { + logger.warn( + `[OllamaService] unloadAllChatModelsExcept: /api/ps unreachable, skipping unload sweep: ${err?.message ?? err}` + ) + return [] + } + + const toUnload = loadedModels.filter( + (name) => name !== EMBEDDING_MODEL_NAME && name !== targetModel + ) + + await Promise.all( + toUnload.map(async (modelName) => { + try { + await axios.post( + `${this.baseUrl}/api/generate`, + { model: modelName, prompt: '', keep_alive: 0 }, + { timeout: 10000 } + ) + } catch (err: any) { + logger.warn( + `[OllamaService] Failed to send unload hint for ${modelName}: ${err?.message ?? err}` + ) + } + }) + ) + + if (toUnload.length > 0) { + logger.info( + `[OllamaService] Sent unload hint for ${toUnload.length} chat model(s): ${toUnload.join(', ')}` + ) + } + return toUnload + } + public async getModels(includeEmbeddings = false): Promise { await this._ensureDependencies() if (!this.baseUrl) { diff --git a/admin/app/services/queue_service.ts b/admin/app/services/queue_service.ts index fa3a050..bad976b 100644 --- a/admin/app/services/queue_service.ts +++ b/admin/app/services/queue_service.ts @@ -1,9 +1,25 @@ import { Queue } from 'bullmq' import queueConfig from '#config/queue' +// Process-wide singleton. Each `Queue` opens two ioredis connections (one for +// commands, one blocking). Instantiating a fresh QueueService per dispatch / +// status lookup leaks both, and under sustained job churn (e.g. multi-batch ZIM +// ingestion enqueueing a continuation every few seconds) it saturates Redis's +// maxclients within hours. export class QueueService { private queues: Map = new Map() + private static _instance: QueueService | null = null + + private constructor() {} + + static getInstance(): QueueService { + if (!QueueService._instance) { + QueueService._instance = new QueueService() + } + return QueueService._instance + } + getQueue(name: string): Queue { if (!this.queues.has(name)) { const queue = new Queue(name, { @@ -18,5 +34,6 @@ export class QueueService { for (const queue of this.queues.values()) { await queue.close() } + this.queues.clear() } } diff --git a/admin/app/services/rag_service.ts b/admin/app/services/rag_service.ts index bd5371d..dd2e224 100644 --- a/admin/app/services/rag_service.ts +++ b/admin/app/services/rag_service.ts @@ -16,10 +16,27 @@ import { removeStopwords } from 'stopword' import { randomUUID } from 'node:crypto' import { join, resolve, sep } from 'node:path' import KVStore from '#models/kv_store' +import KbIngestState from '#models/kb_ingest_state' +import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js' +import KbRatioRegistry from '#models/kb_ratio_registry' +import { decideWarnings } from '../utils/kb_warning_decision.js' +import type { FileWarning, FileWarningsResult, StoredFileInfo } from '../../types/rag.js' +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' import { ZIMExtractionService } from './zim_extraction_service.js' import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js' +import { EMBEDDING_MODEL_NAME } from '../../constants/ollama.js' import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js' +export type EmbedSingleFileFailureCode = + | 'not_found' + | 'inflight' + | 'delete_failed' + | 'dispatch_failed' + +export type EmbedSingleFileResult = + | { success: true; message: string } + | { success: false; code: EmbedSingleFileFailureCode; message: string } + @inject() export class RagService { private qdrant: QdrantClient | null = null @@ -28,7 +45,6 @@ export class RagService { private resolvedEmbeddingModel: string | null = null public static UPLOADS_STORAGE_PATH = 'storage/kb_uploads' public static CONTENT_COLLECTION_NAME = 'nomad_knowledge_base' - public static EMBEDDING_MODEL = 'nomic-embed-text:v1.5' public static EMBEDDING_DIMENSION = 768 // Nomic Embed Text v1.5 dimension is 768 public static MODEL_CONTEXT_LENGTH = 2048 // nomic-embed-text has 2K token context public static MAX_SAFE_TOKENS = 1600 // Leave buffer for prefix and tokenization variance @@ -270,25 +286,25 @@ export class RagService { if (!this.embeddingModelVerified) { const allModels = await this.ollamaService.getModels(true) const embeddingModel = - allModels.find((model) => model.name === RagService.EMBEDDING_MODEL) ?? + allModels.find((model) => model.name === EMBEDDING_MODEL_NAME) ?? allModels.find((model) => model.name.toLowerCase().includes('nomic-embed-text')) if (!embeddingModel) { try { - const downloadResult = await this.ollamaService.downloadModel(RagService.EMBEDDING_MODEL) + const downloadResult = await this.ollamaService.downloadModel(EMBEDDING_MODEL_NAME) if (!downloadResult.success) { throw new Error(downloadResult.message || 'Unknown error during model download') } } catch (modelError) { logger.error( - `[RAG] Embedding model ${RagService.EMBEDDING_MODEL} not found locally and failed to download:`, + `[RAG] Embedding model ${EMBEDDING_MODEL_NAME} not found locally and failed to download:`, modelError ) this.embeddingModelVerified = false return null } } - this.resolvedEmbeddingModel = embeddingModel?.name ?? RagService.EMBEDDING_MODEL + this.resolvedEmbeddingModel = embeddingModel?.name ?? EMBEDDING_MODEL_NAME this.embeddingModelVerified = true } @@ -345,7 +361,7 @@ export class RagService { logger.debug(`[RAG] Embedding batch ${batchIdx + 1}/${totalBatches} (${batch.length} chunks)`) - const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? RagService.EMBEDDING_MODEL, batch) + const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? EMBEDDING_MODEL_NAME, batch) embeddings.push(...response.embeddings) @@ -500,13 +516,13 @@ export class RagService { `[RAG] Extracting ZIM content (batch: offset=${startOffset}, size=${ZIM_BATCH_SIZE})` ) - const zimChunks = await zimExtractionService.extractZIMContent(filepath, { - startOffset, - batchSize: ZIM_BATCH_SIZE, - }) + const { chunks: zimChunks, totalArticles } = await zimExtractionService.extractZIMContent( + filepath, + { startOffset, batchSize: ZIM_BATCH_SIZE } + ) logger.info( - `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata` + `[RAG] Extracted ${zimChunks.length} chunks from ZIM file with enhanced metadata (file totalArticles=${totalArticles})` ) // Process each chunk individually with its metadata @@ -582,6 +598,7 @@ export class RagService { chunks: totalChunks, hasMoreBatches, articlesProcessed: articlesInBatch, + totalArticles, } } @@ -702,7 +719,7 @@ export class RagService { /** * Main pipeline to process and embed an uploaded file into the RAG knowledge base. * This includes text extraction, chunking, embedding, and storing in Qdrant. - * + * * Orchestrates file type detection and delegates to specialized processors. * For ZIM files, supports batch processing via batchOffset parameter. */ @@ -801,12 +818,12 @@ export class RagService { if (!this.embeddingModelVerified) { const allModels = await this.ollamaService.getModels(true) const embeddingModel = - allModels.find((model) => model.name === RagService.EMBEDDING_MODEL) ?? + allModels.find((model) => model.name === EMBEDDING_MODEL_NAME) ?? allModels.find((model) => model.name.toLowerCase().includes('nomic-embed-text')) if (!embeddingModel) { logger.warn( - `[RAG] ${RagService.EMBEDDING_MODEL} not found. Cannot perform similarity search.` + `[RAG] ${EMBEDDING_MODEL_NAME} not found. Cannot perform similarity search.` ) this.embeddingModelVerified = false return [] @@ -838,7 +855,7 @@ export class RagService { return [] } - const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? RagService.EMBEDDING_MODEL, [prefixedQuery]) + const response = await this.ollamaService.embed(this.resolvedEmbeddingModel ?? EMBEDDING_MODEL_NAME, [prefixedQuery]) // Perform semantic search with a higher limit to enable reranking const searchLimit = limit * 3 // Get more results for reranking @@ -1045,7 +1062,7 @@ export class RagService { } } - public async getStoredFiles(): Promise { + public async getStoredFiles(): Promise { try { await this._ensureCollection( RagService.CONTENT_COLLECTION_NAME, @@ -1076,13 +1093,159 @@ export class RagService { offset = scrollResult.next_page_offset || null } while (offset !== null) - return Array.from(sources) + // Union the Qdrant-derived list with the disk-backed file paths the + // state machine has tracked. Without this, files known to the scanner + // but with zero embedded chunks (video-only ZIMs, failed-before-first- + // chunk ingestions, browse_only opt-outs) never get a row in Stored + // Files — which means warnings keyed off those files (#895 zero_chunks + // in particular) have no row to attach to. The state machine is the + // authoritative "what's on disk?" view; Qdrant is "what made it into + // the vector store?". Both are needed to render the KB UI honestly. + const stateByPath = new Map() + try { + const stateRows = await KbIngestState.query().select('file_path', 'state', 'chunks_embedded') + for (const row of stateRows) { + sources.add(row.file_path) + stateByPath.set(row.file_path, { + state: row.state, + chunks_embedded: row.chunks_embedded, + }) + } + } catch (error) { + // Non-fatal: if the state machine query fails for any reason we'd + // rather return the Qdrant-derived list than 500 the whole panel. + logger.warn( + { err: error }, + '[RagService.getStoredFiles] state-machine union skipped; returning Qdrant-only list' + ) + } + + return Array.from(sources).map((source) => { + const row = stateByPath.get(source) + return { + source, + state: row?.state ?? null, + chunksEmbedded: row?.chunks_embedded ?? 0, + } + }) } catch (error) { logger.error('Error retrieving stored files:', error) return [] } } + /** + * Compute whether the first-chat JIT prompt should fire and surface the file + * count the banner uses in its copy ("Index your N existing files?"). The + * banner appears when the user hasn't yet picked a global ingest policy + * (`rag.defaultIngestPolicy` unset) and the scanner has actually seen at + * least one embeddable file — i.e., the prompt is actionable, not theoretical + * on a freshly-installed empty NOMAD. + * + * Once the user picks a policy (Always or Manual) via the banner buttons or + * the KB modal toggle, `shouldPrompt` flips to false for good. + */ + public async getPolicyPromptState(): Promise<{ + shouldPrompt: boolean + hasContent: boolean + totalFiles: number + }> { + const policy = await KVStore.getValue('rag.defaultIngestPolicy') + const countRow = await KbIngestState.query().count('* as total').first() + const totalFiles = Number((countRow as any)?.$extras?.total ?? 0) + return { + shouldPrompt: policy === null && totalFiles > 0, + hasContent: totalFiles > 0, + totalFiles, + } + } + + /** + * Compute conditional warnings (RFC #883 §6) for every source the scanner + * sees on disk. Returns `{ ok, warnings }` — `ok: false` distinguishes a + * computation failure (Qdrant unreachable, DB outage, FS error) from the + * healthy-but-empty case, which is critical because the whole point of this + * surface is to expose silent failures; reporting "everything healthy" when + * we couldn't actually check would reintroduce the bug we set out to fix. + * + * Per-source chunk counts come from a single Qdrant scroll over the + * collection's points; expected-chunk estimates come from the ratio + * registry. Files in the scanner's directories that have no qdrant points + * at all show up with `chunksInQdrant: 0` so Warning A can fire. + */ + public async computeFileWarnings(): Promise { + try { + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + + // Per-source chunk count from a single scroll. We deliberately don't + // assume `kb_ingest_state.chunks_embedded` here so this PR stays + // independent of the state-machine PR (#888) — but a future cleanup can + // read from there for efficiency once both have landed. + const chunksBySource = new Map() + let offset: string | number | null | Record = null + const batchSize = 100 + do { + const scrollResult = await this.qdrant!.scroll(RagService.CONTENT_COLLECTION_NAME, { + limit: batchSize, + offset, + with_payload: ['source'], + with_vector: false, + }) + for (const point of scrollResult.points) { + const source = point.payload?.source + if (source && typeof source === 'string') { + chunksBySource.set(source, (chunksBySource.get(source) ?? 0) + 1) + } + } + offset = scrollResult.next_page_offset || null + } while (offset !== null) + + // Scan the filesystem the same way scanAndSyncStorage does so Warning A + // can fire on files with zero qdrant points (the headline "video-only + // ZIM" case). + const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) + const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) + const allSources = new Set(chunksBySource.keys()) + const sizeByPath = new Map() + + for (const dir of [KB_UPLOADS_PATH, ZIM_PATH]) { + try { + const entries = await listDirectoryContentsRecursive(dir) + for (const entry of entries) { + if (entry.type !== 'file') continue + allSources.add(entry.key) + const stat = await getFileStatsIfExists(entry.key) + if (stat) sizeByPath.set(entry.key, Number(stat.size)) + } + } catch (error: any) { + if (error?.code !== 'ENOENT') throw error + } + } + + const out: Record = {} + for (const source of allSources) { + const fileSizeBytes = sizeByPath.get(source) ?? 0 + const chunksInQdrant = chunksBySource.get(source) ?? 0 + const fileName = source.split(/[/\\]/).pop() ?? source + const expectedChunks = + fileSizeBytes > 0 + ? await KbRatioRegistry.estimateChunks(fileName, fileSizeBytes) + : null + + const warnings = decideWarnings({ fileSizeBytes, chunksInQdrant, expectedChunks }) + if (warnings.length > 0) out[source] = warnings + } + + return { ok: true, warnings: out } + } catch (error) { + logger.error('[RAG] Error computing file warnings:', error) + return { ok: false, warnings: {} } + } + } + /** * Delete all Qdrant points associated with a given source path and remove * the corresponding file from disk if it lives under the uploads directory. @@ -1117,6 +1280,11 @@ export class RagService { logger.warn(`[RAG] File was removed from knowledge base but doesn't live in Nomad's uploads directory, so it can't be safely removed. Skipping deletion of physical file...`) } + // Drop the ingest state row last so the file disappears entirely. Without + // this, the next scanAndSyncStorage would see `indexed + no chunks` for a + // path that no longer exists in storage and try to re-embed nothing. + await KbIngestState.remove(source) + return { success: true, message: 'File removed from knowledge base.' } } catch (error) { logger.error('[RAG] Error deleting file from knowledge base:', error) @@ -1181,12 +1349,182 @@ export class RagService { } } + /** + * Walk kb_uploads and zim storage directories, returning the full path of + * every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are + * filtered out so they aren't dispatched only to fail with "Unsupported file + * type" and retry on every sync. + */ + private async _discoverKbFiles(): Promise { + const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) + const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) + const filesInStorage: string[] = [] + + for (const [label, dirPath] of [ + [RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const, + [ZIM_STORAGE_PATH, ZIM_PATH] as const, + ]) { + try { + const contents = await listDirectoryContentsRecursive(dirPath) + contents.forEach((entry) => { + if (entry.type === 'file') filesInStorage.push(entry.key) + }) + logger.debug(`[RAG] Found ${contents.length} files in ${label}`) + } catch (error) { + if (error.code === 'ENOENT') { + logger.debug(`[RAG] ${label} directory does not exist, skipping`) + } else { + throw error + } + } + } + + return filesInStorage.filter((f) => determineFileType(f) !== 'unknown') + } + + /** + * Dispatch one EmbedFileJob per file path. Returns honest counts: `queuedCount` + * is jobs newly enqueued, `dedupedCount` is jobs that hit BullMQ's per-file + * jobId dedupe (an existing :completed/:waiting/etc. entry was returned + * instead of a new enqueue), and `failedPaths` lists files whose dispatch + * threw. Pass `force: true` for bulk callers that need to bypass dedupe + * entirely. Per-file errors are logged but don't abort the batch — callers + * must inspect `failedPaths` to surface partial failure to the operator. + */ + private async _dispatchEmbedJobsFor( + filePaths: string[], + options?: { force?: boolean } + ): Promise<{ queuedCount: number; dedupedCount: number; failedPaths: string[] }> { + const { EmbedFileJob } = await import('#jobs/embed_file_job') + let queuedCount = 0 + let dedupedCount = 0 + const failedPaths: string[] = [] + for (const filePath of filePaths) { + try { + const fileName = filePath.split(/[/\\]/).pop() || filePath + const stats = await getFileStatsIfExists(filePath) + const result = await EmbedFileJob.dispatch( + { + filePath, + fileName, + fileSize: stats?.size, + }, + { force: options?.force } + ) + if (result.created) { + queuedCount++ + } else { + dedupedCount++ + } + } catch (fileError) { + failedPaths.push(filePath) + logger.error(`[RAG] Error dispatching job for file ${filePath}:`, fileError) + } + } + return { queuedCount, dedupedCount, failedPaths } + } + + /** + * Dispatch an embed job for a single stored file. Wraps `_dispatchEmbedJobsFor` + * with the safety checks needed for a user-triggered per-row action: + * 1. The source must be known to the scanner OR have a state row — prevents + * arbitrary path dispatch from the public API. + * 2. We refuse if any inflight job (waiting/active/delayed/paused) already + * targets this filePath. Otherwise a double-click or a rapid retry could + * enqueue duplicate jobs, producing duplicate chunks. + * 3. When `force` is true (Re-embed of an already-indexed file), we + * pre-delete the prior Qdrant points so the new run doesn't stack on + * top of the old ones. For force=false (Index of a never-embedded file), + * there's nothing to clear. + */ + public async embedSingleFile( + source: string, + force: boolean = false + ): Promise { + const stateRow = await KbIngestState.query().where('file_path', source).first() + if (!stateRow) { + const knownFiles = await this._discoverKbFiles() + if (!knownFiles.includes(source)) { + return { + success: false, + code: 'not_found', + message: 'File is not a tracked knowledge-base source.', + } + } + } + + const { EmbedFileJob } = await import('#jobs/embed_file_job') + const { QueueService } = await import('#services/queue_service') + const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue) + const inflight = await queue.getJobs(['waiting', 'active', 'delayed', 'paused']) + if (inflight.some((j) => j.data?.filePath === source)) { + return { + success: false, + code: 'inflight', + message: 'A job for this file is already in progress. Wait for it to finish before re-queuing.', + } + } + + if (force) { + try { + await this._deletePointsBySource(source) + } catch (err) { + logger.error(`[RAG] Failed to delete prior points for ${source}; aborting re-embed:`, err) + return { + success: false, + code: 'delete_failed', + message: 'Failed to clear prior embeddings before re-embed.', + } + } + } + + const result = await this._dispatchEmbedJobsFor([source], { force }) + if (result.failedPaths.length > 0) { + return { + success: false, + code: 'dispatch_failed', + message: 'Failed to dispatch embed job for this file.', + } + } + return { + success: true, + message: force ? 'Re-embed queued for this file.' : 'Indexing queued for this file.', + } + } + + /** + * Delete all Qdrant points whose `source` payload matches the given path. + * Unlike deleteFileBySource(), this does NOT touch the file on disk — used + * by reembedAll() where the file must remain so it can be re-ingested. + */ + private async _deletePointsBySource(source: string): Promise { + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + await this.qdrant!.delete(RagService.CONTENT_COLLECTION_NAME, { + filter: { must: [{ key: 'source', match: { value: source } }] }, + }) + } + + /** + * Returns true if the file-embeddings queue has any in-flight work + * (waiting, active, delayed, or paused). Bulk re-embed actions use this + * to refuse mid-flight to avoid racing with deletes/dispatches already + * in progress. + */ + private async _hasInflightEmbedJobs(): Promise { + const { EmbedFileJob } = await import('#jobs/embed_file_job') + const { QueueService } = await import('#services/queue_service') + const queue = QueueService.getInstance().getQueue(EmbedFileJob.queue) + const counts = await queue.getJobCounts('waiting', 'active', 'delayed', 'paused') + return (counts.waiting || 0) + (counts.active || 0) + (counts.delayed || 0) + (counts.paused || 0) > 0 + } + /** * Scans the knowledge base storage directories and syncs with Qdrant. * Identifies files that exist in storage but haven't been embedded yet, * and dispatches EmbedFileJob for each missing file. - * - * @returns Object containing success status, message, and counts of scanned/queued files */ public async scanAndSyncStorage(): Promise<{ success: boolean @@ -1197,91 +1535,111 @@ export class RagService { try { logger.info('[RAG] Starting knowledge base sync scan') - const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH) - const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH) - - const filesInStorage: string[] = [] - - // Force resync of Nomad docs await this.discoverNomadDocs(true).catch((error) => { logger.error('[RAG] Error during Nomad docs discovery in sync process:', error) }) - // Scan kb_uploads directory - try { - const kbContents = await listDirectoryContentsRecursive(KB_UPLOADS_PATH) - kbContents.forEach((entry) => { - if (entry.type === 'file') { - filesInStorage.push(entry.key) - } - }) - logger.debug(`[RAG] Found ${kbContents.length} files in ${RagService.UPLOADS_STORAGE_PATH}`) - } catch (error) { - if (error.code === 'ENOENT') { - logger.debug(`[RAG] ${RagService.UPLOADS_STORAGE_PATH} directory does not exist, skipping`) - } else { - throw error - } - } + const filesInStorage = await this._discoverKbFiles() + logger.info(`[RAG] Found ${filesInStorage.length} embeddable files in storage`) - // Scan zim directory - try { - const zimContents = await listDirectoryContentsRecursive(ZIM_PATH) - zimContents.forEach((entry) => { - if (entry.type === 'file') { - filesInStorage.push(entry.key) - } - }) - logger.debug(`[RAG] Found ${zimContents.length} files in ${ZIM_STORAGE_PATH}`) - } catch (error) { - if (error.code === 'ENOENT') { - logger.debug(`[RAG] ${ZIM_STORAGE_PATH} directory does not exist, skipping`) - } else { - throw error - } - } - - logger.info(`[RAG] Found ${filesInStorage.length} total files in storage directories`) - - // Get all stored sources from Qdrant await this._ensureCollection( RagService.CONTENT_COLLECTION_NAME, RagService.EMBEDDING_DIMENSION ) + // Collect every unique `source` already in Qdrant so we can skip files + // that have already been embedded. const sourcesInQdrant = new Set() let offset: string | number | null | Record = null - const batchSize = 100 - - // Scroll through all points to get sources do { const scrollResult = await this.qdrant!.scroll(RagService.CONTENT_COLLECTION_NAME, { - limit: batchSize, - offset: offset, - with_payload: ['source'], // Only fetch source field for efficiency + limit: 100, + offset, + with_payload: ['source'], with_vector: false, }) - scrollResult.points.forEach((point) => { const source = point.payload?.source - if (source && typeof source === 'string') { - sourcesInQdrant.add(source) - } + if (source && typeof source === 'string') sourcesInQdrant.add(source) }) - offset = scrollResult.next_page_offset || null } while (offset !== null) logger.info(`[RAG] Found ${sourcesInQdrant.size} unique sources in Qdrant`) - // Find files that are in storage, not already in Qdrant, and have an embeddable type. - // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise be - // dispatched to EmbedFileJob, fail with "Unsupported file type", and retry on every sync. - const filesToEmbed = filesInStorage.filter( - (filePath) => !sourcesInQdrant.has(filePath) && determineFileType(filePath) !== 'unknown' + // Load all known per-file ingest states. The state row is authoritative + // over the "any chunks in Qdrant" heuristic — it captures user choices + // (browse_only) and terminal outcomes (failed, stalled) that aren't visible + // from Qdrant alone. See RFC #883 for the full state machine. + const stateRows = await KbIngestState.all() + const stateByPath = new Map(stateRows.map((row) => [row.file_path, row])) + + // Non-embeddable files (e.g. kiwix-library.xml in /storage/zim) would otherwise + // be dispatched to EmbedFileJob, fail with "Unsupported file type", and retry + // on every sync — filter them out before state decisions. + const embeddableFiles = filesInStorage.filter( + (filePath) => determineFileType(filePath) !== 'unknown' ) - logger.info(`[RAG] Found ${filesToEmbed.length} files that need embedding`) + // Read the global ingest policy. Unset is treated as 'Always' so legacy + // installs keep their current behavior until the user explicitly opts + // into Manual mode from the KB panel. + const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy') + const policy: IngestPolicy = policyRaw === 'Manual' ? 'Manual' : 'Always' + + const filesToEmbed: string[] = [] + let backfilled = 0 + let createdRows = 0 + let createdPending = 0 + let skipped = 0 + + for (const filePath of embeddableFiles) { + const stateRow = stateByPath.get(filePath) ?? null + const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath), policy) + + switch (action.kind) { + case 'skip': + skipped++ + break + case 'backfill_indexed': + // Pre-RFC install (or a fresh admin pointed at an existing Qdrant volume): + // chunks already exist with no state row, so trust Qdrant and record + // `indexed` without re-embedding. chunks_embedded is left 0 because + // we don't count points-per-source during the scroll above. + await KbIngestState.create({ + file_path: filePath, + state: 'indexed', + chunks_embedded: 0, + }) + backfilled++ + break + case 'create_pending': + // Manual mode: record that we've seen the file but don't dispatch. + // The KB panel surfaces a per-card "Index" affordance for these. + await KbIngestState.create({ + file_path: filePath, + state: 'pending_decision', + chunks_embedded: 0, + }) + createdPending++ + break + case 'dispatch': + if (action.createStateRow) { + await KbIngestState.create({ + file_path: filePath, + state: 'pending_decision', + chunks_embedded: 0, + }) + createdRows++ + } + filesToEmbed.push(filePath) + break + } + } + + logger.info( + `[RAG] Scan results (policy=${policy}): ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${createdPending} waiting on user, ${skipped} skipped` + ) if (filesToEmbed.length === 0) { return { @@ -1292,41 +1650,193 @@ export class RagService { } } - // Import EmbedFileJob dynamically to avoid circular dependencies - const { EmbedFileJob } = await import('#jobs/embed_file_job') - - // Dispatch jobs for files that need embedding - let queuedCount = 0 - for (const filePath of filesToEmbed) { - try { - const fileName = filePath.split(/[/\\]/).pop() || filePath - const stats = await getFileStatsIfExists(filePath) - - logger.info(`[RAG] Dispatching embed job for: ${fileName}`) - await EmbedFileJob.dispatch({ - filePath: filePath, - fileName: fileName, - fileSize: stats?.size, - }) - queuedCount++ - logger.debug(`[RAG] Successfully dispatched job for ${fileName}`) - } catch (fileError) { - logger.error(`[RAG] Error dispatching job for file ${filePath}:`, fileError) - } - } - + const { queuedCount, dedupedCount } = await this._dispatchEmbedJobsFor(filesToEmbed) + const dedupeNote = dedupedCount > 0 ? ` (${dedupedCount} already queued)` : '' return { success: true, - message: `Scanned ${filesInStorage.length} files, queued ${queuedCount} for embedding`, + message: `Scanned ${filesInStorage.length} files, queued ${queuedCount} for embedding${dedupeNote}`, filesScanned: filesInStorage.length, filesQueued: queuedCount, } } catch (error) { logger.error('[RAG] Error scanning and syncing knowledge base:', error) - return { - success: false, - message: 'Error scanning and syncing knowledge base', + return { success: false, message: 'Error scanning and syncing knowledge base' } + } + } + + /** + * Re-embed every file on disk (per-file replace). For each discovered file: + * delete its existing Qdrant points by `source` match, then dispatch a fresh + * EmbedFileJob. Files are NOT removed from disk. Any orphan points (points + * whose source file no longer exists) are intentionally preserved — use + * resetAndRebuild() if a clean slate is required. + * + * Refuses to run if the embeddings queue already has in-flight work. + */ + public async reembedAll(): Promise<{ + success: boolean + message: string + filesScanned?: number + filesQueued?: number + failedPaths?: string[] + }> { + try { + if (await this._hasInflightEmbedJobs()) { + return { + success: false, + message: 'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a bulk re-embed.', + } } + + logger.info('[RAG] Starting full re-embed (per-file replace)') + + await this.discoverNomadDocs(true).catch((error) => { + logger.error('[RAG] Error re-running Nomad docs discovery during re-embed:', error) + }) + + const filesInStorage = await this._discoverKbFiles() + + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + + // Per-file: delete-then-dispatch. We tried dispatch-then-delete but that + // opens a race where a fast worker can write new points before our + // delete-by-source runs, wiping both. Instead we delete first, then + // dispatch — and if dispatch fails, we surface the failed paths in the + // response so the operator knows which files dropped out (rather than + // silently leaving them unindexed). A subsequent sync rescan picks them + // back up. Note: a delete-failure aborts the per-file pair (we don't + // dispatch a job whose old points are still present, since they'd live + // alongside the new vectors forever). + const { EmbedFileJob } = await import('#jobs/embed_file_job') + let queuedCount = 0 + const failedPaths: string[] = [] + for (const filePath of filesInStorage) { + try { + await this._deletePointsBySource(filePath) + } catch (err) { + logger.error(`[RAG] Failed to delete prior points for ${filePath}; skipping dispatch:`, err) + failedPaths.push(filePath) + continue + } + try { + const fileName = filePath.split(/[/\\]/).pop() || filePath + const stats = await getFileStatsIfExists(filePath) + const result = await EmbedFileJob.dispatch( + { filePath, fileName, fileSize: stats?.size }, + { force: true } + ) + if (result.created) queuedCount++ + } catch (fileError) { + // Old points already deleted but the new job never made it onto the + // queue. Logged + surfaced so an operator can rerun a sync. + logger.error(`[RAG] Re-embed dispatch failed for ${filePath} after delete; file is now unindexed until next sync:`, fileError) + failedPaths.push(filePath) + } + } + + logger.info( + `[RAG] Re-embed dispatched ${queuedCount}/${filesInStorage.length} files` + + (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '') + ) + + const failureSuffix = + failedPaths.length > 0 + ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.` + : '' + + return { + success: failedPaths.length === 0, + message: + `Re-embedding ${queuedCount} file${queuedCount === 1 ? '' : 's'}. Existing points were replaced.` + + failureSuffix, + filesScanned: filesInStorage.length, + filesQueued: queuedCount, + ...(failedPaths.length > 0 ? { failedPaths } : {}), + } + } catch (error) { + logger.error('[RAG] Error during re-embed:', error) + return { success: false, message: 'Error during re-embed' } + } + } + + /** + * Destructive rebuild. Drops the entire Qdrant collection (wiping every + * point including orphans), recreates it with the correct dimension, clears + * the Nomad-docs discovery flag, then dispatches an EmbedFileJob for every + * file currently on disk. + * + * Refuses to run if the embeddings queue already has in-flight work. + */ + public async resetAndRebuild(): Promise<{ + success: boolean + message: string + filesScanned?: number + filesQueued?: number + failedPaths?: string[] + }> { + try { + if (await this._hasInflightEmbedJobs()) { + return { + success: false, + message: 'Embed jobs are already in progress. Wait for the queue to drain (or clean up failed jobs) before triggering a reset.', + } + } + + logger.info('[RAG] Starting destructive reset & rebuild') + + await this._initializeQdrantClient() + try { + await this.qdrant!.deleteCollection(RagService.CONTENT_COLLECTION_NAME) + logger.info(`[RAG] Dropped collection ${RagService.CONTENT_COLLECTION_NAME}`) + } catch (err) { + // Collection may not exist yet on a fresh install — log and continue. + logger.warn(`[RAG] deleteCollection failed (may not exist): ${(err as Error).message}`) + } + + await this._ensureCollection( + RagService.CONTENT_COLLECTION_NAME, + RagService.EMBEDDING_DIMENSION + ) + + // Force Nomad docs to be re-dispatched. + await KVStore.setValue('rag.docsEmbedded', false) + await this.discoverNomadDocs(true).catch((error) => { + logger.error('[RAG] Error re-running Nomad docs discovery after reset:', error) + }) + + const filesInStorage = await this._discoverKbFiles() + const { queuedCount, failedPaths } = await this._dispatchEmbedJobsFor(filesInStorage, { + force: true, + }) + + logger.info( + `[RAG] Reset complete — dispatched ${queuedCount}/${filesInStorage.length} files` + + (failedPaths.length > 0 ? ` (${failedPaths.length} failed)` : '') + ) + + // Collection was already dropped, so dispatch failures here mean the + // file is gone from Qdrant with no pending job to repopulate it. Surface + // the count + paths so the operator can rerun a sync rescan to recover. + const failureSuffix = + failedPaths.length > 0 + ? ` ${failedPaths.length} file${failedPaths.length === 1 ? '' : 's'} failed to dispatch and are temporarily unindexed — run a sync rescan to recover.` + : '' + + return { + success: failedPaths.length === 0, + message: + `Collection wiped. Queued ${queuedCount} file${queuedCount === 1 ? '' : 's'} for a full rebuild.` + + failureSuffix, + filesScanned: filesInStorage.length, + filesQueued: queuedCount, + ...(failedPaths.length > 0 ? { failedPaths } : {}), + } + } catch (error) { + logger.error('[RAG] Error during reset & rebuild:', error) + return { success: false, message: 'Error during reset & rebuild' } } } } diff --git a/admin/app/services/system_service.ts b/admin/app/services/system_service.ts index 1a55cfb..42daac6 100644 --- a/admin/app/services/system_service.ts +++ b/admin/app/services/system_service.ts @@ -95,12 +95,40 @@ export class SystemService { if (!ollamaContainer) return null const container = this.dockerService.docker.getContainer(ollamaContainer.Id) - const buf = (await container.logs({ + + // Read logs only from the first 5 minutes after container start. The + // "inference compute" line is written once during Ollama's GPU discovery + // phase, within seconds of startup. Using tail:N here is fragile: under + // active embedding workloads we've seen >1000 lines/min, which pushes the + // line past any reasonable tail in minutes. Pinning to the startup window + // is bounded (~5 min of logs regardless of container uptime) and never + // ages out. + // + // Fall back to the previous tail:500 strategy if StartedAt is missing or + // unparseable — we can't construct a since/until window without it, but + // tail:500 is still useful when the container just started and the line + // is still recent. + const inspect = await container.inspect() + const startedAtRaw = inspect?.State?.StartedAt + const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN + const hasValidStartedAt = Number.isFinite(startedAtMs) && startedAtMs > 0 + + const logsOpts: { stdout: true; stderr: true; follow: false; since?: number; until?: number; tail?: number } = { stdout: true, stderr: true, - tail: 500, follow: false, - })) as unknown as Buffer + } + if (hasValidStartedAt) { + const startedAtSec = Math.floor(startedAtMs / 1000) + logsOpts.since = startedAtSec + logsOpts.until = startedAtSec + 300 // 5-minute window + } else { + logger.warn( + `[SystemService] nomad_ollama State.StartedAt missing or invalid (${startedAtRaw ?? 'undefined'}); falling back to tail:500 for inference-compute probe` + ) + logsOpts.tail = 500 + } + const buf = (await container.logs(logsOpts)) as unknown as Buffer const logs = buf.toString('utf8') const lines = logs.split('\n').filter((l) => l.includes('msg="inference compute"')) @@ -399,9 +427,42 @@ export class SystemService { os.kernel = dockerInfo.KernelVersion } - // If si.graphics() returned no controllers (common inside Docker), - // fall back to runtime + Ollama log probe to figure out what's accessible. - if (!graphics.controllers || graphics.controllers.length === 0) { + // si.graphics() in the admin container uses lspci (pciutils ships in + // the image for AMD detection). lspci has no real VRAM info for + // discrete GPUs, so systeminformation parses the first PCI memory + // Region (BAR0, typically 1-32 MiB) as `vram`. nvidia-smi / ROCm + // tooling enrichment also can't run since neither is in the admin + // image. No real dGPU has under 256 MiB, so any discrete-GPU controller + // below that threshold needs the probes below to give us real data. + // Applies to both NVIDIA and AMD; Intel iGPUs are exempt because their + // shared-system-memory VRAM reading via lspci can legitimately be small. + const DGPU_BOGUS_VRAM_THRESHOLD_MIB = 256 + const isDiscreteGpuVendor = (vendor: string) => + /nvidia|advanced micro devices|amd|ati/i.test(vendor) + const isBogusDgpuVram = (c: { vendor?: string; vram?: number | null }) => + isDiscreteGpuVendor(c.vendor || '') && + typeof c.vram === 'number' && + c.vram < DGPU_BOGUS_VRAM_THRESHOLD_MIB + + // Clear the bogus value up front. If a probe replaces the entry below + // we get the real VRAM; if no probe succeeds (Ollama not installed, + // passthrough_failed) the UI falls back to "N/A" instead of showing + // "1 MB" / "32 MB". The lspci model/vendor strings stay since they're + // still useful for identifying the card. + const hasLspciBogusDgpuVram = (graphics.controllers || []).some(isBogusDgpuVram) + if (hasLspciBogusDgpuVram) { + for (const c of graphics.controllers) { + if (isBogusDgpuVram(c)) c.vram = null + } + } + + // Run the probes when controllers are empty (common inside Docker) or + // when lspci gave us bogus discrete-GPU BAR0 values that need replacing. + if ( + !graphics.controllers || + graphics.controllers.length === 0 || + hasLspciBogusDgpuVram + ) { const runtimes = dockerInfo.Runtimes || {} gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes diff --git a/admin/app/services/zim_extraction_service.ts b/admin/app/services/zim_extraction_service.ts index c48b72d..196add9 100644 --- a/admin/app/services/zim_extraction_service.ts +++ b/admin/app/services/zim_extraction_service.ts @@ -40,7 +40,10 @@ export class ZIMExtractionService { * @param filePath - Path to the ZIM file * @param opts - Options including maxArticles, strategy, onProgress, startOffset, and batchSize */ - async extractZIMContent(filePath: string, opts: ExtractZIMContentOptions = {}): Promise { + async extractZIMContent( + filePath: string, + opts: ExtractZIMContentOptions = {} + ): Promise<{ chunks: ZIMContentChunk[]; totalArticles: number }> { try { logger.info(`[ZIMExtractionService]: Processing ZIM file at path: ${filePath}`) @@ -161,7 +164,7 @@ export class ZIMExtractionService { textPreview: c.text.substring(0, 100) }))) logger.debug("Total structured sections extracted:", toReturn.length) - return toReturn + return { chunks: toReturn, totalArticles: archive.articleCount } } catch (error) { logger.error('Error processing ZIM file:', error) throw error diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 1cc9e97..f092ca2 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -7,6 +7,7 @@ import axios from 'axios' import * as cheerio from 'cheerio' import { XMLParser } from 'fast-xml-parser' import { isRawListRemoteZimFilesResponse, isRawRemoteZimFileEntry } from '../../util/zim.js' +import { findReplacedWikipediaFiles } from '../utils/zim_filename.js' import logger from '@adonisjs/core/services/logger' import { DockerService } from './docker_service.js' import { inject } from '@adonisjs/core' @@ -299,7 +300,7 @@ export class ZimService { await this.onWikipediaDownloadComplete(url, true) } } - + // Update the kiwix library XML after all downloaded ZIM files are in place. // This covers all ZIM types including Wikipedia. Rebuilding once from disk // avoids repeated XML parse/write cycles and reduces the chance of write races @@ -314,7 +315,7 @@ export class ZimService { if (restart) { // Check if there are any remaining ZIM download jobs before restarting const { QueueService } = await import('./queue_service.js') - const queueService = new QueueService() + const queueService = QueueService.getInstance() const queue = queueService.getQueue('downloads') // Get all active and waiting jobs @@ -627,18 +628,21 @@ export class ZimService { logger.info(`[ZimService] Wikipedia download completed successfully: ${filename}`) - // Delete old Wikipedia files (keep only the newly installed one) + // Delete prior versions of THIS specific Wikipedia variant only. + // Earlier logic deleted anything starting with `wikipedia_en_`, which silently + // wiped distinct corpora the user had installed independently (issue #884). const existingFiles = await this.list() - const wikipediaFiles = existingFiles.files.filter((f) => - f.name.startsWith('wikipedia_en_') && f.name !== filename + const wikipediaFiles = findReplacedWikipediaFiles( + filename, + existingFiles.files.map((f) => f.name) ) for (const oldFile of wikipediaFiles) { try { - await this.delete(oldFile.name) - logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile.name}`) + await this.delete(oldFile) + logger.info(`[ZimService] Deleted old Wikipedia file: ${oldFile}`) } catch (error) { - logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile.name}`, error) + logger.warn(`[ZimService] Could not delete old Wikipedia file: ${oldFile}`, error) } } } else { diff --git a/admin/app/utils/downloads.ts b/admin/app/utils/downloads.ts index 5d91d61..23988da 100644 --- a/admin/app/utils/downloads.ts +++ b/admin/app/utils/downloads.ts @@ -47,7 +47,14 @@ export async function doResumableDownload({ timeout, }) - const contentType = headResponse.headers['content-type'] || '' + // Some upstream hosts (notably download.kiwix.org for .zim files) don't set a + // Content-Type header at all. Per RFC 7231 §3.1.1.5, "if no Content-Type is + // provided" the recipient may treat it as application/octet-stream — which is + // already in every binary-content allowlist we use (ZIM, PMTILES, base assets). + // Without this default, the validator below throws `MIME type is not allowed` + // and breaks all downloads from kiwix's primary host (#848). + const contentType = + headResponse.headers['content-type'] || 'application/octet-stream' const totalBytes = parseInt(headResponse.headers['content-length'] || '0') const supportsRangeRequests = headResponse.headers['accept-ranges'] === 'bytes' diff --git a/admin/app/utils/kb_ingest_decision.ts b/admin/app/utils/kb_ingest_decision.ts new file mode 100644 index 0000000..72794d5 --- /dev/null +++ b/admin/app/utils/kb_ingest_decision.ts @@ -0,0 +1,70 @@ +import type { KbIngestStateValue } from '../../types/kb_ingest_state.js' + +/** + * Decision returned by `decideScanAction` describing what scanAndSyncStorage + * should do for one file given its current state row (if any), whether Qdrant + * already has chunks for it, and the global ingest policy. + * + * - `skip` — file is in a settled state (already indexed, deliberately not + * indexed, or in a manual-recovery state); no auto-dispatch. + * - `dispatch` — file needs to be (re-)embedded; an EmbedFileJob should be + * dispatched. `createStateRow` indicates whether a new state row needs to + * be created before dispatch (i.e. first time the scanner has seen it). + * - `backfill_indexed` — Qdrant has chunks but no state row exists yet + * (pre-RFC install, or new admin instance pointed at an existing Qdrant + * volume). Create a row in `indexed` state without re-embedding. + * - `create_pending` — Manual mode: record that we've seen the file but + * don't dispatch. Frontend surfaces a per-card "Index" affordance. + */ +export type ScanAction = + | { kind: 'skip' } + | { kind: 'dispatch'; createStateRow: boolean } + | { kind: 'backfill_indexed' } + | { kind: 'create_pending' } + +export interface KbIngestStateRow { + state: KbIngestStateValue +} + +/** + * Global auto-index policy stored at KV `rag.defaultIngestPolicy`. Unset is + * treated as `Always` so existing installs keep their current behavior until + * the user opts into Manual mode through the KB panel. + */ +export type IngestPolicy = 'Always' | 'Manual' + +/** + * Decide what scanAndSyncStorage should do for a single embeddable file. + * + * Replaces the earlier `!sourcesInQdrant.has(filePath)` binary check, which + * couldn't tell a fully-indexed file from a stalled mid-batch ingestion, and + * couldn't honor a user's "browse only" choice. The state row is now the + * authoritative answer; Qdrant chunk presence is corroborating evidence. + */ +export function decideScanAction( + stateRow: KbIngestStateRow | null, + hasChunksInQdrant: boolean, + policy: IngestPolicy = 'Always' +): ScanAction { + if (!stateRow) { + if (hasChunksInQdrant) return { kind: 'backfill_indexed' } + return policy === 'Always' + ? { kind: 'dispatch', createStateRow: true } + : { kind: 'create_pending' } + } + + switch (stateRow.state) { + case 'indexed': + return hasChunksInQdrant ? { kind: 'skip' } : { kind: 'dispatch', createStateRow: false } + case 'pending_decision': + // Manual mode: file is waiting for the user to opt in via per-card Index. + // Always mode: treat as "user-equivalent of auto-index" and dispatch. + return policy === 'Always' + ? { kind: 'dispatch', createStateRow: false } + : { kind: 'skip' } + case 'browse_only': + case 'failed': + case 'stalled': + return { kind: 'skip' } + } +} diff --git a/admin/app/utils/kb_job_health.ts b/admin/app/utils/kb_job_health.ts new file mode 100644 index 0000000..d5e154f --- /dev/null +++ b/admin/app/utils/kb_job_health.ts @@ -0,0 +1,50 @@ +/** + * Visual status assigned to an in-flight (or stuck) embedding job, used to + * pick the colored status pill in the KB Processing Queue. See RFC #883 §5. + * + * - `waiting` — queued, no batch has started yet + * - `healthy` — last batch < 2 minutes ago + * - `slow` — last batch 2-5 minutes ago (CPU-paced multi-batch ingestion + * falls into this band; not necessarily a problem) + * - `stalled` — last batch > 5 minutes ago (likely a real problem) + * - `failed` — job recorded a failed status + */ +export type JobHealthStatus = 'waiting' | 'healthy' | 'slow' | 'stalled' | 'failed' + +export interface JobHealthInput { + /** BullMQ job.data.status — set by EmbedFileJob.handle on transitions. */ + status: string + /** 0-100. 0 means no work observed yet on this job-row. */ + progress: number + /** ms epoch of the last completed batch. Multi-batch ZIMs update this on + * every continuation; single-batch jobs leave it unset until completion. */ + lastBatchAt?: number + /** ms epoch of the first batch start. Used as a fallback "last activity" + * signal for jobs that haven't yet completed their first batch. */ + startedAt?: number + /** Current ms epoch. Injected for testability. */ + now: number +} + +const SLOW_THRESHOLD_MS = 2 * 60 * 1000 +const STALLED_THRESHOLD_MS = 5 * 60 * 1000 + +export function computeJobHealth(input: JobHealthInput): JobHealthStatus { + if (input.status === 'failed') return 'failed' + + // No progress recorded and no activity timestamps — job is still queued. + if ( + input.progress === 0 && + input.lastBatchAt === undefined && + input.startedAt === undefined + ) { + return 'waiting' + } + + const lastActivity = input.lastBatchAt ?? input.startedAt ?? input.now + const stalenessMs = input.now - lastActivity + + if (stalenessMs > STALLED_THRESHOLD_MS) return 'stalled' + if (stalenessMs > SLOW_THRESHOLD_MS) return 'slow' + return 'healthy' +} diff --git a/admin/app/utils/kb_ratio_lookup.ts b/admin/app/utils/kb_ratio_lookup.ts new file mode 100644 index 0000000..1abd5c6 --- /dev/null +++ b/admin/app/utils/kb_ratio_lookup.ts @@ -0,0 +1,97 @@ +export interface RatioRow { + pattern: string + chunks_per_mb: number +} + +/** + * Bytes of on-disk storage one embedded chunk consumes inside Qdrant. + * + * Rough composition for our pipeline: + * - vector: 768 dims × float32 = 3,072 B + * - chunk text payload: ~3,000 B (target 1,500 tokens × 2 chars/token) + * - source/metadata payload + Qdrant indexes: ~2,000 B + * + * Used for surfacing pre-ingest disk-cost estimates; the actual figure + * varies with collection params and will be replaced by self-calibration + * (RFC #883 Phase 4) once we have real measurements. + */ +export const BYTES_PER_CHUNK_ON_DISK = 8_000 + +export interface BatchEstimateInput { + filename: string + sizeBytes: number +} + +export interface BatchEstimate { + totalChunks: number + totalBytes: number + hasUnknown: boolean +} + +/** + * Aggregate an embedding-disk-cost estimate across a batch of files (curated + * tier add, multi-upload, sync preview, etc). `hasUnknown` is true when at + * least one file did not match any registry row — the totals only include + * matched files, so callers should annotate "estimate excludes unknown files" + * when surfacing the figure. + */ +export function estimateBatch( + files: BatchEstimateInput[], + rows: RatioRow[] +): BatchEstimate { + let totalChunks = 0 + let hasUnknown = false + for (const f of files) { + const chunks = estimateChunkCount(f.filename, f.sizeBytes, rows) + if (chunks === null) { + hasUnknown = true + } else { + totalChunks += chunks + } + } + return { + totalChunks, + totalBytes: totalChunks * BYTES_PER_CHUNK_ON_DISK, + hasUnknown, + } +} + +/** + * Pick the chunks_per_mb estimate for a filename by longest-prefix match. + * + * Patterns are filename prefixes (`devdocs_`, `wikipedia_en_simple_`, ...). + * The longest matching prefix wins, so a specific entry (`wikipedia_en_simple_`) + * overrides the broader fallback (`wikipedia_en_`). An empty-string pattern in + * the registry serves as a catch-all that matches every input. + * + * Returns `null` if no row matches and no empty-string fallback is present — + * caller decides whether to surface "unknown" or use its own default. + */ +export function findChunksPerMb(filename: string, rows: RatioRow[]): number | null { + let best: RatioRow | null = null + for (const row of rows) { + if (!filename.startsWith(row.pattern)) continue + if (best === null || row.pattern.length > best.pattern.length) { + best = row + } + } + return best === null ? null : best.chunks_per_mb +} + +/** + * Estimate the number of embedding chunks a ZIM-style file will produce given + * its size on disk in bytes. Returns `null` when the registry has nothing to + * match against. Caller is responsible for converting the estimate into either + * a disk-footprint estimate (chunks × bytes-per-chunk in Qdrant) or a time + * estimate (chunks ÷ chunks-per-minute-on-this-hardware). + */ +export function estimateChunkCount( + filename: string, + fileSizeBytes: number, + rows: RatioRow[] +): number | null { + const ratio = findChunksPerMb(filename, rows) + if (ratio === null) return null + const megabytes = fileSizeBytes / (1024 * 1024) + return Math.round(ratio * megabytes) +} diff --git a/admin/app/utils/kb_warning_decision.ts b/admin/app/utils/kb_warning_decision.ts new file mode 100644 index 0000000..4551566 --- /dev/null +++ b/admin/app/utils/kb_warning_decision.ts @@ -0,0 +1,70 @@ +/** + * Conditional warnings surfaced on Stored Files rows in the KB panel. + * See RFC #883 §6 — these warnings appear ONLY when their triggering condition + * is met, never on healthy files, to keep the panel silent in the common case. + * + * - `zero_chunks` — a non-trivial file produced 0 embedding chunks. Common + * cause: video-only or image-only ZIMs that the pipeline + * completes "successfully" with no extractable text. + * AI Assistant cannot reference this content. + * - `partial_stall` — the file has embedded chunks but well below the count + * expected from the ratio registry. Likely a mid-batch + * stall (which the binary "any chunks ⇒ embedded" check + * used to mask). Surfaces a Retry affordance. + */ +import type { FileWarning } from '../../types/rag.js' + +export type { FileWarning } + +/** Files smaller than this are too small to flag as suspicious zero-chunk + * cases — a 5 KB upload that produces 0 chunks is much more likely to be a + * legitimate edge case (placeholder file) than the gigabyte-scale video ZIM + * problem this warning targets. */ +export const ZERO_CHUNKS_MIN_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB + +/** Fraction of expected chunks below which we consider a file partially + * stalled. 0.5 (50%) matches the threshold described in RFC #883 §6 Warning B. */ +export const PARTIAL_STALL_RATIO_THRESHOLD = 0.5 + +export interface WarningInputs { + /** Source file size on disk in bytes. */ + fileSizeBytes: number + /** Distinct chunks present in Qdrant for this source. */ + chunksInQdrant: number + /** Best estimate of chunks the file should produce, from the ratio + * registry. `null` when no registry pattern matches and no fallback is + * configured — Warning B is suppressed in that case (we'd rather be silent + * than wrong). */ + expectedChunks: number | null +} + +export function decideWarnings(inputs: WarningInputs): FileWarning[] { + const warnings: FileWarning[] = [] + + // Warning A: file is large but produced nothing. Almost always a video-only + // or image-only ZIM; AI Assistant literally cannot reference this content. + if ( + inputs.chunksInQdrant === 0 && + inputs.fileSizeBytes > ZERO_CHUNKS_MIN_SIZE_BYTES + ) { + warnings.push({ kind: 'zero_chunks', fileSizeBytes: inputs.fileSizeBytes }) + } + + // Warning B: chunks present but far below expectation. Suppresses when we + // have no expectation (registry miss) since the comparison would be + // meaningless and we'd rather under-warn than mislead. + if ( + inputs.expectedChunks !== null && + inputs.expectedChunks > 0 && + inputs.chunksInQdrant > 0 && + inputs.chunksInQdrant < inputs.expectedChunks * PARTIAL_STALL_RATIO_THRESHOLD + ) { + warnings.push({ + kind: 'partial_stall', + chunksEmbedded: inputs.chunksInQdrant, + chunksExpected: inputs.expectedChunks, + }) + } + + return warnings +} diff --git a/admin/app/utils/zim_filename.ts b/admin/app/utils/zim_filename.ts new file mode 100644 index 0000000..a2fb5b2 --- /dev/null +++ b/admin/app/utils/zim_filename.ts @@ -0,0 +1,26 @@ +/** + * Strip the trailing `_YYYY-MM(-DD).zim` date suffix from a Kiwix-style ZIM + * filename so different release dates of the same variant share a stem + * (e.g., `wikipedia_en_all_nopic`) while distinct corpora keep distinct stems + * (`wikipedia_en_simple_all_nopic`, `wikipedia_en_medicine_nopic`, etc.). + */ +export function zimFilenameStem(name: string): string { + return name.replace(/_\d{4}-\d{2}(?:-\d{2})?\.zim$/i, '') +} + +/** + * Of the existing files, return only those that are prior-version replacements + * of `currentFilename` — same Wikipedia variant stem, different release. Used + * by the post-download cleanup to avoid deleting unrelated Wikipedia corpora + * the user has installed independently (issue #884). + */ +export function findReplacedWikipediaFiles( + currentFilename: string, + existingNames: string[] +): string[] { + const currentStem = zimFilenameStem(currentFilename) + return existingNames.filter( + (n) => + n.startsWith('wikipedia_en_') && n !== currentFilename && zimFilenameStem(n) === currentStem + ) +} diff --git a/admin/app/validators/common.ts b/admin/app/validators/common.ts index ba9f107..f9f4e63 100644 --- a/admin/app/validators/common.ts +++ b/admin/app/validators/common.ts @@ -1,4 +1,5 @@ import vine from '@vinejs/vine' +import ipaddr from 'ipaddr.js' /** * Checks whether a URL points to a loopback or link-local address. @@ -15,15 +16,18 @@ export function assertNotPrivateUrl(urlString: string): void { const parsed = new URL(urlString) const hostname = parsed.hostname.toLowerCase() + // `URL.hostname` strips the surrounding brackets from IPv6 literals + // (e.g. `http://[::1]/` → hostname `::1`), so IPv6 patterns must match + // the unbracketed form. const blockedPatterns = [ /^localhost$/, /^127\.\d+\.\d+\.\d+$/, /^0\.0\.0\.0$/, /^169\.254\.\d+\.\d+$/, // Link-local / cloud metadata - /^\[::1\]$/, - /^\[?fe80:/i, // IPv6 link-local - /^\[::ffff:/i, // IPv4-mapped IPv6 (e.g. [::ffff:7f00:1] = 127.0.0.1) - /^\[::\]$/, // IPv6 all-zeros (equivalent to 0.0.0.0) + /^::1$/, // IPv6 loopback + /^fe80:/i, // IPv6 link-local + /^::ffff:/i, // IPv4-mapped IPv6 (e.g. ::ffff:7f00:1 = 127.0.0.1) + /^::$/, // IPv6 all-zeros (equivalent to 0.0.0.0) ] if (blockedPatterns.some((re) => re.test(hostname))) { @@ -31,6 +35,63 @@ export function assertNotPrivateUrl(urlString: string): void { } } +/** + * Narrower SSRF guard for "remote service" URLs the user points NOMAD at + * (e.g. an OpenAI-compatible endpoint like LM Studio, llama.cpp, vLLM, or a + * sibling Ollama container). Unlike `assertNotPrivateUrl`, this intentionally + * ALLOWS loopback, link-local-ish, and RFC1918 hosts because the legitimate + * target is frequently on the same host or LAN (host.docker.internal, + * the docker bridge gateway, or a LAN IP). + * + * It blocks only: + * - the cloud instance-metadata IP (169.254.169.254), to avoid leaking + * IAM creds on a misconfigured cloud VM + * - non-HTTP schemes (file:, gopher:, etc.) + */ +// Canonical cloud instance-metadata addresses. AWS, GCP, Azure, DigitalOcean, +// Oracle Cloud, and Alibaba all expose IMDS at 169.254.169.254 over IPv4; +// AWS additionally exposes it at fd00:ec2::254 over IPv6. +// Compared after `ipaddr.toNormalizedString()`, which expands IPv6 to its +// fully-zero-padded form (e.g. `fd00:ec2::254` → `fd00:ec2:0:0:0:0:0:254`). +const BLOCKED_METADATA_IPV4 = new Set(['169.254.169.254']) +const BLOCKED_METADATA_IPV6 = new Set([ + ipaddr.parse('fd00:ec2::254').toNormalizedString(), +]) + +export function assertNotCloudMetadataUrl(urlString: string): void { + const parsed = new URL(urlString) + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error(`URL must use http or https scheme: ${parsed.protocol}`) + } + + // Node's WHATWG URL parser keeps the brackets on IPv6 literals + // (`http://[::1]/` → hostname `[::1]`), so strip them before parsing. + const hostname = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, '') + + // If the hostname isn't an IP literal it's a DNS name; allow it. (DNS + // rebinding is out of scope here — would require resolving and re-checking + // at fetch time.) + if (!ipaddr.isValid(hostname)) return + + let addr = ipaddr.parse(hostname) + + // Unwrap IPv4-mapped IPv6 (e.g. ::ffff:169.254.169.254, ::ffff:a9fe:a9fe, + // and the fully-expanded 0:0:0:0:0:ffff:a9fe:a9fe) so the IPv4 check below + // sees the embedded address. + if (addr.kind() === 'ipv6' && (addr as ipaddr.IPv6).isIPv4MappedAddress()) { + addr = (addr as ipaddr.IPv6).toIPv4Address() + } + + const canonical = addr.toNormalizedString() + + const blocked = + addr.kind() === 'ipv4' ? BLOCKED_METADATA_IPV4 : BLOCKED_METADATA_IPV6 + if (blocked.has(canonical)) { + throw new Error(`URL must not point to the cloud instance metadata endpoint: ${canonical}`) + } +} + export const remoteDownloadValidator = vine.compile( vine.object({ url: vine diff --git a/admin/app/validators/ollama.ts b/admin/app/validators/ollama.ts index d83ed3b..1784ada 100644 --- a/admin/app/validators/ollama.ts +++ b/admin/app/validators/ollama.ts @@ -14,6 +14,12 @@ export const chatSchema = vine.compile( }) ) +export const unloadChatModelsSchema = vine.compile( + vine.object({ + targetModel: vine.string().trim().minLength(1).nullable().optional(), + }) +) + export const getAvailableModelsSchema = vine.compile( vine.object({ sort: vine.enum(['pulls', 'name'] as const).optional(), diff --git a/admin/app/validators/rag.ts b/admin/app/validators/rag.ts index a9124b4..5bef836 100644 --- a/admin/app/validators/rag.ts +++ b/admin/app/validators/rag.ts @@ -11,3 +11,24 @@ export const deleteFileSchema = vine.compile( source: vine.string(), }) ) + +export const embedFileSchema = vine.compile( + vine.object({ + source: vine.string().minLength(1), + force: vine.boolean().optional(), + }) +) + +export const estimateBatchSchema = vine.compile( + vine.object({ + files: vine + .array( + vine.object({ + filename: vine.string().minLength(1).maxLength(255), + sizeBytes: vine.number().min(0), + }) + ) + .minLength(1) + .maxLength(500), + }) +) diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index c49416c..6caeb72 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -1,3 +1,3 @@ import { KVStoreKey } from "../types/kv_store.js"; -export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention']; \ No newline at end of file +export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy']; \ No newline at end of file diff --git a/admin/constants/ollama.ts b/admin/constants/ollama.ts index 5581832..23df8f0 100644 --- a/admin/constants/ollama.ts +++ b/admin/constants/ollama.ts @@ -64,6 +64,8 @@ export const FALLBACK_RECOMMENDED_OLLAMA_MODELS: NomadOllamaModel[] = [ export const DEFAULT_QUERY_REWRITE_MODEL = 'qwen2.5:3b' // default to qwen2.5 for query rewriting with good balance of text task performance and resource usage +export const EMBEDDING_MODEL_NAME = 'nomic-embed-text:v1.5' + /** * Adaptive RAG context limits based on model size. * Smaller models get overwhelmed with too much context, so we cap it. diff --git a/admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts b/admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts new file mode 100644 index 0000000..18e8ab4 --- /dev/null +++ b/admin/database/migrations/1776000000001_create_kb_ingest_state_table.ts @@ -0,0 +1,26 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'kb_ingest_state' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id').primary() + // utf8mb4 caps an indexed varchar at 768 chars (3072 byte InnoDB key limit); + // 512 leaves headroom and is plenty for any NOMAD-managed file path. + table.string('file_path', 512).notNullable().unique() + table + .enum('state', ['pending_decision', 'indexed', 'browse_only', 'failed', 'stalled']) + .notNullable() + .defaultTo('pending_decision') + table.integer('chunks_embedded').notNullable().defaultTo(0) + table.text('last_error').nullable() + table.timestamp('created_at').notNullable() + table.timestamp('updated_at').notNullable() + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts b/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts new file mode 100644 index 0000000..604e6e8 --- /dev/null +++ b/admin/database/migrations/1776100000001_create_kb_ratio_registry_table.ts @@ -0,0 +1,64 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' +import { DateTime } from 'luxon' + +const SEED_ROWS: Array<{ pattern: string; chunks_per_mb: number; notes: string }> = [ + // Dense technical reference — every paragraph carries content + { pattern: 'devdocs_', chunks_per_mb: 1100, notes: 'Heuristic seed: dense API references' }, + // Encyclopedia prose — Simple English & general Wikipedia variants + { + pattern: 'wikipedia_en_simple_', + chunks_per_mb: 270, + notes: 'Heuristic seed: Simple English Wikipedia', + }, + { + pattern: 'wikipedia_en_', + chunks_per_mb: 270, + notes: 'Heuristic seed: general Wikipedia variants', + }, + // Sparse text, image-heavy + { pattern: 'ifixit_', chunks_per_mb: 50, notes: 'Heuristic seed: image-heavy repair guides' }, + // Q&A pages — moderate density, mostly short answers + { + pattern: 'cooking.stackexchange.com_', + chunks_per_mb: 200, + notes: 'Heuristic seed: Stack Exchange Q&A', + }, + // Video-only ZIMs produce zero text chunks. Listing these explicitly keeps + // the cost estimator from spinning up "indexing in progress" UI for content + // that has no embeddable text whatsoever. + { pattern: 'lrnselfreliance_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' }, + { pattern: 'ted_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' }, + { pattern: 'freedom-of-religion_', chunks_per_mb: 0, notes: 'Heuristic seed: video-only ZIM' }, + // Empty-pattern fallback — every filename startsWith('') is true. The lookup + // picks the longest matching pattern, so this only fires for ZIMs that match + // none of the above (medium prose density). + { pattern: '', chunks_per_mb: 100, notes: 'Heuristic fallback' }, +] + +export default class extends BaseSchema { + protected tableName = 'kb_ratio_registry' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.increments('id').primary() + table.string('pattern', 255).notNullable().unique() + table.integer('chunks_per_mb').unsigned().notNullable() + // 0 = heuristic seed, >0 = number of observed ZIMs that have updated this entry. + // Phase 4 self-calibration increments this on each successful ingestion. + table.integer('sample_count').notNullable().defaultTo(0) + table.text('notes').nullable() + table.timestamp('created_at').notNullable() + table.timestamp('updated_at').notNullable() + }) + + const now = DateTime.utc().toSQL({ includeOffset: false }) as string + const rows = SEED_ROWS.map((row) => ({ ...row, created_at: now, updated_at: now })) + this.defer(async (db) => { + await db.table(this.tableName).multiInsert(rows) + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/admin/docs/faq.md b/admin/docs/faq.md index aa8aac1..e41c611 100644 --- a/admin/docs/faq.md +++ b/admin/docs/faq.md @@ -114,6 +114,18 @@ The Maps feature requires downloaded map data. If you see a blank area: 3. Wait for downloads to complete 4. Return to Maps and refresh +### ERROR: Failed to load the XML library file '/data/kiwix-library.xml' + +This usually means the Information Library service started before its Kiwix library index was fully initialized. + +Try this recovery flow: +1. Go to **[Apps](/settings/apps)** +2. Stop **Information Library (Kiwix)** +3. Wait 10-15 seconds, then start it again +4. If the error persists, run **Force Reinstall** for Information Library from the same page + +After restart/reinstall completes, refresh the Information Library page. + ### AI responses are slow Local AI requires significant computing power. To improve speed: diff --git a/admin/docs/release-notes.md b/admin/docs/release-notes.md index cc7b654..2e0bd34 100644 --- a/admin/docs/release-notes.md +++ b/admin/docs/release-notes.md @@ -1,32 +1,92 @@ # Release Notes -## Unreleased +## Version 1.31.1 - April 21, 2026 ### Features -- **AI Assistant**: Added improved support for AMD GPU acceleration for Ollama via ROCm + HSA override. Thanks @chriscrosstalk for the contribution! -- **Content Explorer**: Added support for custom ZIM library sources and pre-seeded ZIM library mirrors in addition to the default Kiwix library. Thanks @chriscrosstalk for the contribution! -- **Content Manager**: Content update sizes and downloads are now properly displayed in Active Downloads with progress bars and friendly names. Thanks @chriscrosstalk for the contribution! -- **Maps**: Map regions can now be extracted and downloaded locally from PMTiles to avoid the need for a full global map download for users who only want specific regions. Thanks @bgauger for the contribution! +- feat(content): custom ZIM library sources with pre-seeded mirrors (#593). Thanks @chriscrosstalk! +- feat(content-manager): add sortable file size column (#698). Thanks @chriscrosstalk! +- feat(ai-chat): allow cancelling in-progress model downloads (#701). Thanks @chriscrosstalk! +- feat(content-updates): show size, surface downloads in Active Downloads (#773). Thanks @chriscrosstalk! +- feat(maps): regional map downloads via go-pmtiles extract (#780). Thanks @bgauger! +- feat(maps): show map coordinates on mouse move (#786). Thanks @kennethbrewer3! +- feat(AI): re-enable AMD GPU acceleration for Ollama via ROCm + HSA override (#804). Thanks @chriscrosstalk! +- feat(GPU): auto-remediate nomad_ollama passthrough loss on admin boot (#878). Thanks @chriscrosstalk! +- feat(KB): per-file ingest state machine (Phase 1 of RFC #883) (#888). Thanks @chriscrosstalk! +- feat(KB): ratio registry for disk + time estimates (Phase 1B of RFC #883) (#891). Thanks @chriscrosstalk! +- feat(KB): group admin docs into single row in Stored Files (§9) (#892). Thanks @chriscrosstalk! +- feat(KB): status pill + last-activity on Processing Queue (§5/§10) (#893). Thanks @chriscrosstalk! +- feat(KB): Always/Manual ingest policy toggle (§1/§4) (#894). Thanks @chriscrosstalk! +- feat(KB): conditional warnings A + B on Stored Files (§6) (#895). Thanks @chriscrosstalk! +- feat(KB): surface embedding-disk estimate in curated tier-change modal (§1) (#897). Thanks @chriscrosstalk! +- feat(KB): first-chat JIT prompt for ingest policy (Phase 3 task 12) (#899). Thanks @chriscrosstalk! +- feat(KB): wizard AI policy step (Phase 3 task 13) (#900). Thanks @chriscrosstalk! +- feat(KB): guardrail modal at 50GB / 10%-free thresholds (§7) (#901). Thanks @chriscrosstalk! +- feat(easy-setup): split AI into its own conditional step (#908). Thanks @chriscrosstalk! +- feat(KB): per-file ingest action + state indicator on Stored Files (§5) (#909). Thanks @chriscrosstalk! +- feat(chat): confirm-on-switch + one-chat-model-at-a-time enforcement (#916). Thanks @chriscrosstalk! ### Bug Fixes -- **API**: Compression is now skipped for Server-Sent Events (SSE) responses to prevent issues with streaming endpoints. Thanks @chriscrosstalk for the fix! -- **Maps**: Fixed logic issues with the global map banner display. Thanks @Gujiassh for the fix! -- **Maps**: The selected map file is now properly deleted after confirming the action in the UI. Thanks @cuyua9 for the fix! -- **System**: Fixed an issue where the a pending update could still be indicated in the UI even after the system was updated successfully. Thanks @jakeaturner for the fix! +- fix(downloads): stage downloads to .tmp to prevent Kiwix loading partial files (#448). Thanks @artbird309! +- fix(security): close remaining security audit items 3 & 4 (CWE-918, CWE-209) (#552). Thanks @LuisMIguelFurlanettoSousa! +- fix(ai-chat): add null check to model name (#645). Thanks @hestela! +- fix(ai-chat): qwen2.5 loading on every chat message (#649). Thanks @hestela! +- fix(disk-collector): fix storage reporting for NFS mounts (#686). Thanks @bgauger! +- fix(rag): add start button in kb modal and ensure restart policy exists (#700). Thanks @hestela! +- fix(admin): only hide global map banner after download (#702). Thanks @Gujiassh! +- fix(maps): wire delete confirmation to API (#732). Thanks @cuyua9! +- fix: prevent ZIM corrupt file crash and deduplicate Ollama download logs (#741). Thanks @jakeaturner! +- fix(ai): stop local nomad_ollama when remote Ollama is configured (#744). Thanks @chriscrosstalk! +- fix(rag): repair ZIM embedding pipeline (sync filter, batch gate, DOM walk) (#745). Thanks @chriscrosstalk! +- fix(zim): accumulate across Kiwix pages to prevent empty Content Explorer (#746). Thanks @chriscrosstalk! +- fix(qdrant): disable anonymous telemetry by default (#747). Thanks @chriscrosstalk! +- fix(disk-display): gate NAS Storage label on network filesystem type (#749). Thanks @bgauger! +- fix(docker): write /app/version.json from VERSION build-arg (#754). Thanks @chriscrosstalk! +- fix(rag): pass num_ctx and truncate to Ollama embed call (#763). Thanks @chriscrosstalk! +- fix(api): accept notes, marker_type, and position on markers endpoints (#770). Thanks @jrsphoto! +- fix(install): warn loudly on non-x86_64 architectures before pulling images (#797). Thanks @chriscrosstalk! +- fix(stream): skip compression for Server-Sent Events (#798). Thanks @chriscrosstalk! +- fix(maps): Country Picker UX polish + auto-refresh stored files (#817). Thanks @chriscrosstalk! +- fix(System): self-heal stale updateAvailable flag after sidecar-driven update (#825). Thanks @jakeaturner! +- fix(settings/update): four UI/UX fixes for the System Update page (#827). Thanks @chriscrosstalk! +- fix(Maps): send filename instead of full path to delete endpoint (#829). Thanks @bgauger! +- fix(Maps): render notes in marker popup when populated (#830). Thanks @chriscrosstalk! +- fix(AI): vendor-aware AMD HSA override + benchmark discrete-GPU detection (#832). Thanks @chriscrosstalk! +- fix(System): correct NVIDIA VRAM in Graphics card (#850). Thanks @bgauger! +- fix(Downloads): treat missing Content-Type as octet-stream (#859). Thanks @bgauger! +- fix(AI): preserve semver tag in DB on AMD Ollama updates (#868). Thanks @chriscrosstalk! +- fix(AI): rewrite RAG query on first chat follow-up (#869). Thanks @chriscrosstalk! +- fix(RAG): unbreak multi-batch ZIM ingestion (jobId dedupe) (#872). Thanks @chriscrosstalk! +- fix(RAG): pace continuation batches when embedding is CPU-only (#873). Thanks @chriscrosstalk! +- fix(queue): singleton QueueService to stop ioredis connection leak (#877). Thanks @chriscrosstalk! +- fix(System): correct AMD VRAM in Graphics card + harden log probe (#879). Thanks @chriscrosstalk! +- fix(RAG): report ZIM ingestion progress in overall-file frame (#880). Thanks @chriscrosstalk! +- fix(KB): add re-embed and reset & rebuild options to fix broken embeddings (#886). Thanks @jakeaturner! +- fix(ZIM): preserve co-existing Wikipedia corpora on cleanup (#887). Thanks @chriscrosstalk! +- fix(RAG): anchor continuation-batch initial progress to overall-file frame (#889). Thanks @chriscrosstalk! +- fix(AI): pre-cap embed input + log fallback reason (#890). Thanks @chriscrosstalk! +- fix(KB): remove redundant Refresh button from Processing Queue (#896). Thanks @chriscrosstalk! +- fix(KB): union Stored Files list with state-machine file paths (#898). Thanks @chriscrosstalk! +- fix(KB): blank-screen on panel open + tooltips on bulk-action buttons (#907). Thanks @chriscrosstalk! +- fix(KB): TierSelectionModal hook order + register IconLibrary (#917). Thanks @chriscrosstalk! +- fix(content): show selected tier on cards while downloads are in flight (#918). Thanks @chriscrosstalk! +- fix(KB): respect Manual ingest policy on post-download dispatch (#919). Thanks @chriscrosstalk! +- fix(AI): improve remote Ollama url validation to prevent SSRF vuln (#920). Thanks @jakeaturner! +- fix(models): correct inverted belongsTo keys on ChatMessage.session (#921). Thanks @jakeaturner! ### Improvements -- **Build**: The Command Center image now uses the VERSION build arg to write `app/version.json` with the current version for improved version tracking and debugging, even in RC environments. Thanks @chriscrosstalk for the contribution! -- **Content Manager**: Added a sortable file size column to the ZIM files table in the Content Manager for easier management of storage space. Thanks @chriscrosstalk for the contribution! -- **Dependencies**: All package.json dependencies have been pinned to specific versions to ensure stability and reduce the risk of unexpected breaking changes/supply-chain compromises from upstream packages. Thanks @jakeaturner for the contribution! -- **Dependencies**: Updated various dependencies to close security vulnerabilities and improve stability -- **Docs**: Update CONTIRBUTING.md to require an issue to be opened before submitting a PR for non-trivial changes to ensure proper discussion and review of proposed changes. Thanks @chriscrosstalk for the contribution! -- **Docs**: Added the map markers endpoints to the API reference documentation. Thanks @kennethbrewer3 for the contribution! -- **Docs**: Added a link to the new WSL2 install guide in the README and FAQ. Thanks @chriscrosstalk for the contribution! -- **Install**: The install script now warns loudly if the user is attempting to install on a non-x86_64/amd64 platform to prevent unsupported installations and potential issues. Thanks @chriscrosstalk for the contribution! -- **Maps**: The maps API endpoints now properly accept and validate notes, marker_type, and position data for map markers and persist them in the database for retrieval in the UI. Thanks @jrsphoto for the contribution! -- **Maps**: The current coordinates of the mouse pointer can now be displayed in the map viewer for easier navigation and exploration. Thanks @kennethbrewer3 for the contribution! -- **RAG**: NOMAD now properly passed `num_ctx` and truncation to the Ollama embedding endpoint to ensure that the context window of the model is best utilized for embeddings. Thanks @chriscrosstalk for the contribution! -- **RAG**: Added a manual start button for Qdrant and a self-healing mechanism for Qdrant's restart-policy to ensure that the vector database is running properly for embedding and retrieval tasks. Thanks @hestela for the contribution! +- docs: add Community Add-Ons page with field manuals + W3Schools packs (#753). Thanks @chriscrosstalk! +- docs: add map marker API reference (#783). Thanks @kennethbrewer3! +- docs: require linked issue for non-trivial PRs (#799). Thanks @chriscrosstalk! +- docs(map): updated notes on the map pin api (#803). Thanks @kennethbrewer3! +- docs: link to new WSL2 install guide from README and FAQ (#811). Thanks @chriscrosstalk! +- build(deps): bump picomatch in /admin (#544). Thanks @dependabot[bot]! +- build(deps): bump lodash from 4.17.23 to 4.18.1 in /admin (#643). Thanks @dependabot[bot]! +- build(deps-dev): bump vite from 6.4.1 to 6.4.2 in /admin (#677). Thanks @dependabot[bot]! +- build(deps): bump axios from 1.13.5 to 1.15.0 in /admin (#708). Thanks @dependabot[bot]! +- build(deps): bump @adonisjs/http-server from 7.8.0 to 7.8.1 in /admin (#724). Thanks @dependabot[bot]! +- build(deps): bump follow-redirects from 1.15.11 to 1.16.0 in /admin (#729). Thanks @dependabot[bot]! +- build(deps): bump protocol-buffers-schema from 3.6.0 to 3.6.1 in /admin (#736). Thanks @dependabot[bot]! +- build(deps): bump protobufjs from 7.5.4 to 7.5.5 in /admin (#737). Thanks @dependabot[bot]! ## Version 1.31.1 - April 21, 2026 diff --git a/admin/inertia/components/ActiveEmbedJobs.tsx b/admin/inertia/components/ActiveEmbedJobs.tsx index 9da78bc..bb8fbd1 100644 --- a/admin/inertia/components/ActiveEmbedJobs.tsx +++ b/admin/inertia/components/ActiveEmbedJobs.tsx @@ -1,6 +1,12 @@ +import { useEffect, useState } from 'react' import useEmbedJobs from '~/hooks/useEmbedJobs' import HorizontalBarChart from './HorizontalBarChart' import StyledSectionHeader from './StyledSectionHeader' +import { + JOB_HEALTH_DISPLAY, + computeJobHealth, + formatTimeAgo, +} from '~/lib/kb_job_health_display' interface ActiveEmbedJobsProps { withHeader?: boolean @@ -9,31 +15,70 @@ interface ActiveEmbedJobsProps { const ActiveEmbedJobs = ({ withHeader = false }: ActiveEmbedJobsProps) => { const { data: jobs } = useEmbedJobs() + // Re-render every 5s to keep per-job "last activity Xs ago" timestamps fresh. + const [tick, setTick] = useState(() => Date.now()) + useEffect(() => { + const id = setInterval(() => setTick(Date.now()), 5000) + return () => clearInterval(id) + }, []) + return ( <> {withHeader && ( )} +
{jobs && jobs.length > 0 ? ( - jobs.map((job) => ( -
- -
- )) + jobs.map((job) => { + const health = computeJobHealth({ + status: job.status, + progress: job.progress, + lastBatchAt: job.lastBatchAt, + startedAt: job.startedAt, + now: tick, + }) + const display = JOB_HEALTH_DISPLAY[health] + const lastActivityMs = job.lastBatchAt ?? job.startedAt + return ( +
+
+ + + {display.label} + + {lastActivityMs !== undefined && ( + + · last activity {formatTimeAgo(lastActivityMs, tick)} + + )} + {typeof job.chunks === 'number' && job.chunks > 0 && ( + + · {job.chunks.toLocaleString()} chunks + + )} +
+ +
+ ) + }) ) : (

No files are currently being processed

)} diff --git a/admin/inertia/components/CategoryCard.tsx b/admin/inertia/components/CategoryCard.tsx index ae60b4c..6b63b4d 100644 --- a/admin/inertia/components/CategoryCard.tsx +++ b/admin/inertia/components/CategoryCard.tsx @@ -2,7 +2,7 @@ import { formatBytes } from '~/lib/util' import DynamicIcon, { DynamicIconName } from './DynamicIcon' import type { CategoryWithStatus, SpecTier } from '../../types/collections' import classNames from 'classnames' -import { IconChevronRight, IconCircleCheck } from '@tabler/icons-react' +import { IconChevronRight, IconCircleCheck, IconLoader2 } from '@tabler/icons-react' export interface CategoryCardProps { category: CategoryWithStatus @@ -29,14 +29,34 @@ const CategoryCard: React.FC = ({ category, selectedTier, onC const minSize = getTierTotalSize(category.tiers[0], category.tiers) const maxSize = getTierTotalSize(category.tiers[category.tiers.length - 1], category.tiers) - // Determine which tier to highlight: selectedTier (wizard) > installedTierSlug (persisted) - const highlightedTierSlug = selectedTier?.slug || category.installedTierSlug + // Priority order for the prominent corner badge + lime border: + // 1. selectedTier — in-session wizard pick (highest priority, reflects + // what the user is editing right now) + // 2. downloadingTierSlug — backend-derived from in-flight downloads, so + // the card shows the user's intent immediately after Submit, before + // any single file has finished downloading + // 3. installedTierSlug — fully on disk + const downloadingTier = !selectedTier && category.downloadingTierSlug + ? category.tiers.find((t) => t.slug === category.downloadingTierSlug) + : null + const installedTier = !selectedTier && !downloadingTier && category.installedTierSlug + ? category.tiers.find((t) => t.slug === category.installedTierSlug) + : null + const badgeTier = selectedTier || downloadingTier || installedTier + const badgeStatus: 'selected' | 'downloading' | 'installed' | null = selectedTier + ? 'selected' + : downloadingTier + ? 'downloading' + : installedTier + ? 'installed' + : null + const highlightedTierSlug = badgeTier?.slug return (
onClick?.(category)} > @@ -46,10 +66,17 @@ const CategoryCard: React.FC = ({ category, selectedTier, onC

{category.name}

- {selectedTier ? ( + {badgeTier ? (
- - {selectedTier.name} + {badgeStatus === 'downloading' ? ( + + ) : ( + + )} + + {badgeTier.name} + {badgeStatus === 'downloading' && ' (downloading)'} +
) : ( diff --git a/admin/inertia/components/KbGuardrailModal.tsx b/admin/inertia/components/KbGuardrailModal.tsx new file mode 100644 index 0000000..498cc4c --- /dev/null +++ b/admin/inertia/components/KbGuardrailModal.tsx @@ -0,0 +1,109 @@ +import { Fragment } from 'react' +import { Dialog, Transition } from '@headlessui/react' +import { IconAlertTriangle, IconX } from '@tabler/icons-react' +import { formatBytes } from '~/lib/util' +import StyledButton from './StyledButton' +import type { GuardrailVerdict } from '~/lib/kb_guardrail' + +/** + * One-time confirmation modal for bulk indexing actions that trip the + * disk-usage thresholds in `lib/kb_guardrail.ts`. The caller (e.g. + * TierSelectionModal) decides whether to show the modal by evaluating the + * guardrail BEFORE submit; this component just presents the verdict and + * passes the user's choice back via `onConfirm` / `onCancel`. + */ +interface KbGuardrailModalProps { + isOpen: boolean + verdict: GuardrailVerdict + onConfirm: () => void + onCancel: () => void +} + +export default function KbGuardrailModal({ + isOpen, + verdict, + onConfirm, + onCancel, +}: KbGuardrailModalProps) { + // The primary number to surface — every triggered reason carries the same + // estimateBytes, so just grab the first one. `0` is a defensive fallback + // for the (impossible-by-construction) "open with empty verdict" case. + const estimateBytes = verdict.reasons[0]?.estimateBytes ?? 0 + const freeReason = verdict.reasons.find((r) => r.kind === 'over_free_disk') + + return ( + + + +
+ + +
+
+ + +
+
+ + + Confirm large AI indexing operation + +
+ +
+ +
+

+ Indexing this batch for the AI Assistant will use approximately{' '} + {formatBytes(estimateBytes, 1)} of disk space for embeddings, on top of the raw downloads. +

+ + {freeReason && ( +

+ That's more than 10% of your remaining free disk space ({formatBytes(freeReason.freeBytes, 1)} free). Embedding can take several hours and is hard to interrupt cleanly once started. +

+ )} + +

+ If you'd rather review per-item before indexing, cancel here and switch your Auto-index setting to Manual from the Knowledge Base panel. +

+
+ +
+ + Cancel + + + Proceed anyway + +
+ + +
+
+
+ + ) +} diff --git a/admin/inertia/components/TierSelectionModal.tsx b/admin/inertia/components/TierSelectionModal.tsx index 0a67169..255adb3 100644 --- a/admin/inertia/components/TierSelectionModal.tsx +++ b/admin/inertia/components/TierSelectionModal.tsx @@ -1,12 +1,28 @@ -import { Fragment, useState, useEffect } from 'react' +import { Fragment, useState, useEffect, useMemo } from 'react' import { Dialog, Transition } from '@headlessui/react' import { IconX, IconCheck, IconInfoCircle } from '@tabler/icons-react' +import { useQuery } from '@tanstack/react-query' import type { CategoryWithStatus, SpecTier, SpecResource } from '../../types/collections' import { resolveTierResources } from '~/lib/collections' import { formatBytes } from '~/lib/util' +import api from '~/lib/api' import classNames from 'classnames' import DynamicIcon, { DynamicIconName } from './DynamicIcon' import StyledButton from './StyledButton' +import KbGuardrailModal from './KbGuardrailModal' +import { evaluateGuardrail, type GuardrailVerdict } from '~/lib/kb_guardrail' +import { useSystemInfo } from '~/hooks/useSystemInfo' +import { getPrimaryDiskInfo } from '~/hooks/useDiskDisplayData' + +/** + * Filename for the embed-estimate registry lookup. Strips the URL path so + * patterns like `wikipedia_en_simple_` continue to match upstream filenames + * regardless of mirror domain. + */ +function resourceFilename(resource: SpecResource): string { + const last = resource.url.split('/').pop() + return last && last.length > 0 ? last : resource.id +} interface TierSelectionModalProps { isOpen: boolean @@ -33,13 +49,70 @@ const TierSelectionModal: React.FC = ({ } }, [isOpen, category, selectedTierSlug]) - if (!category) return null - - // Get all resources for a tier (including inherited resources) + // Get all resources for a tier (including inherited resources). Defined as a + // hook-safe closure (always callable, returns [] when no category) so the + // memo below can depend on `category` without breaking hook order. const getAllResourcesForTier = (tier: SpecTier): SpecResource[] => { + if (!category) return [] return resolveTierResources(tier, category.tiers) } + // Pre-compute the selected tier's resources outside the JSX so hooks below + // don't re-run on every render. Empty array when no selection. + const selectedTierResources = useMemo(() => { + if (!category || !localSelectedSlug) return [] + const tier = category.tiers.find((t) => t.slug === localSelectedSlug) + return tier ? resolveTierResources(tier, category.tiers) : [] + }, [category, localSelectedSlug]) + + const embedEstimateRequest = useMemo( + () => + selectedTierResources.map((r) => ({ + filename: resourceFilename(r), + sizeBytes: Math.round(r.size_mb * 1024 * 1024), + })), + [selectedTierResources] + ) + + const { data: embedEstimate, isLoading: isEstimating } = useQuery({ + queryKey: ['embedEstimateBatch', embedEstimateRequest], + queryFn: () => api.estimateEmbeddingBatch(embedEstimateRequest), + enabled: embedEstimateRequest.length > 0, + staleTime: 5 * 60_000, + }) + + const { data: ingestPolicySetting } = useQuery({ + queryKey: ['ingestPolicy'], + queryFn: () => api.getSetting('rag.defaultIngestPolicy'), + }) + + // System info for the disk-free side of the guardrail. Shared queryKey with + // the home / easy-setup pages so we don't refetch when the user already has + // a fresh copy in cache from a sibling component. + const { data: systemInfo } = useSystemInfo({ enabled: true }) + + // Open state for the guardrail modal — separate from the tier modal so the + // user sees the warning as an overlay without losing their tier selection + // underneath. Cancel returns to the tier modal as-is; Proceed closes both + // and runs the original onSelectTier path. + const [guardrailVerdict, setGuardrailVerdict] = useState(null) + + // Compute disk-free bytes from system info; 0 means "unknown", which the + // guardrail helper treats as "skip the relative-disk check". + // Must be declared before the `!category` early return so the hook count + // stays constant across renders (category transitions null → non-null when + // the user opens the modal). + const freeBytes = useMemo(() => { + const primary = getPrimaryDiskInfo(systemInfo?.disk, systemInfo?.fsSize) + if (!primary) return 0 + return Math.max(0, primary.totalSize - primary.totalUsed) + }, [systemInfo]) + + const ingestPolicy: 'Always' | 'Manual' = + ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always' + + if (!category) return null + const getTierTotalSize = (tier: SpecTier): number => { return getAllResourcesForTier(tier).reduce((acc, r) => acc + r.size_mb * 1024 * 1024, 0) } @@ -53,17 +126,43 @@ const TierSelectionModal: React.FC = ({ } } - const handleSubmit = () => { - if (!localSelectedSlug) return - - const selectedTier = category.tiers.find(t => t.slug === localSelectedSlug) + /** + * Runs the original onSelectTier-then-onClose flow. Pulled out of + * handleSubmit so the guardrail modal's confirm path can call it after + * the user has consented to the large operation. + */ + const finalizeSubmit = () => { + if (!localSelectedSlug || !category) return + const selectedTier = category.tiers.find((t) => t.slug === localSelectedSlug) if (selectedTier) { onSelectTier(category, selectedTier) } onClose() } + const handleSubmit = () => { + if (!localSelectedSlug || !category) return + + // Guardrail only runs when we have an estimate AND the global policy + // would auto-index this batch. Under Manual the user has already opted + // out of automatic ingestion, so the bulk-disk warning would be a false + // alarm — the files would just queue as pending_decision. + if (ingestPolicy === 'Always' && embedEstimate) { + const verdict = evaluateGuardrail({ + estimateBytes: embedEstimate.totalBytes, + freeBytes, + }) + if (verdict.trips) { + setGuardrailVerdict(verdict) + return + } + } + + finalizeSubmit() + } + return ( + <> = ({ })}
+ {/* Embedding-cost preview — visible whenever a tier is + selected. The estimate uses #891's ratio registry to + project how much extra disk space the AI Assistant will + need for these files on top of the raw downloads. */} + {localSelectedSlug && embedEstimate && embedEstimate.totalBytes > 0 && ( +
+
+ +
+

+ +~{formatBytes(embedEstimate.totalBytes, 1)} + {' '}of additional storage if these are indexed for the AI Assistant + {embedEstimate.hasUnknown && ( + (estimate excludes some files we have no prior data for) + )} + . +

+

+ {ingestPolicy === 'Always' ? ( + <> + Your Auto-index setting is Always, so these files will be indexed automatically once downloaded. You can change this in the Knowledge Base settings. + + ) : ( + <> + Your Auto-index setting is Manual, so these files will sit unindexed until you opt in from the Knowledge Base settings. + + )} +

+
+
+
+ )} + {/* Info note */} -
+

You can change your selection at any time. Click Submit to confirm your choice. @@ -218,7 +350,7 @@ const TierSelectionModal: React.FC = ({ variant='primary' size='lg' onClick={handleSubmit} - disabled={!localSelectedSlug} + disabled={!localSelectedSlug || (embedEstimateRequest.length > 0 && isEstimating)} > Submit @@ -229,6 +361,18 @@ const TierSelectionModal: React.FC = ({

+ {guardrailVerdict && ( + { + setGuardrailVerdict(null) + finalizeSubmit() + }} + onCancel={() => setGuardrailVerdict(null)} + /> + )} + ) } diff --git a/admin/inertia/components/chat/KbPolicyPromptBanner.tsx b/admin/inertia/components/chat/KbPolicyPromptBanner.tsx new file mode 100644 index 0000000..b2fd99b --- /dev/null +++ b/admin/inertia/components/chat/KbPolicyPromptBanner.tsx @@ -0,0 +1,126 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { usePage } from '@inertiajs/react' +import { IconBrain } from '@tabler/icons-react' +import api from '~/lib/api' +import StyledButton from '~/components/StyledButton' +import { useNotifications } from '~/context/NotificationContext' + +/** + * First-chat onboarding banner (RFC #883 Phase 3 task 12). + * + * Renders above the chat header when the scanner has seen at least one + * embeddable file AND the user has not yet picked a global ingest policy + * (`rag.defaultIngestPolicy` unset). Two buttons let the user decide once, + * after which the prompt never returns: + * + * - "Index existing content" → sets policy=Always and dispatches a sync so + * anything already on disk + in `pending_decision` gets queued for embed. + * - "Maybe later" → sets policy=Manual. New content waits in + * `pending_decision` until the user opts in from the KB modal. + * + * The "dismiss without deciding" X is intentionally NOT here. Dismissing + * without setting policy would make the banner reappear on every visit until + * a choice is recorded — annoying. The two action buttons each set policy, + * and the user can change their mind any time via the Always/Manual radio in + * the KB modal. + */ +export default function KbPolicyPromptBanner() { + const queryClient = useQueryClient() + const { addNotification } = useNotifications() + // Inertia injects `aiAssistantName` as a shared page prop on chat-mounted + // pages so the banner pulls the user-set name when surfaced. Default to + // "AI Assistant" when accessed outside that context (no-op for chat pages, + // but keeps the component safe for future reuse elsewhere). + const aiAssistantName = + usePage<{ aiAssistantName?: string }>().props?.aiAssistantName || 'AI Assistant' + + const { data: promptState } = useQuery({ + queryKey: ['kbPolicyPromptState'], + queryFn: () => api.getKbPolicyPromptState(), + staleTime: Infinity, + }) + + const indexNowMutation = useMutation({ + mutationFn: async () => { + await api.updateSetting('rag.defaultIngestPolicy', 'Always') + await api.syncRAGStorage() + }, + onSuccess: () => { + addNotification({ + type: 'success', + message: `${aiAssistantName} will index your existing content. You can track progress in the Knowledge Base panel.`, + }) + queryClient.invalidateQueries({ queryKey: ['kbPolicyPromptState'] }) + queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + }, + onError: (error: any) => { + addNotification({ + type: 'error', + message: error?.message || 'Could not start indexing. Try again from the Knowledge Base panel.', + }) + }, + }) + + const maybeLaterMutation = useMutation({ + mutationFn: () => api.updateSetting('rag.defaultIngestPolicy', 'Manual'), + onSuccess: () => { + addNotification({ + type: 'success', + message: 'Your content stays unindexed for now. You can opt in any time from the Knowledge Base panel.', + }) + queryClient.invalidateQueries({ queryKey: ['kbPolicyPromptState'] }) + queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] }) + }, + onError: (error: any) => { + addNotification({ + type: 'error', + message: error?.message || 'Could not save your choice. Try again.', + }) + }, + }) + + if (!promptState?.shouldPrompt) return null + + const fileCount = promptState.totalFiles + const isBusy = indexNowMutation.isPending || maybeLaterMutation.isPending + + return ( +
+
+ +
+

+ + {fileCount === 1 + ? `Index your existing file for ${aiAssistantName}?` + : `Index your ${fileCount.toLocaleString()} existing files for ${aiAssistantName}?`} + + {' '}When indexed, {aiAssistantName} can reference them while answering your questions. +

+
+
+ indexNowMutation.mutate()} + variant="primary" + size="sm" + disabled={isBusy} + loading={indexNowMutation.isPending} + > + Index existing content + + maybeLaterMutation.mutate()} + variant="ghost" + size="sm" + disabled={isBusy} + loading={maybeLaterMutation.isPending} + > + Maybe later + +
+
+
+ ) +} diff --git a/admin/inertia/components/chat/KnowledgeBaseModal.tsx b/admin/inertia/components/chat/KnowledgeBaseModal.tsx index 6230398..301bf67 100644 --- a/admin/inertia/components/chat/KnowledgeBaseModal.tsx +++ b/admin/inertia/components/chat/KnowledgeBaseModal.tsx @@ -2,10 +2,16 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { useEffect, useRef, useState } from 'react' import FileUploader from '~/components/file-uploader' import StyledButton from '~/components/StyledButton' +import type { DynamicIconName } from '~/lib/icons' import StyledSectionHeader from '~/components/StyledSectionHeader' import StyledTable from '~/components/StyledTable' import { useNotifications } from '~/context/NotificationContext' import api from '~/lib/api' +import { + groupAndSortKbFiles, + type KbFileGroup, +} from '~/lib/kb_file_grouping' +import type { KbIngestStateValue } from '../../../types/kb_ingest_state' import { IconX } from '@tabler/icons-react' import { useModals } from '~/context/ModalContext' import StyledModal from '../StyledModal' @@ -17,9 +23,74 @@ interface KnowledgeBaseModalProps { onClose: () => void } -function sourceToDisplayName(source: string): string { - const parts = source.split(/[/\\]/) - return parts[parts.length - 1] +/** + * Compact label for the per-row ingestion state. Files that exist in Qdrant + * with no `kb_ingest_state` row (`state === null`) are legacy/pre-RFC-883 + * installs whose chunks are real, so we display them as "Indexed" rather than + * surfacing the absent-row detail. Admin-docs group has no pill (the "Managed + * by NOMAD" message in the action column carries the same signal). + */ +function renderStatePill(record: KbFileGroup): React.ReactNode { + if (record.bucket === 'admin_docs') return null + const effective: KbIngestStateValue = record.state ?? 'indexed' + + const base = 'inline-flex items-center text-xs font-medium rounded px-2 py-0.5 border' + switch (effective) { + case 'indexed': + return ( + + Indexed + + ) + case 'pending_decision': + case 'browse_only': + return ( + + Not Indexed + + ) + case 'failed': + return ( + + Failed + + ) + case 'stalled': + return ( + + Stalled + + ) + } +} + +type RowAction = + | { kind: 'index'; label: string; force: boolean; variant: 'primary'; icon: DynamicIconName } + | { kind: 'reembed'; label: string; force: true; variant: 'secondary'; icon: DynamicIconName } + +/** + * Pick the single adaptive per-row action button. Returns null when no action + * makes sense for the current state (e.g. healthy indexed file with no + * warnings — bulk Re-embed All covers that case). `hasWarnings` lets us + * surface a Re-embed affordance specifically when a file *looks* indexed but + * has zero chunks or a stalled-mid-ingestion warning attached. + */ +function pickRowAction(record: KbFileGroup, hasWarnings: boolean): RowAction | null { + if (record.bucket === 'admin_docs') return null + const effective: KbIngestStateValue = record.state ?? 'indexed' + switch (effective) { + case 'indexed': + return hasWarnings + ? { kind: 'reembed', label: 'Re-embed', force: true, variant: 'secondary', icon: 'IconRefreshAlert' } + : null + case 'pending_decision': + return { kind: 'index', label: 'Index', force: false, variant: 'primary', icon: 'IconDownload' } + case 'browse_only': + return { kind: 'index', label: 'Index', force: true, variant: 'primary', icon: 'IconDownload' } + case 'failed': + case 'stalled': + return { kind: 'index', label: 'Retry', force: true, variant: 'primary', icon: 'IconRefresh' } + } } export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", onClose }: KnowledgeBaseModalProps) { @@ -27,6 +98,9 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o const [files, setFiles] = useState([]) const [isUploading, setIsUploading] = useState(false) const [confirmDeleteSource, setConfirmDeleteSource] = useState(null) + const [confirmReembed, setConfirmReembed] = useState<{ source: string; displayName: string } | null>(null) + const [bulkMode, setBulkMode] = useState(null) + const [resetTyped, setResetTyped] = useState('') const fileUploaderRef = useRef>(null) const { openModal, closeModal } = useModals() const queryClient = useQueryClient() @@ -50,6 +124,49 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o select: (data) => data || [], }) + // Per-file conditional warnings (RFC #883 §6). `ok: false` means the + // computation itself failed (Qdrant/DB/FS) — distinct from `ok: true` with + // an empty map, which means everything is healthy. We surface the failure + // explicitly so a silent backend failure doesn't masquerade as health. + const { data: warningsResult } = useQuery({ + queryKey: ['kbFileWarnings'], + queryFn: () => api.getKbFileWarnings(), + refetchInterval: 30_000, + }) + const fileWarnings = warningsResult?.warnings ?? {} + const warningsUnavailable = warningsResult !== undefined && warningsResult.ok === false + + // Global auto-index policy. KVStore returns `null` for an unset key, which + // we treat as 'Always' for backward compatibility with installs that predate + // this UI. The user can opt into Manual mode from the toggle below. + const { data: ingestPolicySetting } = useQuery({ + queryKey: ['ingestPolicy'], + queryFn: () => api.getSetting('rag.defaultIngestPolicy'), + }) + const ingestPolicy: 'Always' | 'Manual' = + ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always' + + const updateIngestPolicyMutation = useMutation({ + mutationFn: (policy: 'Always' | 'Manual') => + api.updateSetting('rag.defaultIngestPolicy', policy), + onSuccess: (_data, policy) => { + queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] }) + addNotification({ + type: 'success', + message: + policy === 'Always' + ? 'New content will be auto-indexed for AI.' + : 'New content will wait for you to opt in.', + }) + }, + onError: (error: any) => { + addNotification({ + type: 'error', + message: error?.message || 'Failed to update indexing policy.', + }) + }, + }) + const uploadMutation = useMutation({ mutationFn: (file: File) => api.uploadDocument(file), }) @@ -67,6 +184,25 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }) + const embedMutation = useMutation({ + mutationFn: ({ source, force }: { source: string; force: boolean }) => + api.embedSingleRAGFile(source, force), + onSuccess: (data) => { + addNotification({ + type: 'success', + message: data?.message || 'File queued for embedding.', + }) + setConfirmReembed(null) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + queryClient.invalidateQueries({ queryKey: ['kbFileWarnings'] }) + }, + onError: (error: any) => { + addNotification({ type: 'error', message: error?.message || 'Failed to queue file.' }) + setConfirmReembed(null) + }, + }) + const cleanupFailedMutation = useMutation({ mutationFn: () => api.cleanupFailedEmbedJobs(), onSuccess: (data) => { @@ -105,6 +241,44 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }) + const reembedMutation = useMutation({ + mutationFn: () => api.reembedAllRAG(), + onSuccess: (data) => { + addNotification({ + type: data?.success ? 'success' : 'error', + message: data?.message || 'Re-embed completed.', + }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + setBulkMode(null) + setResetTyped('') + }, + onError: () => { + addNotification({ type: 'error', message: 'Failed to re-embed knowledge base.' }) + setBulkMode(null) + }, + }) + + const resetMutation = useMutation({ + mutationFn: () => api.resetAndRebuildRAG(), + onSuccess: (data) => { + addNotification({ + type: data?.success ? 'success' : 'error', + message: data?.message || 'Reset complete.', + }) + queryClient.invalidateQueries({ queryKey: ['storedFiles'] }) + queryClient.invalidateQueries({ queryKey: ['embed-jobs'] }) + setBulkMode(null) + setResetTyped('') + }, + onError: () => { + addNotification({ type: 'error', message: 'Failed to reset knowledge base.' }) + setBulkMode(null) + }, + }) + + const bulkBusy = reembedMutation.isPending || resetMutation.isPending + const handleUpload = async () => { if (files.length === 0) return setIsUploading(true) @@ -268,6 +442,48 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
+
+
+
+

+ Auto-index new content for AI? +

+

+ Indexed content typically uses 5–10× the original file size on disk. + Changes apply to new content added after this setting changes. +

+
+
+ {(['Always', 'Manual'] as const).map((option) => { + const isActive = ingestPolicy === option + return ( + + ) + })} +
+
+
+
@@ -286,20 +502,54 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
-
+
- - Sync Storage - +
+ { setResetTyped(''); setBulkMode('reset') }} + disabled={isUploading || qdrantOffline || bulkBusy} + loading={resetMutation.isPending} + title="Drop the entire embeddings collection and re-embed everything from scratch. Permanently removes vectors for files no longer on disk. Destructive: requires typing RESET to confirm." + > + Reset & Rebuild + + setBulkMode('reembed')} + disabled={isUploading || qdrantOffline || bulkBusy || storedFiles.length === 0} + loading={reembedMutation.isPending} + title="Re-embed every file on disk, replacing existing vectors file-by-file. Vectors for files no longer on disk are preserved. Use this if the chunker or embedding model has changed." + > + Re-embed All + + + Sync Storage + + +
- + {warningsUnavailable && ( +
+ + + File warnings unavailable — couldn't read storage state. Retrying… + +
+ )} + className="font-semibold" rowLines={true} columns={[ @@ -307,13 +557,60 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o accessor: 'source', title: 'File Name', render(record) { - return {sourceToDisplayName(record.source)} + const warnings = fileWarnings[record.source] ?? [] + const pill = renderStatePill(record) + return ( +
+ + {record.displayName} + + {(pill || warnings.length > 0) && ( +
+ {pill} + {warnings.map((w, i) => ( + + + {w.kind === 'zero_chunks' && ( + + Embedded 0 chunks — this file has no text content. + AI Assistant cannot reference it. + + )} + {w.kind === 'partial_stall' && ( + + Only {w.chunksEmbedded.toLocaleString()} of est.{' '} + {w.chunksExpected.toLocaleString()} chunks embedded — + ingestion may have stalled. + + )} + + ))} +
+ )} +
+ ) }, }, { accessor: 'source', title: '', render(record) { + // Admin docs are auto-discovered and managed by NOMAD itself — + // deleting one would just be re-embedded on the next sync, so + // we surface them as informational only and hide Delete. + if (record.bucket === 'admin_docs') { + return ( +
+ + Managed by NOMAD + +
+ ) + } + const isConfirming = confirmDeleteSource === record.source const isDeleting = deleteMutation.isPending && confirmDeleteSource === record.source if (isConfirming) { @@ -339,14 +636,38 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
) } + + const warnings = fileWarnings[record.source] ?? [] + const action = pickRowAction(record, warnings.length > 0) + const actionPendingForThisRow = + embedMutation.isPending && embedMutation.variables?.source === record.source + return ( -
+
+ {action && ( + { + if (action.kind === 'reembed') { + setConfirmReembed({ source: record.source, displayName: record.displayName }) + } else { + embedMutation.mutate({ source: record.source, force: action.force }) + } + }} + disabled={qdrantOffline || deleteMutation.isPending || embedMutation.isPending} + loading={actionPendingForThisRow} + > + {action.label} + + )} setConfirmDeleteSource(record.source)} - disabled={deleteMutation.isPending} + disabled={deleteMutation.isPending || embedMutation.isPending} loading={deleteMutation.isPending && confirmDeleteSource === record.source} >Delete
@@ -354,12 +675,138 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o }, }, ]} - data={storedFiles.map((source) => ({ source }))} + data={groupAndSortKbFiles(storedFiles)} loading={isLoadingFiles} />
+ + {bulkMode === 'reembed' && ( + reembedMutation.mutate()} + onCancel={() => setBulkMode(null)} + > +
+

+ This will re-process every document currently in your knowledge base — about + {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'}. + For each file, NOMAD will delete the existing embeddings from Qdrant and queue a fresh + embedding job using the current chunking and embedding model. +

+
+

What this is for

+

+ Use this when the embedding model or chunking logic has changed, or when you suspect + stored vectors are stale. Files on disk are not deleted, and any orphan + points whose source file is no longer present will be preserved untouched (see + Reset & Rebuild if you want a fully clean slate). +

+
+
+

Heads up

+
    +
  • Embedding {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'} may take a long time, especially for large PDFs or ZIM archives.
    • +
  • On systems without GPU acceleration, expect sustained high CPU usage for the duration.
    • +
  • Knowledge Base search results may be incomplete until every file finishes re-embedding.
    • +
  • If embed jobs are already in progress, this action will be refused — wait for the queue to drain first.
    • +
+
+
+ + )} + + {bulkMode === 'reset' && ( + { + if (resetTyped === 'RESET') resetMutation.mutate() + }} + onCancel={() => { setBulkMode(null); setResetTyped('') }} + > +
+

+ This will permanently delete every point in the + nomad_knowledge_base Qdrant collection and rebuild from the + {storedFiles.length} file{storedFiles.length === 1 ? '' : 's'} currently + on disk. The collection is dropped, recreated, and every file is re-queued for embedding. +

+
+

How this differs from Re-embed All

+
    +
  • Re-embed All replaces vectors file-by-file. Any orphan points (vectors whose source file was deleted from disk at some point) are preserved.
    • +
  • Reset & Rebuild drops the entire collection. Orphan points are gone forever. Only files currently on disk will exist in Qdrant afterwards.
    • +
+
+
+

This action is destructive and cannot be undone

+
    +
  • Knowledge Base search will be empty until embedding finishes (potentially hours on CPU-only systems).
    • +
  • For a few seconds during the reset, the Qdrant collection does not exist — any chat-with-RAG queries in that window may return a "collection not found" error. Avoid using chat until the rebuild has begun.
    • +
  • If embed jobs are already in progress, this action will be refused — wait for the queue to drain first.
    • +
+
+
+ + setResetTyped(e.target.value)} + placeholder='RESET' + autoFocus + className='w-full rounded border border-border-subtle bg-surface-primary px-3 py-2 text-text-primary focus:outline-none focus:ring-2 focus:ring-red-500' + /> + {resetTyped.length > 0 && resetTyped !== 'RESET' && ( +

Type RESET exactly (uppercase, no spaces) to enable the confirm button.

+ )} +
+
+ + )} + + {confirmReembed && ( + + embedMutation.mutate({ source: confirmReembed.source, force: true }) + } + onCancel={() => setConfirmReembed(null)} + > +
+

+ This will delete the existing embeddings for{' '} + {confirmReembed.displayName} and queue + a fresh embedding job. The file on disk is not touched. +

+
+

Heads up

+
    +
  • For large ZIM archives this can take a long time, especially on CPU-only systems.
    • +
  • Search results that referenced this file will be incomplete until the new embedding finishes.
    • +
  • If a job for this file is already running, the re-embed will be refused — wait for it to finish first.
    • +
+
+
+ + )} ) } diff --git a/admin/inertia/components/chat/index.tsx b/admin/inertia/components/chat/index.tsx index 577579a..9be60d4 100644 --- a/admin/inertia/components/chat/index.tsx +++ b/admin/inertia/components/chat/index.tsx @@ -2,6 +2,7 @@ import { useState, useCallback, useEffect, useRef, useMemo } from 'react' import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' import ChatSidebar from './ChatSidebar' import ChatInterface from './ChatInterface' +import KbPolicyPromptBanner from './KbPolicyPromptBanner' import StyledModal from '../StyledModal' import api from '~/lib/api' import { formatBytes } from '~/lib/util' @@ -32,6 +33,8 @@ export default function Chat({ const [activeSessionId, setActiveSessionId] = useState(null) const [messages, setMessages] = useState([]) const [selectedModel, setSelectedModel] = useState('') + const [pendingModelSwitch, setPendingModelSwitch] = useState(null) + const pageLoadNormalizedRef = useRef(false) const [isStreamingResponse, setIsStreamingResponse] = useState(false) const streamAbortRef = useRef(null) @@ -150,6 +153,62 @@ export default function Chat({ } }, [selectedModel]) + // Page-load normalization: enforce the "one chat model at a time" invariant + // when the chat page first mounts. Anything stacked from a prior session + // gets `keep_alive: 0` so it can be evicted; the embedding model is exempt + // server-side. We wait for `selectedModel` to be populated by the + // first-installed / lastModel effect so the request has a target to preserve. + useEffect(() => { + if (!enabled) return + if (!selectedModel) return + if (pageLoadNormalizedRef.current) return + pageLoadNormalizedRef.current = true + api.unloadChatModels(selectedModel).catch((err) => { + console.warn('Failed to normalize loaded models on chat-page mount:', err) + }) + }, [enabled, selectedModel]) + + const handleUserSelectedModel = useCallback( + (newModel: string) => { + if (newModel === selectedModel) return + // No active chat session yet → no conversation to lose, no popup needed. + // Just update the dropdown silently; the next "New Chat" will use it. + if (!activeSessionId) { + setSelectedModel(newModel) + return + } + // Active session: defer the actual model swap until the user confirms. + // Setting `pendingModelSwitch` drives the dropdown's effective value + // *and* opens the confirm modal — clearing it on cancel reverts the + // visible selection without us having to touch `selectedModel`. + setPendingModelSwitch(newModel) + }, + [selectedModel, activeSessionId] + ) + + const handleConfirmModelSwitch = useCallback(async () => { + const newModel = pendingModelSwitch + if (!newModel) return + // Best-effort unload of the previously-active chat model. Fire-and-forget: + // Ollama queues the eviction until the runner is idle, so an in-flight + // request on the old model finishes cleanly. We don't await this before + // clearing the session — UI responsiveness wins over housekeeping. + api.unloadChatModels(newModel).catch((err) => { + console.warn('Failed to unload previous chat model:', err) + }) + setSelectedModel(newModel) + setPendingModelSwitch(null) + // Clear the active session and messages — the next user message will + // lazily create a new session via the existing handleSendMessage path, + // which already calls api.createChatSession with `selectedModel`. + setActiveSessionId(null) + setMessages([]) + }, [pendingModelSwitch]) + + const handleCancelModelSwitch = useCallback(() => { + setPendingModelSwitch(null) + }, []) + const handleNewChat = useCallback(() => { // Just clear the active session and messages - don't create a session yet setActiveSessionId(null) @@ -201,8 +260,19 @@ export default function Chat({ if (sessionData?.model) { setSelectedModel(sessionData.model) } + + // Enforce the one-chat-model-at-a-time invariant: ask the backend to + // unload anything that isn't the target session's model. Fire-and-forget; + // this is housekeeping. Note we pass the *session's* model here rather + // than reading `selectedModel`, because setSelectedModel above is async + // and the effect-driven page-load normalize wouldn't catch a sidebar + // click after the first render. + const targetModel = sessionData?.model ?? selectedModel ?? null + api.unloadChatModels(targetModel).catch((err) => { + console.warn('Failed to unload non-target chat models on session switch:', err) + }) }, - [installedModels, queryClient] + [installedModels, queryClient, selectedModel] ) const handleSendMessage = useCallback( @@ -351,6 +421,23 @@ export default function Chat({ ) return ( + <> + {pendingModelSwitch && ( + +

+ Switching to {pendingModelSwitch} will start a new chat. Your current + conversation stays available in the sidebar. +

+ + )}
+

{activeSession?.title || 'New Chat'} @@ -394,8 +482,8 @@ export default function Chat({ ) : (