RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(rag): add explicit opt-in for ZIM files in the knowledge base 

ZIM/Kiwix files were previously auto-discovered and embedded into the AI
knowledge base as soon as they existed on disk, with no way to exclude one
without deleting it entirely. This made browsing content in Kiwix and
including it in AI search the same decision, which isn't what users expect
or consented to.

- Stop auto-discovering ZIM_STORAGE_PATH during KB sync; ZIM files must be
  explicitly added via the Content Manager's new 'Add to Knowledge Base'
  action (embedSingleFile validates directly against disk instead).
- Add a per-file active/inactive toggle (kb_ingest_state.active) so a file
  already indexed can be excluded from/re-included in search instantly,
  without re-embedding.
- searchSimilarDocuments excludes inactive sources via a Qdrant filter.
- Hard-deleting an indexed ZIM now also cleans up its kb_ingest_state row
  and Qdrant points, instead of leaving an orphaned KB entry behind.
- Batch the kb_ingest_state lookup in ZimService.list() into a single
  query instead of one per file.
This commit is contained in:
John Cortright 2026-07-18 15:18:39 -07:00
parent cb66faacbc
commit 5571e9cd1e
10 changed files with 199 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
admin/app/controllers/rag_controller.ts
View File

@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app'
import { randomBytes } from 'node:crypto'
import { sanitizeFilename } from '../utils/fs.js'
import { basename } from 'node:path'
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag'
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema, toggleActiveSchema } from '#validators/rag'
import logger from '@adonisjs/core/services/logger'
@inject()
@ -97,6 +97,17 @@ export default class RagController {
return response.status(202).json({ message: result.message })
}
public async toggleActive({ request, response }: HttpContext) {
const { source, active } = await request.validateUsing(toggleActiveSchema)
const found = await this.ragService.toggleFileActive(source, active)
if (!found) {
return response.status(404).json({ error: 'File is not a tracked knowledge-base source.' })
}
return response.status(200).json({
message: active ? 'File re-included in search.' : 'File excluded from search.',
})
}
public async getFailedJobs({ response }: HttpContext) {
const jobs = await EmbedFileJob.listFailedJobs()
return response.status(200).json(jobs)

11
admin/app/models/kb_ingest_state.ts
View File

@ -31,6 +31,9 @@ export default class KbIngestState extends BaseModel {
@column()
declare last_error: string | null
@column()
declare active: boolean
@column.dateTime({ autoCreate: true })
declare created_at: DateTime
@ -71,6 +74,14 @@ export default class KbIngestState extends BaseModel {
await row.save()
}
static async setActive(filePath: string, active: boolean): Promise<KbIngestState | null> {
const row = await this.query().where('file_path', filePath).first()
if (!row) return null
row.active = active
await row.save()
return row
}
static async remove(filePath: string): Promise<void> {
await this.query().where('file_path', filePath).delete()
}

44
admin/app/services/rag_service.ts
View File

@ -868,11 +868,21 @@ export class RagService {
`[RAG] Searching for top ${searchLimit} semantic matches (threshold: ${scoreThreshold})`
)
// Exclude sources the user has explicitly toggled inactive (e.g. a ZIM
// added to the KB but temporarily excluded from search) without
// touching their underlying embeddings -- see KbIngestState.active.
const inactiveSources = (
await KbIngestState.query().where('active', false).select('file_path')
).map((row) => row.file_path)
const searchResults = await this.qdrant!.search(RagService.CONTENT_COLLECTION_NAME, {
vector: response.embeddings[0],
limit: searchLimit,
score_threshold: scoreThreshold,
with_payload: true,
...(inactiveSources.length > 0
? { filter: { must_not: [{ key: 'source', match: { any: inactiveSources } }] } }
: {}),
})
logger.debug(`[RAG] Found ${searchResults.length} results above threshold ${scoreThreshold}`)
@ -1530,19 +1540,21 @@ export class RagService {
}
/**
* Walk kb_uploads and zim storage directories, returning the full path of
* every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are
* filtered out so they aren't dispatched only to fail with "Unsupported file
* type" and retry on every sync.
* Walk the kb_uploads storage directory, returning the full path of every
* embeddable file. ZIM_STORAGE_PATH is intentionally NOT walked here -- ZIM
* / Kiwix content must be explicitly opted into the knowledge base via
* embedSingleFile() (e.g. the Content Manager's "Add to Knowledge Base"
* action) rather than being auto-discovered by Sync Storage / Always-policy
* ingestion. Non-embeddable types (e.g. kiwix-library.xml) are filtered out
* so they aren't dispatched only to fail with "Unsupported file type" and
* retry on every sync.
*/
private async _discoverKbFiles(): Promise<string[]> {
const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH)
const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH)
const filesInStorage: string[] = []
for (const [label, dirPath] of [
[RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const,
[ZIM_STORAGE_PATH, ZIM_PATH] as const,
]) {
try {
const contents = await listDirectoryContentsRecursive(dirPath)
@ -1623,8 +1635,13 @@ export class RagService {
): Promise<EmbedSingleFileResult> {
const stateRow = await KbIngestState.query().where('file_path', source).first()
if (!stateRow) {
const knownFiles = await this._discoverKbFiles()
if (!knownFiles.includes(source)) {
// Deliberate opt-in path (e.g. "Add to Knowledge Base" on a ZIM file):
// validate directly against disk rather than via _discoverKbFiles(),
// which intentionally excludes ZIM_STORAGE_PATH from automatic
// discovery/sync. A file can still be explicitly added even though it
// won't be picked up by Sync Storage.
const stats = await getFileStatsIfExists(source)
if (!stats || determineFileType(source) === 'unknown') {
return {
success: false,
code: 'not_found',
@ -1672,6 +1689,17 @@ export class RagService {
}
}
/**
* Flip whether a tracked knowledge-base source is eligible for RAG search,
* without touching its underlying Qdrant points or re-embedding anything.
* Returns null if there's no kb_ingest_state row for this source (nothing
* to toggle -- it was never indexed).
*/
public async toggleFileActive(source: string, active: boolean): Promise<boolean> {
const row = await KbIngestState.setActive(source, active)
return row !== null
}
/**
* Delete all Qdrant points whose `source` payload matches the given path.
* Unlike deleteFileBySource(), this does NOT touch the file on disk — used

57
admin/app/services/zim_service.ts
View File

@ -25,6 +25,7 @@ import vine from '@vinejs/vine'
import { wikipediaOptionsFileSchema } from '#validators/curated_collections'
import WikipediaSelection from '#models/wikipedia_selection'
import InstalledResource from '#models/installed_resource'
import KbIngestState from '#models/kb_ingest_state'
import CollectionManifest from '#models/collection_manifest'
import { RunDownloadJob } from '#jobs/run_download_job'
import { SERVICE_NAMES } from '../../constants/service_names.js'
@ -48,16 +49,34 @@ export class ZimService {
const all = await listDirectoryContents(dirPath)
const zimEntries = all.filter((item) => item.name.endsWith('.zim'))
// Resolve each entry's file path up front so the kb_ingest_state lookup
// can be a single batched query instead of one query per file (N+1) --
// matters once there are dozens/hundreds of ZIMs (see #followup-proposal).
const filePaths = zimEntries.map((entry) =>
entry.type === 'file' ? entry.key : join(dirPath, entry.name)
)
const kbStates = filePaths.length
? await KbIngestState.query().whereIn('file_path', filePaths)
: []
const kbStateByPath = new Map(kbStates.map((row) => [row.file_path, row]))
const files = await Promise.all(
zimEntries.map(async (entry) => {
const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name)
zimEntries.map(async (entry, i) => {
const filePath = filePaths[i]
const stats = await getFileStatsIfExists(filePath)
const kbState = kbStateByPath.get(filePath)
const kb_status: 'not_added' | 'active' | 'inactive' = !kbState
? 'not_added'
: kbState.active
? 'active'
: 'inactive'
return {
...entry,
title: null,
summary: null,
author: null,
size_bytes: stats ? Number(stats.size) : null,
kb_status,
}
})
)
@ -552,18 +571,11 @@ export class ZimService {
logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error)
}
const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama')
if (ollamaUrl) {
try {
const { EmbedFileJob } = await import('#jobs/embed_file_job')
await EmbedFileJob.dispatch({
fileName: filename,
filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename),
})
} catch (error) {
logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error)
}
}
// NOTE: uploaded ZIM files are intentionally NOT auto-embedded into the
// knowledge base. ZIM/Kiwix content must be explicitly opted in via
// RagService.embedSingleFile() (e.g. the Content Manager's "Add to
// Knowledge Base" action) so browsing content in Kiwix never implies
// consent to include it in AI search.
return { added }
}
@ -605,6 +617,23 @@ export class ZimService {
logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`)
}
// If this ZIM was indexed into the AI knowledge base, clean up its Qdrant
// points + kb_ingest_state row too -- otherwise a hard delete here leaves
// an orphaned "Indexed" entry behind that never gets cleaned up until a
// full Reset & Rebuild.
try {
const kbRow = await KbIngestState.query().where('file_path', fullPath).first()
if (kbRow) {
const appModule = await import('@adonisjs/core/services/app')
const { RagService } = await import('#services/rag_service')
const ragService = await appModule.default.container.make(RagService)
await ragService.deleteFileBySource(fullPath)
logger.info(`[ZimService] Cleaned up knowledge-base entry for deleted ZIM: ${fileName}`)
}
} catch (error) {
logger.error(`[ZimService] Failed to clean up knowledge-base entry for ${fileName}:`, error)
}
// If this file was the active Wikipedia selection, clear the selection
try {
const selection = await WikipediaSelection.query().first()

7
admin/app/validators/rag.ts
View File

@ -19,6 +19,13 @@ export const embedFileSchema = vine.compile(
})
)
export const toggleActiveSchema = vine.compile(
vine.object({
source: vine.string().minLength(1),
active: vine.boolean(),
})
)
export const fileSourceSchema = vine.compile(
vine.object({
source: vine.string().minLength(1),

17
admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts Normal file
View File

@ -0,0 +1,17 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'kb_ingest_state'
async up() {
this.schema.alterTable(this.tableName, (table) => {
table.boolean('active').notNullable().defaultTo(true)
})
}
async down() {
this.schema.alterTable(this.tableName, (table) => {
table.dropColumn('active')
})
}
}

7
admin/inertia/lib/api.ts
View File

@ -507,6 +507,13 @@ class API {
})()
}
async toggleRagFileActive(source: string, active: boolean) {
return catchInternal(async () => {
const response = await this.client.post<{ message: string }>('/rag/files/toggle-active', { source, active })
return response.data
})()
}
async getKbFileWarnings() {
return catchInternal(async () => {
const response = await this.client.get<FileWarningsResult>('/rag/file-warnings')

62
admin/inertia/pages/settings/zim/index.tsx
View File

@ -11,6 +11,7 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
import Alert from '~/components/Alert'
import { useNotifications } from '~/context/NotificationContext'
import ZimUploader from '~/components/ZimUploader'
import Switch from '~/components/inputs/Switch'
import { ZimFileWithMetadata } from '../../../../types/zim'
import { SERVICE_NAMES } from '../../../../constants/service_names'
import { formatBytes } from '~/lib/util'
@ -126,6 +127,34 @@ export default function ZimPage() {
},
})
const embedMutation = useMutation({
mutationFn: async (file: ZimFileWithMetadata) =>
api.embedSingleRAGFile(file.type === 'file' ? file.key : file.name),
onSuccess: (result) => {
if (!result) return
queryClient.invalidateQueries({ queryKey: ['zim-files'] })
addNotification({
type: 'success',
message: result.message || 'Added to the knowledge base. It will be searchable once embedding finishes.',
})
},
})
const toggleActiveMutation = useMutation({
mutationFn: async ({ file, active }: { file: ZimFileWithMetadata; active: boolean }) =>
api.toggleRagFileActive(file.type === 'file' ? file.key : file.name, active),
onSuccess: (result, { active }) => {
if (!result) return
queryClient.invalidateQueries({ queryKey: ['zim-files'] })
addNotification({
type: 'success',
message: active
? 'Re-included in AI search.'
: 'Excluded from AI search. It stays indexed, so re-enabling is instant.',
})
},
})
return (
<SettingsLayout>
<Head title="Content Manager | Project NOMAD" />
@ -223,6 +252,39 @@ export default function ZimPage() {
</span>
),
},
{
accessor: 'kb_status',
title: 'Knowledge Base',
render: (record) => {
if (record.kb_status === 'not_added') {
const pending =
embedMutation.isPending && embedMutation.variables?.name === record.name
return (
<StyledButton
variant="secondary"
size="sm"
icon="IconPlus"
loading={pending}
onClick={() => embedMutation.mutate(record)}
>
Add to Knowledge Base
</StyledButton>
)
}
const isActive = record.kb_status === 'active'
const pending =
toggleActiveMutation.isPending &&
toggleActiveMutation.variables?.file.name === record.name
return (
<Switch
checked={isActive}
onChange={(checked) => toggleActiveMutation.mutate({ file: record, active: checked })}
disabled={pending}
label={isActive ? 'Active' : 'Inactive'}
/>
)
},
},
{
accessor: 'actions',
title: 'Actions',

1
admin/start/routes.ts
View File

@ -149,6 +149,7 @@ router
router.get('/file-warnings', [RagController, 'getFileWarnings'])
router.delete('/files', [RagController, 'deleteFile'])
router.post('/files/embed', [RagController, 'embedFile'])
router.post('/files/toggle-active', [RagController, 'toggleActive'])
router.get('/files/content', [RagController, 'getFileContent'])
router.get('/files/download', [RagController, 'downloadFile'])
router.get('/active-jobs', [RagController, 'getActiveJobs'])

3
admin/types/zim.ts
View File

@ -1,10 +1,13 @@
import { FileEntry } from './files.js'
export type ZimKbStatus = 'not_added' | 'active' | 'inactive'
export type ZimFileWithMetadata = FileEntry & {
title: string | null
summary: string | null
author: string | null
size_bytes: number | null
kb_status: ZimKbStatus
}
export type ListZimFilesResponse = {