RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 5571e9cd1e into cb66faacbc

This commit is contained in:
just-jbc 2026-07-18 22:44:32 +00:00 committed by GitHub
parent cb66faacbc 5571e9cd1e
commit 960e043907
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 199 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
admin/app/controllers/rag_controller.ts
View File

@ -7,7 +7,7 @@ import app from '@adonisjs/core/services/app'
import { randomBytes } from 'node:crypto'
import { sanitizeFilename } from '../utils/fs.js'
import { basename } from 'node:path'
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema } from '#validators/rag'
import { deleteFileSchema, embedFileSchema, estimateBatchSchema, fileSourceSchema, getJobStatusSchema, toggleActiveSchema } from '#validators/rag'
import logger from '@adonisjs/core/services/logger'
@inject()
@ -97,6 +97,17 @@ export default class RagController {
return response.status(202).json({ message: result.message })
}
public async toggleActive({ request, response }: HttpContext) {
const { source, active } = await request.validateUsing(toggleActiveSchema)
const found = await this.ragService.toggleFileActive(source, active)
if (!found) {
return response.status(404).json({ error: 'File is not a tracked knowledge-base source.' })
}
return response.status(200).json({
message: active ? 'File re-included in search.' : 'File excluded from search.',
})
}
public async getFailedJobs({ response }: HttpContext) {
const jobs = await EmbedFileJob.listFailedJobs()
return response.status(200).json(jobs)

11
admin/app/models/kb_ingest_state.ts
View File

@ -31,6 +31,9 @@ export default class KbIngestState extends BaseModel {
@column()
declare last_error: string | null
@column()
declare active: boolean
@column.dateTime({ autoCreate: true })
declare created_at: DateTime
@ -71,6 +74,14 @@ export default class KbIngestState extends BaseModel {
await row.save()
}
static async setActive(filePath: string, active: boolean): Promise<KbIngestState | null> {
const row = await this.query().where('file_path', filePath).first()
if (!row) return null
row.active = active
await row.save()
return row
}
static async remove(filePath: string): Promise<void> {
await this.query().where('file_path', filePath).delete()
}

44
admin/app/services/rag_service.ts
View File

@ -868,11 +868,21 @@ export class RagService {
`[RAG] Searching for top ${searchLimit} semantic matches (threshold: ${scoreThreshold})`
)
// Exclude sources the user has explicitly toggled inactive (e.g. a ZIM
// added to the KB but temporarily excluded from search) without
// touching their underlying embeddings -- see KbIngestState.active.
const inactiveSources = (
await KbIngestState.query().where('active', false).select('file_path')
).map((row) => row.file_path)
const searchResults = await this.qdrant!.search(RagService.CONTENT_COLLECTION_NAME, {
vector: response.embeddings[0],
limit: searchLimit,
score_threshold: scoreThreshold,
with_payload: true,
...(inactiveSources.length > 0
? { filter: { must_not: [{ key: 'source', match: { any: inactiveSources } }] } }
: {}),
})
logger.debug(`[RAG] Found ${searchResults.length} results above threshold ${scoreThreshold}`)
@ -1530,19 +1540,21 @@ export class RagService {
}
/**
* Walk kb_uploads and zim storage directories, returning the full path of
* every embeddable file. Non-embeddable types (e.g. kiwix-library.xml) are
* filtered out so they aren't dispatched only to fail with "Unsupported file
* type" and retry on every sync.
* Walk the kb_uploads storage directory, returning the full path of every
* embeddable file. ZIM_STORAGE_PATH is intentionally NOT walked here -- ZIM
* / Kiwix content must be explicitly opted into the knowledge base via
* embedSingleFile() (e.g. the Content Manager's "Add to Knowledge Base"
* action) rather than being auto-discovered by Sync Storage / Always-policy
* ingestion. Non-embeddable types (e.g. kiwix-library.xml) are filtered out
* so they aren't dispatched only to fail with "Unsupported file type" and
* retry on every sync.
*/
private async _discoverKbFiles(): Promise<string[]> {
const KB_UPLOADS_PATH = join(process.cwd(), RagService.UPLOADS_STORAGE_PATH)
const ZIM_PATH = join(process.cwd(), ZIM_STORAGE_PATH)
const filesInStorage: string[] = []
for (const [label, dirPath] of [
[RagService.UPLOADS_STORAGE_PATH, KB_UPLOADS_PATH] as const,
[ZIM_STORAGE_PATH, ZIM_PATH] as const,
]) {
try {
const contents = await listDirectoryContentsRecursive(dirPath)
@ -1623,8 +1635,13 @@ export class RagService {
): Promise<EmbedSingleFileResult> {
const stateRow = await KbIngestState.query().where('file_path', source).first()
if (!stateRow) {
const knownFiles = await this._discoverKbFiles()
if (!knownFiles.includes(source)) {
// Deliberate opt-in path (e.g. "Add to Knowledge Base" on a ZIM file):
// validate directly against disk rather than via _discoverKbFiles(),
// which intentionally excludes ZIM_STORAGE_PATH from automatic
// discovery/sync. A file can still be explicitly added even though it
// won't be picked up by Sync Storage.
const stats = await getFileStatsIfExists(source)
if (!stats || determineFileType(source) === 'unknown') {
return {
success: false,
code: 'not_found',
@ -1672,6 +1689,17 @@ export class RagService {
}
}
/**
* Flip whether a tracked knowledge-base source is eligible for RAG search,
* without touching its underlying Qdrant points or re-embedding anything.
* Returns null if there's no kb_ingest_state row for this source (nothing
* to toggle -- it was never indexed).
*/
public async toggleFileActive(source: string, active: boolean): Promise<boolean> {
const row = await KbIngestState.setActive(source, active)
return row !== null
}
/**
* Delete all Qdrant points whose `source` payload matches the given path.
* Unlike deleteFileBySource(), this does NOT touch the file on disk — used

57
admin/app/services/zim_service.ts
View File

@ -25,6 +25,7 @@ import vine from '@vinejs/vine'
import { wikipediaOptionsFileSchema } from '#validators/curated_collections'
import WikipediaSelection from '#models/wikipedia_selection'
import InstalledResource from '#models/installed_resource'
import KbIngestState from '#models/kb_ingest_state'
import CollectionManifest from '#models/collection_manifest'
import { RunDownloadJob } from '#jobs/run_download_job'
import { SERVICE_NAMES } from '../../constants/service_names.js'
@ -48,16 +49,34 @@ export class ZimService {
const all = await listDirectoryContents(dirPath)
const zimEntries = all.filter((item) => item.name.endsWith('.zim'))
// Resolve each entry's file path up front so the kb_ingest_state lookup
// can be a single batched query instead of one query per file (N+1) --
// matters once there are dozens/hundreds of ZIMs (see #followup-proposal).
const filePaths = zimEntries.map((entry) =>
entry.type === 'file' ? entry.key : join(dirPath, entry.name)
)
const kbStates = filePaths.length
? await KbIngestState.query().whereIn('file_path', filePaths)
: []
const kbStateByPath = new Map(kbStates.map((row) => [row.file_path, row]))
const files = await Promise.all(
zimEntries.map(async (entry) => {
const filePath = entry.type === 'file' ? entry.key : join(dirPath, entry.name)
zimEntries.map(async (entry, i) => {
const filePath = filePaths[i]
const stats = await getFileStatsIfExists(filePath)
const kbState = kbStateByPath.get(filePath)
const kb_status: 'not_added' | 'active' | 'inactive' = !kbState
? 'not_added'
: kbState.active
? 'active'
: 'inactive'
return {
...entry,
title: null,
summary: null,
author: null,
size_bytes: stats ? Number(stats.size) : null,
kb_status,
}
})
)
@ -552,18 +571,11 @@ export class ZimService {
logger.error(`[ZimService] Failed to update WikipediaSelection for ${filename}:`, error)
}
const ollamaUrl = await this.dockerService.getServiceURL('nomad_ollama')
if (ollamaUrl) {
try {
const { EmbedFileJob } = await import('#jobs/embed_file_job')
await EmbedFileJob.dispatch({
fileName: filename,
filePath: join(process.cwd(), ZIM_STORAGE_PATH, filename),
})
} catch (error) {
logger.error(`[ZimService] EmbedFileJob dispatch failed after local upload:`, error)
}
}
// NOTE: uploaded ZIM files are intentionally NOT auto-embedded into the
// knowledge base. ZIM/Kiwix content must be explicitly opted in via
// RagService.embedSingleFile() (e.g. the Content Manager's "Add to
// Knowledge Base" action) so browsing content in Kiwix never implies
// consent to include it in AI search.
return { added }
}
@ -605,6 +617,23 @@ export class ZimService {
logger.info(`[ZimService] Deleted InstalledResource entry for: ${parsed.resource_id}`)
}
// If this ZIM was indexed into the AI knowledge base, clean up its Qdrant
// points + kb_ingest_state row too -- otherwise a hard delete here leaves
// an orphaned "Indexed" entry behind that never gets cleaned up until a
// full Reset & Rebuild.
try {
const kbRow = await KbIngestState.query().where('file_path', fullPath).first()
if (kbRow) {
const appModule = await import('@adonisjs/core/services/app')
const { RagService } = await import('#services/rag_service')
const ragService = await appModule.default.container.make(RagService)
await ragService.deleteFileBySource(fullPath)
logger.info(`[ZimService] Cleaned up knowledge-base entry for deleted ZIM: ${fileName}`)
}
} catch (error) {
logger.error(`[ZimService] Failed to clean up knowledge-base entry for ${fileName}:`, error)
}
// If this file was the active Wikipedia selection, clear the selection
try {
const selection = await WikipediaSelection.query().first()

7
admin/app/validators/rag.ts
View File

@ -19,6 +19,13 @@ export const embedFileSchema = vine.compile(
})
)
export const toggleActiveSchema = vine.compile(
vine.object({
source: vine.string().minLength(1),
active: vine.boolean(),
})
)
export const fileSourceSchema = vine.compile(
vine.object({
source: vine.string().minLength(1),

17
admin/database/migrations/1784300872597_add_active_to_kb_ingest_state.ts Normal file
View File

@ -0,0 +1,17 @@
import { BaseSchema } from '@adonisjs/lucid/schema'
export default class extends BaseSchema {
protected tableName = 'kb_ingest_state'
async up() {
this.schema.alterTable(this.tableName, (table) => {
table.boolean('active').notNullable().defaultTo(true)
})
}
async down() {
this.schema.alterTable(this.tableName, (table) => {
table.dropColumn('active')
})
}
}

7
admin/inertia/lib/api.ts
View File

@ -507,6 +507,13 @@ class API {
})()
}
async toggleRagFileActive(source: string, active: boolean) {
return catchInternal(async () => {
const response = await this.client.post<{ message: string }>('/rag/files/toggle-active', { source, active })
return response.data
})()
}
async getKbFileWarnings() {
return catchInternal(async () => {
const response = await this.client.get<FileWarningsResult>('/rag/file-warnings')

62
admin/inertia/pages/settings/zim/index.tsx
View File

@ -11,6 +11,7 @@ import useServiceInstalledStatus from '~/hooks/useServiceInstalledStatus'
import Alert from '~/components/Alert'
import { useNotifications } from '~/context/NotificationContext'
import ZimUploader from '~/components/ZimUploader'
import Switch from '~/components/inputs/Switch'
import { ZimFileWithMetadata } from '../../../../types/zim'
import { SERVICE_NAMES } from '../../../../constants/service_names'
import { formatBytes } from '~/lib/util'
@ -126,6 +127,34 @@ export default function ZimPage() {
},
})
const embedMutation = useMutation({
mutationFn: async (file: ZimFileWithMetadata) =>
api.embedSingleRAGFile(file.type === 'file' ? file.key : file.name),
onSuccess: (result) => {
if (!result) return
queryClient.invalidateQueries({ queryKey: ['zim-files'] })
addNotification({
type: 'success',
message: result.message || 'Added to the knowledge base. It will be searchable once embedding finishes.',
})
},
})
const toggleActiveMutation = useMutation({
mutationFn: async ({ file, active }: { file: ZimFileWithMetadata; active: boolean }) =>
api.toggleRagFileActive(file.type === 'file' ? file.key : file.name, active),
onSuccess: (result, { active }) => {
if (!result) return
queryClient.invalidateQueries({ queryKey: ['zim-files'] })
addNotification({
type: 'success',
message: active
? 'Re-included in AI search.'
: 'Excluded from AI search. It stays indexed, so re-enabling is instant.',
})
},
})
return (
<SettingsLayout>
<Head title="Content Manager | Project NOMAD" />
@ -223,6 +252,39 @@ export default function ZimPage() {
</span>
),
},
{
accessor: 'kb_status',
title: 'Knowledge Base',
render: (record) => {
if (record.kb_status === 'not_added') {
const pending =
embedMutation.isPending && embedMutation.variables?.name === record.name
return (
<StyledButton
variant="secondary"
size="sm"
icon="IconPlus"
loading={pending}
onClick={() => embedMutation.mutate(record)}
>
Add to Knowledge Base
</StyledButton>
)
}
const isActive = record.kb_status === 'active'
const pending =
toggleActiveMutation.isPending &&
toggleActiveMutation.variables?.file.name === record.name
return (
<Switch
checked={isActive}
onChange={(checked) => toggleActiveMutation.mutate({ file: record, active: checked })}
disabled={pending}
label={isActive ? 'Active' : 'Inactive'}
/>
)
},
},
{
accessor: 'actions',
title: 'Actions',

1
admin/start/routes.ts
View File

@ -149,6 +149,7 @@ router
router.get('/file-warnings', [RagController, 'getFileWarnings'])
router.delete('/files', [RagController, 'deleteFile'])
router.post('/files/embed', [RagController, 'embedFile'])
router.post('/files/toggle-active', [RagController, 'toggleActive'])
router.get('/files/content', [RagController, 'getFileContent'])
router.get('/files/download', [RagController, 'downloadFile'])
router.get('/active-jobs', [RagController, 'getActiveJobs'])

3
admin/types/zim.ts
View File

@ -1,10 +1,13 @@
import { FileEntry } from './files.js'
export type ZimKbStatus = 'not_added' | 'active' | 'inactive'
export type ZimFileWithMetadata = FileEntry & {
title: string | null
summary: string | null
author: string | null
size_bytes: number | null
kb_status: ZimKbStatus
}
export type ListZimFilesResponse = {