Merge remote-tracking branch 'origin/feat/kb-policy-toggle' into integration/v1.32.0-rfc883
This commit is contained in:
Chris Sherwood
commit
4237b237d5
|
|
@ -17,7 +17,7 @@ import { randomUUID } from 'node:crypto'
|
|||
import { join, resolve, sep } from 'node:path'
|
||||
import KVStore from '#models/kv_store'
|
||||
import KbIngestState from '#models/kb_ingest_state'
|
||||
import { decideScanAction } from '../utils/kb_ingest_decision.js'
|
||||
import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js'
|
||||
import { ZIMExtractionService } from './zim_extraction_service.js'
|
||||
import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
|
||||
import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js'
|
||||
|
|
@ -1296,14 +1296,21 @@ export class RagService {
|
|||
(filePath) => determineFileType(filePath) !== 'unknown'
|
||||
)
|
||||
|
||||
// Read the global ingest policy. Unset is treated as 'Always' so legacy
|
||||
// installs keep their current behavior until the user explicitly opts
|
||||
// into Manual mode from the KB panel.
|
||||
const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy')
|
||||
const policy: IngestPolicy = policyRaw === 'Manual' ? 'Manual' : 'Always'
|
||||
|
||||
const filesToEmbed: string[] = []
|
||||
let backfilled = 0
|
||||
let createdRows = 0
|
||||
let createdPending = 0
|
||||
let skipped = 0
|
||||
|
||||
for (const filePath of embeddableFiles) {
|
||||
const stateRow = stateByPath.get(filePath) ?? null
|
||||
const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath))
|
||||
const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath), policy)
|
||||
|
||||
switch (action.kind) {
|
||||
case 'skip':
|
||||
|
|
@ -1321,6 +1328,16 @@ export class RagService {
|
|||
})
|
||||
backfilled++
|
||||
break
|
||||
case 'create_pending':
|
||||
// Manual mode: record that we've seen the file but don't dispatch.
|
||||
// The KB panel surfaces a per-card "Index" affordance for these.
|
||||
await KbIngestState.create({
|
||||
file_path: filePath,
|
||||
state: 'pending_decision',
|
||||
chunks_embedded: 0,
|
||||
})
|
||||
createdPending++
|
||||
break
|
||||
case 'dispatch':
|
||||
if (action.createStateRow) {
|
||||
await KbIngestState.create({
|
||||
|
|
@ -1336,7 +1353,7 @@ export class RagService {
|
|||
}
|
||||
|
||||
logger.info(
|
||||
`[RAG] Scan results: ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${skipped} skipped`
|
||||
`[RAG] Scan results (policy=${policy}): ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${createdPending} waiting on user, ${skipped} skipped`
|
||||
)
|
||||
|
||||
if (filesToEmbed.length === 0) {
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
|
|||
|
||||
/**
|
||||
* Decision returned by `decideScanAction` describing what scanAndSyncStorage
|
||||
* should do for one file given its current state row (if any) and whether
|
||||
* Qdrant already has chunks for it.
|
||||
* should do for one file given its current state row (if any), whether Qdrant
|
||||
* already has chunks for it, and the global ingest policy.
|
||||
*
|
||||
* - `skip` — file is in a settled state (already indexed, deliberately not
|
||||
* indexed, or in a manual-recovery state); no auto-dispatch.
|
||||
|
|
@ -13,16 +13,26 @@ import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
|
|||
* - `backfill_indexed` — Qdrant has chunks but no state row exists yet
|
||||
* (pre-RFC install, or new admin instance pointed at an existing Qdrant
|
||||
* volume). Create a row in `indexed` state without re-embedding.
|
||||
* - `create_pending` — Manual mode: record that we've seen the file but
|
||||
* don't dispatch. Frontend surfaces a per-card "Index" affordance.
|
||||
*/
|
||||
export type ScanAction =
|
||||
| { kind: 'skip' }
|
||||
| { kind: 'dispatch'; createStateRow: boolean }
|
||||
| { kind: 'backfill_indexed' }
|
||||
| { kind: 'create_pending' }
|
||||
|
||||
export interface KbIngestStateRow {
|
||||
state: KbIngestStateValue
|
||||
}
|
||||
|
||||
/**
|
||||
* Global auto-index policy stored at KV `rag.defaultIngestPolicy`. Unset is
|
||||
* treated as `Always` so existing installs keep their current behavior until
|
||||
* the user opts into Manual mode through the KB panel.
|
||||
*/
|
||||
export type IngestPolicy = 'Always' | 'Manual'
|
||||
|
||||
/**
|
||||
* Decide what scanAndSyncStorage should do for a single embeddable file.
|
||||
*
|
||||
|
|
@ -33,18 +43,25 @@ export interface KbIngestStateRow {
|
|||
*/
|
||||
export function decideScanAction(
|
||||
stateRow: KbIngestStateRow | null,
|
||||
hasChunksInQdrant: boolean
|
||||
hasChunksInQdrant: boolean,
|
||||
policy: IngestPolicy = 'Always'
|
||||
): ScanAction {
|
||||
if (!stateRow) {
|
||||
if (hasChunksInQdrant) return { kind: 'backfill_indexed' }
|
||||
return { kind: 'dispatch', createStateRow: true }
|
||||
return policy === 'Always'
|
||||
? { kind: 'dispatch', createStateRow: true }
|
||||
: { kind: 'create_pending' }
|
||||
}
|
||||
|
||||
switch (stateRow.state) {
|
||||
case 'indexed':
|
||||
return hasChunksInQdrant ? { kind: 'skip' } : { kind: 'dispatch', createStateRow: false }
|
||||
case 'pending_decision':
|
||||
return { kind: 'dispatch', createStateRow: false }
|
||||
// Manual mode: file is waiting for the user to opt in via per-card Index.
|
||||
// Always mode: treat as "user-equivalent of auto-index" and dispatch.
|
||||
return policy === 'Always'
|
||||
? { kind: 'dispatch', createStateRow: false }
|
||||
: { kind: 'skip' }
|
||||
case 'browse_only':
|
||||
case 'failed':
|
||||
case 'stalled':
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
import { KVStoreKey } from "../types/kv_store.js";
|
||||
|
||||
export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention'];
|
||||
export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy'];
|
||||
|
|
@ -49,6 +49,37 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
select: (data) => data || [],
|
||||
})
|
||||
|
||||
// Global auto-index policy. KVStore returns `null` for an unset key, which
|
||||
// we treat as 'Always' for backward compatibility with installs that predate
|
||||
// this UI. The user can opt into Manual mode from the toggle below.
|
||||
const { data: ingestPolicySetting } = useQuery({
|
||||
queryKey: ['ingestPolicy'],
|
||||
queryFn: () => api.getSetting('rag.defaultIngestPolicy'),
|
||||
})
|
||||
const ingestPolicy: 'Always' | 'Manual' =
|
||||
ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always'
|
||||
|
||||
const updateIngestPolicyMutation = useMutation({
|
||||
mutationFn: (policy: 'Always' | 'Manual') =>
|
||||
api.updateSetting('rag.defaultIngestPolicy', policy),
|
||||
onSuccess: (_data, policy) => {
|
||||
queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] })
|
||||
addNotification({
|
||||
type: 'success',
|
||||
message:
|
||||
policy === 'Always'
|
||||
? 'New content will be auto-indexed for AI.'
|
||||
: 'New content will wait for you to opt in.',
|
||||
})
|
||||
},
|
||||
onError: (error: any) => {
|
||||
addNotification({
|
||||
type: 'error',
|
||||
message: error?.message || 'Failed to update indexing policy.',
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
const uploadMutation = useMutation({
|
||||
mutationFn: (file: File) => api.uploadDocument(file),
|
||||
})
|
||||
|
|
@ -267,6 +298,48 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="my-8 p-4 rounded-lg border border-border-subtle bg-surface-secondary">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<div className="flex-1 min-w-[14rem]">
|
||||
<p className="text-sm font-medium text-text-primary">
|
||||
Auto-index new content for AI?
|
||||
</p>
|
||||
<p className="text-xs text-text-muted mt-1">
|
||||
Indexed content typically uses 5–10× the original file size on disk.
|
||||
Changes apply to new content added after this setting changes.
|
||||
</p>
|
||||
</div>
|
||||
<div
|
||||
role="radiogroup"
|
||||
aria-label="Ingest policy"
|
||||
className="inline-flex rounded-md overflow-hidden border border-border-subtle"
|
||||
>
|
||||
{(['Always', 'Manual'] as const).map((option) => {
|
||||
const isActive = ingestPolicy === option
|
||||
return (
|
||||
<button
|
||||
key={option}
|
||||
type="button"
|
||||
role="radio"
|
||||
aria-checked={isActive}
|
||||
onClick={() =>
|
||||
!isActive && updateIngestPolicyMutation.mutate(option)
|
||||
}
|
||||
disabled={updateIngestPolicyMutation.isPending}
|
||||
className={`px-4 py-2 text-sm font-medium transition-colors ${
|
||||
isActive
|
||||
? 'bg-desert-green text-white'
|
||||
: 'bg-surface-primary text-text-secondary hover:bg-surface-tertiary'
|
||||
} ${updateIngestPolicyMutation.isPending ? 'opacity-50 cursor-not-allowed' : ''}`}
|
||||
>
|
||||
{option}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="my-8">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<StyledSectionHeader title="Processing Queue" className="!mb-0" />
|
||||
|
|
|
|||
|
|
@ -3,44 +3,80 @@ import { test } from 'node:test'
|
|||
|
||||
import { decideScanAction } from '../../app/utils/kb_ingest_decision.js'
|
||||
|
||||
test('no state row, no chunks → dispatch and create row (new file)', () => {
|
||||
// ---------- Always-policy cases (default behavior; preserves pre-policy install) ----------
|
||||
|
||||
test('Always: no state row, no chunks → dispatch and create row (new file)', () => {
|
||||
assert.deepEqual(decideScanAction(null, false, 'Always'), {
|
||||
kind: 'dispatch',
|
||||
createStateRow: true,
|
||||
})
|
||||
})
|
||||
|
||||
test('Always: no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => {
|
||||
assert.deepEqual(decideScanAction(null, true, 'Always'), { kind: 'backfill_indexed' })
|
||||
})
|
||||
|
||||
test('Always: indexed + chunks present → skip', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'indexed' }, true, 'Always'), { kind: 'skip' })
|
||||
})
|
||||
|
||||
test('Always: indexed + chunks missing → re-dispatch (Qdrant collection rebuilt or chunks deleted)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Always'), {
|
||||
kind: 'dispatch',
|
||||
createStateRow: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('Always: pending_decision → dispatch', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Always'), {
|
||||
kind: 'dispatch',
|
||||
createStateRow: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('Always: browse_only → skip (user opted out of indexing)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Always'), { kind: 'skip' })
|
||||
})
|
||||
|
||||
test('Always: failed → skip (manual retry needed, do not auto-redispatch)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'failed' }, false, 'Always'), { kind: 'skip' })
|
||||
})
|
||||
|
||||
test('Always: stalled → skip (manual retry needed)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'stalled' }, false, 'Always'), { kind: 'skip' })
|
||||
})
|
||||
|
||||
// ---------- Manual-policy cases ----------
|
||||
|
||||
test('Manual: no state row, no chunks → create_pending (do not auto-dispatch new content)', () => {
|
||||
assert.deepEqual(decideScanAction(null, false, 'Manual'), { kind: 'create_pending' })
|
||||
})
|
||||
|
||||
test('Manual: no state row, chunks present → backfill_indexed (same as Always — Qdrant is authoritative)', () => {
|
||||
assert.deepEqual(decideScanAction(null, true, 'Manual'), { kind: 'backfill_indexed' })
|
||||
})
|
||||
|
||||
test('Manual: pending_decision → skip (waiting for user to opt in via Index button)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Manual'), {
|
||||
kind: 'skip',
|
||||
})
|
||||
})
|
||||
|
||||
test('Manual: indexed + chunks missing → re-dispatch (user has already opted in for this file)', () => {
|
||||
// Policy switch from Always→Manual must not break in-flight or partially-deleted indexes
|
||||
// for files the user previously chose to index.
|
||||
assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Manual'), {
|
||||
kind: 'dispatch',
|
||||
createStateRow: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('Manual: browse_only → skip (same as Always)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Manual'), { kind: 'skip' })
|
||||
})
|
||||
|
||||
// ---------- Policy default ----------
|
||||
|
||||
test('omitted policy defaults to Always (unset KV preserves legacy behavior)', () => {
|
||||
assert.deepEqual(decideScanAction(null, false), { kind: 'dispatch', createStateRow: true })
|
||||
})
|
||||
|
||||
test('no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => {
|
||||
assert.deepEqual(decideScanAction(null, true), { kind: 'backfill_indexed' })
|
||||
})
|
||||
|
||||
test('indexed + chunks present → skip', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'indexed' }, true), { kind: 'skip' })
|
||||
})
|
||||
|
||||
test('indexed + chunks missing → re-dispatch (state stale, Qdrant collection rebuilt or chunks deleted)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'indexed' }, false), {
|
||||
kind: 'dispatch',
|
||||
createStateRow: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('pending_decision → dispatch (preserves current Always behavior until policy is consumed)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false), {
|
||||
kind: 'dispatch',
|
||||
createStateRow: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('browse_only → skip (user opted out of indexing)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'browse_only' }, false), { kind: 'skip' })
|
||||
})
|
||||
|
||||
test('browse_only + chunks present → skip (do not silently re-index after un-index)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'browse_only' }, true), { kind: 'skip' })
|
||||
})
|
||||
|
||||
test('failed → skip (manual retry needed, do not auto-redispatch)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'failed' }, false), { kind: 'skip' })
|
||||
})
|
||||
|
||||
test('stalled → skip (manual retry needed)', () => {
|
||||
assert.deepEqual(decideScanAction({ state: 'stalled' }, false), { kind: 'skip' })
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue