Merge remote-tracking branch 'origin/feat/kb-policy-toggle' into integration/v1.32.0-rfc883

admin/app/services/rag_service.ts

@@ -17,7 +17,7 @@ import { randomUUID } from 'node:crypto'
 import { join, resolve, sep } from 'node:path'
 import KVStore from '#models/kv_store'
 import KbIngestState from '#models/kb_ingest_state'
-import { decideScanAction } from '../utils/kb_ingest_decision.js'
+import { decideScanAction, type IngestPolicy } from '../utils/kb_ingest_decision.js'
 import { ZIMExtractionService } from './zim_extraction_service.js'
 import { ZIM_BATCH_SIZE } from '../../constants/zim_extraction.js'
 import { ProcessAndEmbedFileResponse, ProcessZIMFileResponse, RAGResult, RerankedRAGResult } from '../../types/rag.js'
@@ -1296,14 +1296,21 @@ export class RagService {
         (filePath) => determineFileType(filePath) !== 'unknown'
       )
 
+      // Read the global ingest policy. Unset is treated as 'Always' so legacy
+      // installs keep their current behavior until the user explicitly opts
+      // into Manual mode from the KB panel.
+      const policyRaw = await KVStore.getValue('rag.defaultIngestPolicy')
+      const policy: IngestPolicy = policyRaw === 'Manual' ? 'Manual' : 'Always'
+
       const filesToEmbed: string[] = []
       let backfilled = 0
       let createdRows = 0
+      let createdPending = 0
       let skipped = 0
 
       for (const filePath of embeddableFiles) {
         const stateRow = stateByPath.get(filePath) ?? null
-        const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath))
+        const action = decideScanAction(stateRow, sourcesInQdrant.has(filePath), policy)
 
         switch (action.kind) {
           case 'skip':
@@ -1321,6 +1328,16 @@ export class RagService {
             })
             backfilled++
             break
+          case 'create_pending':
+            // Manual mode: record that we've seen the file but don't dispatch.
+            // The KB panel surfaces a per-card "Index" affordance for these.
+            await KbIngestState.create({
+              file_path: filePath,
+              state: 'pending_decision',
+              chunks_embedded: 0,
+            })
+            createdPending++
+            break
           case 'dispatch':
             if (action.createStateRow) {
               await KbIngestState.create({
@@ -1336,7 +1353,7 @@ export class RagService {
       }
 
       logger.info(
-        `[RAG] Scan results: ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${skipped} skipped`
+        `[RAG] Scan results (policy=${policy}): ${filesToEmbed.length} to embed, ${backfilled} backfilled, ${createdRows} new pending, ${createdPending} waiting on user, ${skipped} skipped`
       )
 
       if (filesToEmbed.length === 0) {

admin/app/utils/kb_ingest_decision.ts

@@ -2,8 +2,8 @@ import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
 
 /**
  * Decision returned by `decideScanAction` describing what scanAndSyncStorage
- * should do for one file given its current state row (if any) and whether
+ * should do for one file given its current state row (if any), whether Qdrant
- * Qdrant already has chunks for it.
+ * already has chunks for it, and the global ingest policy.
  *
  * - `skip` — file is in a settled state (already indexed, deliberately not
  *   indexed, or in a manual-recovery state); no auto-dispatch.
@@ -13,16 +13,26 @@ import type { KbIngestStateValue } from '../../types/kb_ingest_state.js'
  * - `backfill_indexed` — Qdrant has chunks but no state row exists yet
  *   (pre-RFC install, or new admin instance pointed at an existing Qdrant
  *   volume). Create a row in `indexed` state without re-embedding.
+ * - `create_pending` — Manual mode: record that we've seen the file but
+ *   don't dispatch. Frontend surfaces a per-card "Index" affordance.
  */
 export type ScanAction =
   | { kind: 'skip' }
   | { kind: 'dispatch'; createStateRow: boolean }
   | { kind: 'backfill_indexed' }
+  | { kind: 'create_pending' }
 
 export interface KbIngestStateRow {
   state: KbIngestStateValue
 }
 
+/**
+ * Global auto-index policy stored at KV `rag.defaultIngestPolicy`. Unset is
+ * treated as `Always` so existing installs keep their current behavior until
+ * the user opts into Manual mode through the KB panel.
+ */
+export type IngestPolicy = 'Always' | 'Manual'
+
 /**
  * Decide what scanAndSyncStorage should do for a single embeddable file.
  *
@@ -33,18 +43,25 @@ export interface KbIngestStateRow {
  */
 export function decideScanAction(
   stateRow: KbIngestStateRow | null,
-  hasChunksInQdrant: boolean
+  hasChunksInQdrant: boolean,
+  policy: IngestPolicy = 'Always'
 ): ScanAction {
   if (!stateRow) {
     if (hasChunksInQdrant) return { kind: 'backfill_indexed' }
-    return { kind: 'dispatch', createStateRow: true }
+    return policy === 'Always'
+      ? { kind: 'dispatch', createStateRow: true }
+      : { kind: 'create_pending' }
   }
 
   switch (stateRow.state) {
     case 'indexed':
       return hasChunksInQdrant ? { kind: 'skip' } : { kind: 'dispatch', createStateRow: false }
     case 'pending_decision':
-      return { kind: 'dispatch', createStateRow: false }
+      // Manual mode: file is waiting for the user to opt in via per-card Index.
+      // Always mode: treat as "user-equivalent of auto-index" and dispatch.
+      return policy === 'Always'
+        ? { kind: 'dispatch', createStateRow: false }
+        : { kind: 'skip' }
     case 'browse_only':
     case 'failed':
     case 'stalled':

admin/constants/kv_store.ts

@@ -1,3 +1,3 @@
 import { KVStoreKey } from "../types/kv_store.js";
 
-export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention'];
+export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy'];

admin/inertia/components/chat/KnowledgeBaseModal.tsx

@@ -49,6 +49,37 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
     select: (data) => data || [],
   })
 
+  // Global auto-index policy. KVStore returns `null` for an unset key, which
+  // we treat as 'Always' for backward compatibility with installs that predate
+  // this UI. The user can opt into Manual mode from the toggle below.
+  const { data: ingestPolicySetting } = useQuery({
+    queryKey: ['ingestPolicy'],
+    queryFn: () => api.getSetting('rag.defaultIngestPolicy'),
+  })
+  const ingestPolicy: 'Always' | 'Manual' =
+    ingestPolicySetting?.value === 'Manual' ? 'Manual' : 'Always'
+
+  const updateIngestPolicyMutation = useMutation({
+    mutationFn: (policy: 'Always' | 'Manual') =>
+      api.updateSetting('rag.defaultIngestPolicy', policy),
+    onSuccess: (_data, policy) => {
+      queryClient.invalidateQueries({ queryKey: ['ingestPolicy'] })
+      addNotification({
+        type: 'success',
+        message:
+          policy === 'Always'
+            ? 'New content will be auto-indexed for AI.'
+            : 'New content will wait for you to opt in.',
+      })
+    },
+    onError: (error: any) => {
+      addNotification({
+        type: 'error',
+        message: error?.message || 'Failed to update indexing policy.',
+      })
+    },
+  })
+
   const uploadMutation = useMutation({
     mutationFn: (file: File) => api.uploadDocument(file),
   })
@@ -267,6 +298,48 @@ export default function KnowledgeBaseModal({ aiAssistantName = "AI Assistant", o
               </div>
             </div>
           </div>
+          <div className="my-8 p-4 rounded-lg border border-border-subtle bg-surface-secondary">
+            <div className="flex flex-wrap items-center justify-between gap-3">
+              <div className="flex-1 min-w-[14rem]">
+                <p className="text-sm font-medium text-text-primary">
+                  Auto-index new content for AI?
+                </p>
+                <p className="text-xs text-text-muted mt-1">
+                  Indexed content typically uses 5–10× the original file size on disk.
+                  Changes apply to new content added after this setting changes.
+                </p>
+              </div>
+              <div
+                role="radiogroup"
+                aria-label="Ingest policy"
+                className="inline-flex rounded-md overflow-hidden border border-border-subtle"
+              >
+                {(['Always', 'Manual'] as const).map((option) => {
+                  const isActive = ingestPolicy === option
+                  return (
+                    <button
+                      key={option}
+                      type="button"
+                      role="radio"
+                      aria-checked={isActive}
+                      onClick={() =>
+                        !isActive && updateIngestPolicyMutation.mutate(option)
+                      }
+                      disabled={updateIngestPolicyMutation.isPending}
+                      className={`px-4 py-2 text-sm font-medium transition-colors ${
+                        isActive
+                          ? 'bg-desert-green text-white'
+                          : 'bg-surface-primary text-text-secondary hover:bg-surface-tertiary'
+                      } ${updateIngestPolicyMutation.isPending ? 'opacity-50 cursor-not-allowed' : ''}`}
+                    >
+                      {option}
+                    </button>
+                  )
+                })}
+              </div>
+            </div>
+          </div>
+
           <div className="my-8">
             <div className="flex items-center justify-between mb-4">
               <StyledSectionHeader title="Processing Queue" className="!mb-0" />

admin/tests/unit/kb_ingest_decision.spec.ts

@@ -3,44 +3,80 @@ import { test } from 'node:test'
 
 import { decideScanAction } from '../../app/utils/kb_ingest_decision.js'
 
-test('no state row, no chunks → dispatch and create row (new file)', () => {
+// ---------- Always-policy cases (default behavior; preserves pre-policy install) ----------
+
+test('Always: no state row, no chunks → dispatch and create row (new file)', () => {
+  assert.deepEqual(decideScanAction(null, false, 'Always'), {
+    kind: 'dispatch',
+    createStateRow: true,
+  })
+})
+
+test('Always: no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => {
+  assert.deepEqual(decideScanAction(null, true, 'Always'), { kind: 'backfill_indexed' })
+})
+
+test('Always: indexed + chunks present → skip', () => {
+  assert.deepEqual(decideScanAction({ state: 'indexed' }, true, 'Always'), { kind: 'skip' })
+})
+
+test('Always: indexed + chunks missing → re-dispatch (Qdrant collection rebuilt or chunks deleted)', () => {
+  assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Always'), {
+    kind: 'dispatch',
+    createStateRow: false,
+  })
+})
+
+test('Always: pending_decision → dispatch', () => {
+  assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Always'), {
+    kind: 'dispatch',
+    createStateRow: false,
+  })
+})
+
+test('Always: browse_only → skip (user opted out of indexing)', () => {
+  assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Always'), { kind: 'skip' })
+})
+
+test('Always: failed → skip (manual retry needed, do not auto-redispatch)', () => {
+  assert.deepEqual(decideScanAction({ state: 'failed' }, false, 'Always'), { kind: 'skip' })
+})
+
+test('Always: stalled → skip (manual retry needed)', () => {
+  assert.deepEqual(decideScanAction({ state: 'stalled' }, false, 'Always'), { kind: 'skip' })
+})
+
+// ---------- Manual-policy cases ----------
+
+test('Manual: no state row, no chunks → create_pending (do not auto-dispatch new content)', () => {
+  assert.deepEqual(decideScanAction(null, false, 'Manual'), { kind: 'create_pending' })
+})
+
+test('Manual: no state row, chunks present → backfill_indexed (same as Always — Qdrant is authoritative)', () => {
+  assert.deepEqual(decideScanAction(null, true, 'Manual'), { kind: 'backfill_indexed' })
+})
+
+test('Manual: pending_decision → skip (waiting for user to opt in via Index button)', () => {
+  assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false, 'Manual'), {
+    kind: 'skip',
+  })
+})
+
+test('Manual: indexed + chunks missing → re-dispatch (user has already opted in for this file)', () => {
+  // Policy switch from Always→Manual must not break in-flight or partially-deleted indexes
+  // for files the user previously chose to index.
+  assert.deepEqual(decideScanAction({ state: 'indexed' }, false, 'Manual'), {
+    kind: 'dispatch',
+    createStateRow: false,
+  })
+})
+
+test('Manual: browse_only → skip (same as Always)', () => {
+  assert.deepEqual(decideScanAction({ state: 'browse_only' }, false, 'Manual'), { kind: 'skip' })
+})
+
+// ---------- Policy default ----------
+
+test('omitted policy defaults to Always (unset KV preserves legacy behavior)', () => {
   assert.deepEqual(decideScanAction(null, false), { kind: 'dispatch', createStateRow: true })
 })
-
-test('no state row, chunks present → backfill_indexed (pre-RFC install, existing Qdrant volume)', () => {
-  assert.deepEqual(decideScanAction(null, true), { kind: 'backfill_indexed' })
-})
-
-test('indexed + chunks present → skip', () => {
-  assert.deepEqual(decideScanAction({ state: 'indexed' }, true), { kind: 'skip' })
-})
-
-test('indexed + chunks missing → re-dispatch (state stale, Qdrant collection rebuilt or chunks deleted)', () => {
-  assert.deepEqual(decideScanAction({ state: 'indexed' }, false), {
-    kind: 'dispatch',
-    createStateRow: false,
-  })
-})
-
-test('pending_decision → dispatch (preserves current Always behavior until policy is consumed)', () => {
-  assert.deepEqual(decideScanAction({ state: 'pending_decision' }, false), {
-    kind: 'dispatch',
-    createStateRow: false,
-  })
-})
-
-test('browse_only → skip (user opted out of indexing)', () => {
-  assert.deepEqual(decideScanAction({ state: 'browse_only' }, false), { kind: 'skip' })
-})
-
-test('browse_only + chunks present → skip (do not silently re-index after un-index)', () => {
-  assert.deepEqual(decideScanAction({ state: 'browse_only' }, true), { kind: 'skip' })
-})
-
-test('failed → skip (manual retry needed, do not auto-redispatch)', () => {
-  assert.deepEqual(decideScanAction({ state: 'failed' }, false), { kind: 'skip' })
-})
-
-test('stalled → skip (manual retry needed)', () => {
-  assert.deepEqual(decideScanAction({ state: 'stalled' }, false), { kind: 'skip' })
-})