fix(rag): stop re-creating payload indexes on every embedded document (#1135)

_ensureCollection() runs once per document on the embed path, but only
createCollection sat behind the collectionExists guard — the
getCollections probe and the three createPayloadIndex calls fired
unconditionally every time. On large ZIM ingestions those redundant
requests consumed roughly 45% of per-document Qdrant time, making jobs
look stalled while they were slowly progressing.

Memoize ensured collections in a per-instance Set, recorded only after
every step succeeds so partial failures retry. The cache is cleared
when the Qdrant health check resets the client (server may have been
recreated), and the entry is dropped before resetAndRebuild()
recreates the collection it just deleted.

Memoizing instead of moving the index calls inside the guard keeps
missing indexes healing on collections that predate the current
payload schema.

Closes #1129

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

admin/app/services/rag_service.ts

@@ -45,6 +45,10 @@ export class RagService {
   private qdrantInitPromise: Promise<void> | null = null
   private embeddingModelVerified = false
   private resolvedEmbeddingModel: string | null = null
+  // Collections already verified this session (created + payload indexes in place).
+  // Skips the getCollections/createPayloadIndex round-trips that otherwise run on
+  // every embed call — ~45% of per-document Qdrant time on large ingestions (#1129)
+  private ensuredCollections = new Set<string>()
   public static UPLOADS_STORAGE_PATH = 'storage/kb_uploads'
   public static CONTENT_COLLECTION_NAME = 'nomad_knowledge_base'
   public static EMBEDDING_DIMENSION = 768 // Nomic Embed Text v1.5 dimension is 768
@@ -94,6 +98,8 @@ export class RagService {
     } catch {
       this.qdrant = null
       this.qdrantInitPromise = null
+      // Qdrant may have restarted (or been recreated) — re-verify collections on reconnect
+      this.ensuredCollections.clear()
       return {
         online: false,
         message: 'Qdrant vector database is offline. Restart the AI Assistant service in Settings to restore the Knowledge Base.',
@@ -113,6 +119,11 @@ export class RagService {
   ) {
     try {
       await this._ensureDependencies()
+
+      if (this.ensuredCollections.has(collectionName)) {
+        return
+      }
+
       const collections = await this.qdrant!.getCollections()
       const collectionExists = collections.collections.some((col) => col.name === collectionName)
 
@@ -138,6 +149,9 @@ export class RagService {
         field_name: 'collection',
         field_schema: 'keyword',
       })
+
+      // Only memoize after every step succeeded, so a partial failure is retried
+      this.ensuredCollections.add(collectionName)
     } catch (error) {
       logger.error('Error ensuring Qdrant collection:', error)
       throw error
@@ -2104,6 +2118,10 @@ export class RagService {
         logger.warn(`[RAG] deleteCollection failed (may not exist): ${(err as Error).message}`)
       }
 
+      // The collection is gone — drop it from the ensured cache so the
+      // _ensureCollection call below actually recreates it
+      this.ensuredCollections.delete(RagService.CONTENT_COLLECTION_NAME)
+
       await this._ensureCollection(
         RagService.CONTENT_COLLECTION_NAME,
         RagService.EMBEDDING_DIMENSION