From a22c6408e639948b1cddf0eae040f8d0d5fdae16 Mon Sep 17 00:00:00 2001
From: Chris Sherwood <chris@crosstalksolutions.com>
Date: Tue, 12 May 2026 20:35:57 -0700
Subject: [PATCH] fix(RAG): pace continuation batches when embedding is
 CPU-only
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Stacks on top of the multi-batch ZIM ingestion fix. After that fix,
multi-batch ZIM ingestion completes correctly — but on installs where
Ollama runs the embedding model on CPU (currently every AMD ROCm
install, since Ollama's ROCm build doesn't accelerate nomic-bert),
the now-correct sustained 100% CPU saturation across all cores can
starve other services hard enough to take the box down. Confirmed
on a Threadripper 3960X + RX 6800 NOMAD: a wikipedia-class ZIM
ingestion pegged 48 threads cleanly enough that sshd lost
banner-exchange responsiveness and the box ultimately required a
power-cycle.

NVIDIA installs aren't affected — nomic-embed-text:v1.5 runs at
100% GPU on RTX 5060 (verified via `ollama ps`).

Detect placement at runtime, pace only when needed:

1. OllamaService.isEmbeddingGpuAccelerated() — queries /api/ps and
   returns true if any loaded embedding model reports size_vram > 0.
   Fails closed (returns false) if /api/ps is unreachable or no embed
   model is loaded yet — over-pacing is safer than crashing.

2. EmbedFileJob.handle() — between batches (hasMoreBatches: true
   branch), check placement and `await setTimeout(CPU_BATCH_DELAY_MS)`
   when CPU-only. CPU_BATCH_DELAY_MS = 1000 (1s) — enough to give the
   OS scheduler a window for sshd/disk-collector/etc., small enough
   that total ingestion time isn't meaningfully affected (each batch
   is ~60-90s of work).

GPU-accelerated installs see zero behavior change.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 admin/app/jobs/embed_file_job.ts     | 19 +++++++++++++++
 admin/app/services/ollama_service.ts | 36 ++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts
index 771cedf..426608a 100644
--- a/admin/app/jobs/embed_file_job.ts
+++ b/admin/app/jobs/embed_file_job.ts
@@ -27,6 +27,12 @@ export class EmbedFileJob {
     return 'embed-file'
   }
 
+  // Delay between continuation batches when embedding runs CPU-only. Gives the OS
+  // scheduler a brief idle window so sshd / disk-collector / other services don't
+  // starve during long multi-batch ZIM ingestions. Skipped entirely when the
+  // embedding model is GPU-offloaded — see OllamaService.isEmbeddingGpuAccelerated().
+  static readonly CPU_BATCH_DELAY_MS = 1000
+
   static getJobId(filePath: string): string {
     return createHash('sha256').update(filePath).digest('hex').slice(0, 16)
   }
@@ -114,6 +120,19 @@ export class EmbedFileJob {
           `[EmbedFileJob] Batch complete. Dispatching next batch at offset ${nextOffset}`
         )
 
+        // Pace continuation batches when embedding is CPU-bound. Sustained 100% CPU
+        // saturation across all cores during multi-batch ZIM ingestion can starve
+        // other services (sshd has been seen to lose responsiveness hard enough to
+        // require a power-cycle). When GPU-accelerated, embeddings stream through
+        // the GPU and CPUs stay free — no pacing needed.
+        const isGpuAccelerated = await ollamaService.isEmbeddingGpuAccelerated()
+        if (!isGpuAccelerated) {
+          logger.info(
+            `[EmbedFileJob] Embedding is CPU-only — pacing ${EmbedFileJob.CPU_BATCH_DELAY_MS}ms before dispatching next batch`
+          )
+          await new Promise((resolve) => setTimeout(resolve, EmbedFileJob.CPU_BATCH_DELAY_MS))
+        }
+
         // Dispatch next batch (not final yet)
         await EmbedFileJob.dispatch({
           filePath,
diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts
index fe0cb1c..78fbf69 100644
--- a/admin/app/services/ollama_service.ts
+++ b/admin/app/services/ollama_service.ts
@@ -513,6 +513,42 @@ export class OllamaService {
     }
   }
 
+  /**
+   * Returns true if Ollama is currently running an embedding model with non-zero VRAM
+   * (i.e., GPU-offloaded). Returns false if the model is running CPU-only OR if it's
+   * not currently loaded OR if /api/ps is unreachable.
+   *
+   * Used by EmbedFileJob to pace continuation batches when the embedding model is
+   * CPU-bound — sustained 100% CPU on a multi-batch ZIM ingestion can starve other
+   * services (sshd, etc.) hard enough to require a power-cycle. AMD ROCm installs
+   * hit this today because Ollama's ROCm build doesn't accelerate nomic-bert; on
+   * NVIDIA, nomic-embed-text runs at 100% GPU and pacing is unnecessary.
+   *
+   * Only the Ollama-native endpoint is supported — backends that expose
+   * `/v1/embeddings` (LM Studio, llama.cpp) don't surface placement info.
+   */
+  public async isEmbeddingGpuAccelerated(): Promise<boolean> {
+    await this._ensureDependencies()
+    if (!this.baseUrl) return false
+
+    try {
+      const response = await axios.get(`${this.baseUrl}/api/ps`, { timeout: 5000 })
+      const models: Array<{ name?: string; size_vram?: number }> = response.data?.models ?? []
+      // Match any loaded model whose name signals it's an embedding model.
+      // nomic-embed-text, mxbai-embed-large, snowflake-arctic-embed, etc. all follow this convention.
+      return models.some(
+        (m) => m.name?.toLowerCase().includes('embed') && (m.size_vram ?? 0) > 0
+      )
+    } catch (err: any) {
+      // /api/ps unreachable (Ollama down, non-native backend, etc.) — fail closed: assume CPU,
+      // which means we'll pace. Better to over-pace than risk box-killing CPU saturation.
+      logger.warn(
+        `[OllamaService] Could not check embedding placement via /api/ps: ${err?.message ?? err}`
+      )
+      return false
+    }
+  }
+
   public async getModels(includeEmbeddings = false): Promise<NomadInstalledModel[]> {
     await this._ensureDependencies()
     if (!this.baseUrl) {