fix(gpu): detect Ollama CPU fallback from logs so auto-remediation fires

The GpuPassthroughRemediationProvider probed GPU passthrough by exec'ing
`nvidia-smi` inside nomad_ollama. When Ollama was created CPU-only — most
commonly because the NVIDIA container runtime was registered with Docker
only AFTER the AI Assistant was first installed (DockerService attaches a
GPU DeviceRequest only when 'nvidia' is in docker.info().Runtimes at
install time) — the container ships no nvidia-smi, so the exec returned an
error string whose letters the alphabetic-output check mistook for
"passthrough healthy". That false negative suppressed all auto-remediation,
leaving Ollama silently stuck on CPU (models run at a fraction of a token
per second) until the user manually clicked "Fix: Reinstall AI Assistant".

Replace the nvidia-smi probe with Ollama's own "inference compute" startup
log line — the ground truth for whether a GPU backend actually loaded — via
a new, unit-tested classifyOllamaComputeBackend() helper. One signal covers
both the "created CPU-only" case and the original "DeviceRequests present
but the toolkit binding tore" case. Add a cooldown on gpu.autoRemediatedAt
so a GPU that genuinely cannot be accelerated (an architecture Ollama has
no kernels for) does not trigger a reinstall on every admin boot.

Co-Authored-By: Claude <noreply@anthropic.com>

admin/app/services/ollama_compute.ts

@@ -0,0 +1,41 @@
+/**
+ * Pure helpers for classifying which compute backend Ollama actually loaded,
+ * parsed from its container logs. Extracted so both SystemService (GPU health
+ * display) and GpuPassthroughRemediationProvider (auto-reinstall decision) share
+ * one source of truth, and so the logic is unit-testable without Docker.
+ *
+ * Ollama writes one `inference compute` log line per detected device a few seconds
+ * after startup, e.g.:
+ *   GPU:  ... msg="inference compute" id=0 library=CUDA compute=12.1 name=CUDA0 description="NVIDIA GB10" ...
+ *   CPU:  ... msg="inference compute" id=cpu library=cpu compute="" name=cpu ...
+ */
+
+export type OllamaComputeBackend = 'gpu' | 'cpu' | 'unknown'
+
+const INFERENCE_LINE_MARKER = 'msg="inference compute"'
+const GPU_LIBRARY_RE = /library=(CUDA|ROCm|Vulkan)\b/
+
+/**
+ * Classify Ollama's compute backend from raw container log text.
+ *
+ * - 'gpu'     if any `inference compute` line reports a GPU backend
+ *             (CUDA/ROCm/Vulkan). Ollama may emit a CPU line alongside a GPU
+ *             line (CPU is always listed as a fallback device), so a GPU line
+ *             anywhere means the GPU is usable.
+ * - 'cpu'     if `inference compute` lines exist but none report a GPU backend —
+ *             i.e. Ollama fell back to / was created with CPU only.
+ * - 'unknown' if no `inference compute` line is present (container too fresh,
+ *             logs rotated, or Ollama not started).
+ */
+export function classifyOllamaComputeBackend(logText: string): OllamaComputeBackend {
+  const lines = logText.split('\n').filter((line) => line.includes(INFERENCE_LINE_MARKER))
+  if (lines.length === 0) return 'unknown'
+
+  if (lines.some((line) => GPU_LIBRARY_RE.test(line))) return 'gpu'
+
+  // Lines exist but none matched a GPU library. A `library=cpu` line is a
+  // definitive CPU fallback; anything else (unrecognized library) stays unknown.
+  if (lines.some((line) => /library=cpu\b/.test(line))) return 'cpu'
+
+  return 'unknown'
+}

admin/providers/gpu_passthrough_remediation_provider.ts

@@ -1,22 +1,33 @@
 import logger from '@adonisjs/core/services/logger'
 import type { ApplicationService } from '@adonisjs/core/types'
+import type Docker from 'dockerode'
 
 /**
  * Auto-remediates NVIDIA GPU passthrough loss after admin / host restart.
  *
- * After an update or container recreate, nomad_ollama's HostConfig.DeviceRequests
- * still lists the nvidia driver, but the NVIDIA Container Toolkit binding inside
- * the container is torn. `nvidia-smi` inside the container returns
- * "Failed to initialize NVML: Unknown Error" and Ollama silently falls back to
- * CPU inference. PR #208 added detection + a one-click "Fix: Reinstall AI Assistant"
- * banner. This provider does that click automatically on admin boot when the
- * condition is detected.
+ * Detects the condition from Ollama's own "inference compute" startup log line:
+ * if the NVIDIA container runtime is registered with Docker but Ollama still
+ * loaded a CPU-only backend, passthrough is broken and nomad_ollama is recreated.
+ * This single signal covers two failure modes:
+ *   1. nomad_ollama was created CPU-only because the nvidia runtime was registered
+ *      only AFTER the AI Assistant was first installed (DockerService attaches a
+ *      GPU DeviceRequest only when 'nvidia' is in docker.info().Runtimes at install
+ *      time). Common on a host where the runtime was added later.
+ *   2. After an update or container recreate, DeviceRequests still lists the nvidia
+ *      driver but the toolkit binding inside the container is torn, and Ollama
+ *      silently falls back to CPU.
+ *
+ * PR #208 added detection + a one-click "Fix: Reinstall AI Assistant" banner; this
+ * provider performs that click automatically on admin boot.
  *
  * Guards:
  *   - NVIDIA-only. AMD passthrough_failed has a different fix path (HSA override
  *     handling in PR #804) and is left to the user.
  *   - One-shot per admin boot. The provider runs once on startup; if the recreate
  *     itself fails the banner remains as a fallback.
+ *   - Cooldown: will not auto-reinstall more than once within
+ *     AUTO_REMEDIATE_COOLDOWN_MS, to avoid a reinstall loop when the GPU cannot be
+ *     accelerated (e.g. an architecture Ollama has no kernels for).
  *   - Opt-out via KV `ai.autoFixGpuPassthrough = false`.
  *   - Skipped entirely when no NVIDIA runtime is registered with Docker.
  */
@@ -63,38 +74,52 @@ export default class GpuPassthroughRemediationProvider {
           return
         }
 
-        // Probe: exec nvidia-smi inside the Ollama container. NVML init failure
-        // is the signature of a broken passthrough that DeviceRequests can't see.
+        // Probe: read Ollama's own "inference compute" startup line from its logs.
+        // This is the ground truth for whether Ollama loaded a GPU backend, and it
+        // catches every failure mode:
+        //   - nomad_ollama created CPU-only because the nvidia runtime was registered
+        //     only AFTER first install (Ollama logs library=cpu), and
+        //   - DeviceRequests present but the toolkit binding tore after a recreate,
+        //     where Ollama silently falls back to CPU (also library=cpu).
+        // The previous implementation probed by exec'ing `nvidia-smi` inside the
+        // container, but a CPU-only container does not ship nvidia-smi, so the exec
+        // returned an error string that the alphabetic-output check mistook for
+        // "healthy" — a false negative that suppressed all auto-remediation.
         const container = docker.getContainer(ollama.Id)
-        const exec = await container.exec({
-          Cmd: ['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'],
-          AttachStdout: true,
-          AttachStderr: true,
-        })
-        const stream = await exec.start({ Tty: true })
-        const output = await new Promise<string>((resolve) => {
-          let buf = ''
-          const timer = setTimeout(() => resolve(buf || 'TIMEOUT'), 8000)
-          stream.on('data', (chunk: Buffer) => (buf += chunk.toString('utf8')))
-          stream.on('end', () => {
-            clearTimeout(timer)
-            resolve(buf)
-          })
-        })
+        const { classifyOllamaComputeBackend } = await import('#services/ollama_compute')
+        const backend = classifyOllamaComputeBackend(await readOllamaStartupLogs(container))
 
-        const passthroughBroken =
-          /Failed to initialize NVML|Unknown Error|TIMEOUT/i.test(output) ||
-          !/[A-Za-z]/.test(output)
-
-        if (!passthroughBroken) {
+        if (backend === 'gpu') {
           logger.info(
-            '[GpuPassthroughRemediationProvider] NVIDIA passthrough healthy — no action needed.'
+            '[GpuPassthroughRemediationProvider] Ollama is using a GPU backend — no action needed.'
+          )
+          return
+        }
+
+        if (backend === 'unknown') {
+          logger.info(
+            '[GpuPassthroughRemediationProvider] No "inference compute" line found in nomad_ollama logs yet — skipping.'
+          )
+          return
+        }
+
+        // backend === 'cpu': an NVIDIA runtime is registered but Ollama still runs on
+        // CPU. Cooldown: don't auto-reinstall more than once within the window. A
+        // reinstall that fails to move Ollama onto the GPU (e.g. an architecture
+        // Ollama has no kernels for) would otherwise trigger a reinstall every boot.
+        const remediatedAtRaw = await KVStore.getValue('gpu.autoRemediatedAt')
+        const remediatedAtMs = remediatedAtRaw ? new Date(String(remediatedAtRaw)).getTime() : NaN
+        const withinCooldown =
+          Number.isFinite(remediatedAtMs) && Date.now() - remediatedAtMs < AUTO_REMEDIATE_COOLDOWN_MS
+        if (withinCooldown) {
+          logger.warn(
+            `[GpuPassthroughRemediationProvider] Ollama is on CPU but auto-remediation already ran within the last ${AUTO_REMEDIATE_COOLDOWN_MS / 60000}m (at ${remediatedAtRaw}). Skipping to avoid a reinstall loop; the manual "Fix: Reinstall AI Assistant" banner remains available.`
           )
           return
         }
 
         logger.warn(
-          '[GpuPassthroughRemediationProvider] NVIDIA passthrough broken (nvidia-smi inside nomad_ollama failed). ' +
+          '[GpuPassthroughRemediationProvider] NVIDIA runtime registered but Ollama fell back to CPU. ' +
             'Auto-reinstalling nomad_ollama; volumes and installed models are preserved.'
         )
 
@@ -120,3 +145,45 @@ export default class GpuPassthroughRemediationProvider {
     })
   }
 }
+
+/**
+ * Minimum gap between automatic nomad_ollama reinstalls triggered by this
+ * provider. Bounds a reinstall loop when the GPU cannot be accelerated.
+ */
+const AUTO_REMEDIATE_COOLDOWN_MS = 60 * 60 * 1000 // 1 hour
+
+/**
+ * Read nomad_ollama's logs from its startup window (the five minutes after it
+ * booted), where the "inference compute" line is emitted. Mirrors the windowing
+ * in SystemService.getOllamaInferenceComputeFromLogs. Returns '' on any error so
+ * the caller classifies the backend as 'unknown' rather than throwing.
+ */
+async function readOllamaStartupLogs(container: Docker.Container): Promise<string> {
+  try {
+    const inspect = await container.inspect()
+    const startedAtRaw = inspect?.State?.StartedAt
+    const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN
+
+    const logsOpts: {
+      stdout: true
+      stderr: true
+      follow: false
+      since?: number
+      until?: number
+      tail?: number
+    } = { stdout: true, stderr: true, follow: false }
+
+    if (Number.isFinite(startedAtMs) && startedAtMs > 0) {
+      const startedAtSec = Math.floor(startedAtMs / 1000)
+      logsOpts.since = startedAtSec
+      logsOpts.until = startedAtSec + 300
+    } else {
+      logsOpts.tail = 500
+    }
+
+    const buf = (await container.logs(logsOpts)) as unknown as Buffer
+    return buf.toString('utf8')
+  } catch {
+    return ''
+  }
+}

admin/tests/unit/ollama_compute.spec.ts

@@ -0,0 +1,45 @@
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import { classifyOllamaComputeBackend } from '../../app/services/ollama_compute.js'
+
+test('CUDA inference line is classified as gpu', () => {
+  const log =
+    'time=2026-07-16T00:49:00.371Z level=INFO source=types.go:32 msg="inference compute" id=0 filter_id=0 library=CUDA compute=12.1 name=CUDA0 description="NVIDIA GB10" libdirs=ollama,cuda_v13 driver=13.0 pci_id=000f:01:00.0 type=iGPU total="121.7 GiB" available="96.8 GiB"'
+  assert.equal(classifyOllamaComputeBackend(log), 'gpu')
+})
+
+test('CPU-only inference line is classified as cpu', () => {
+  const log =
+    'time=2026-07-15T01:51:42.511Z level=INFO source=types.go:50 msg="inference compute" id=cpu library=cpu compute="" name=cpu description=cpu libdirs=ollama driver="" pci_id="" type="" total="121.7 GiB" available="121.7 GiB"'
+  assert.equal(classifyOllamaComputeBackend(log), 'cpu')
+})
+
+test('a GPU line alongside a CPU fallback line is still gpu', () => {
+  const log =
+    'msg="inference compute" id=0 library=CUDA compute=12.1 name=CUDA0\n' +
+    'msg="inference compute" id=1 library=cpu name=cpu'
+  assert.equal(classifyOllamaComputeBackend(log), 'gpu')
+})
+
+test('no inference compute line is unknown', () => {
+  assert.equal(
+    classifyOllamaComputeBackend('time=... level=INFO msg="some other line"\nno compute line here'),
+    'unknown'
+  )
+})
+
+test('empty log is unknown', () => {
+  assert.equal(classifyOllamaComputeBackend(''), 'unknown')
+})
+
+test('ROCm and Vulkan are gpu backends', () => {
+  assert.equal(
+    classifyOllamaComputeBackend('msg="inference compute" id=0 library=ROCm name=AMD'),
+    'gpu'
+  )
+  assert.equal(
+    classifyOllamaComputeBackend('msg="inference compute" id=0 library=Vulkan name=Intel'),
+    'gpu'
+  )
+})