From 342a5c7195d48cbbe6b4250cc9a3c727e0e22a17 Mon Sep 17 00:00:00 2001 From: Fabricio Policarpo Date: Thu, 16 Jul 2026 01:14:09 +0000 Subject: [PATCH] fix(gpu): detect Ollama CPU fallback from logs so auto-remediation fires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GpuPassthroughRemediationProvider probed GPU passthrough by exec'ing `nvidia-smi` inside nomad_ollama. When Ollama was created CPU-only — most commonly because the NVIDIA container runtime was registered with Docker only AFTER the AI Assistant was first installed (DockerService attaches a GPU DeviceRequest only when 'nvidia' is in docker.info().Runtimes at install time) — the container ships no nvidia-smi, so the exec returned an error string whose letters the alphabetic-output check mistook for "passthrough healthy". That false negative suppressed all auto-remediation, leaving Ollama silently stuck on CPU (models run at a fraction of a token per second) until the user manually clicked "Fix: Reinstall AI Assistant". Replace the nvidia-smi probe with Ollama's own "inference compute" startup log line — the ground truth for whether a GPU backend actually loaded — via a new, unit-tested classifyOllamaComputeBackend() helper. One signal covers both the "created CPU-only" case and the original "DeviceRequests present but the toolkit binding tore" case. Add a cooldown on gpu.autoRemediatedAt so a GPU that genuinely cannot be accelerated (an architecture Ollama has no kernels for) does not trigger a reinstall on every admin boot. Co-Authored-By: Claude --- admin/app/services/ollama_compute.ts | 41 ++++++ .../gpu_passthrough_remediation_provider.ts | 129 +++++++++++++----- admin/tests/unit/ollama_compute.spec.ts | 45 ++++++ 3 files changed, 184 insertions(+), 31 deletions(-) create mode 100644 admin/app/services/ollama_compute.ts create mode 100644 admin/tests/unit/ollama_compute.spec.ts diff --git a/admin/app/services/ollama_compute.ts b/admin/app/services/ollama_compute.ts new file mode 100644 index 0000000..a947e24 --- /dev/null +++ b/admin/app/services/ollama_compute.ts @@ -0,0 +1,41 @@ +/** + * Pure helpers for classifying which compute backend Ollama actually loaded, + * parsed from its container logs. Extracted so both SystemService (GPU health + * display) and GpuPassthroughRemediationProvider (auto-reinstall decision) share + * one source of truth, and so the logic is unit-testable without Docker. + * + * Ollama writes one `inference compute` log line per detected device a few seconds + * after startup, e.g.: + * GPU: ... msg="inference compute" id=0 library=CUDA compute=12.1 name=CUDA0 description="NVIDIA GB10" ... + * CPU: ... msg="inference compute" id=cpu library=cpu compute="" name=cpu ... + */ + +export type OllamaComputeBackend = 'gpu' | 'cpu' | 'unknown' + +const INFERENCE_LINE_MARKER = 'msg="inference compute"' +const GPU_LIBRARY_RE = /library=(CUDA|ROCm|Vulkan)\b/ + +/** + * Classify Ollama's compute backend from raw container log text. + * + * - 'gpu' if any `inference compute` line reports a GPU backend + * (CUDA/ROCm/Vulkan). Ollama may emit a CPU line alongside a GPU + * line (CPU is always listed as a fallback device), so a GPU line + * anywhere means the GPU is usable. + * - 'cpu' if `inference compute` lines exist but none report a GPU backend — + * i.e. Ollama fell back to / was created with CPU only. + * - 'unknown' if no `inference compute` line is present (container too fresh, + * logs rotated, or Ollama not started). + */ +export function classifyOllamaComputeBackend(logText: string): OllamaComputeBackend { + const lines = logText.split('\n').filter((line) => line.includes(INFERENCE_LINE_MARKER)) + if (lines.length === 0) return 'unknown' + + if (lines.some((line) => GPU_LIBRARY_RE.test(line))) return 'gpu' + + // Lines exist but none matched a GPU library. A `library=cpu` line is a + // definitive CPU fallback; anything else (unrecognized library) stays unknown. + if (lines.some((line) => /library=cpu\b/.test(line))) return 'cpu' + + return 'unknown' +} diff --git a/admin/providers/gpu_passthrough_remediation_provider.ts b/admin/providers/gpu_passthrough_remediation_provider.ts index e06aa44..d60cf92 100644 --- a/admin/providers/gpu_passthrough_remediation_provider.ts +++ b/admin/providers/gpu_passthrough_remediation_provider.ts @@ -1,22 +1,33 @@ import logger from '@adonisjs/core/services/logger' import type { ApplicationService } from '@adonisjs/core/types' +import type Docker from 'dockerode' /** * Auto-remediates NVIDIA GPU passthrough loss after admin / host restart. * - * After an update or container recreate, nomad_ollama's HostConfig.DeviceRequests - * still lists the nvidia driver, but the NVIDIA Container Toolkit binding inside - * the container is torn. `nvidia-smi` inside the container returns - * "Failed to initialize NVML: Unknown Error" and Ollama silently falls back to - * CPU inference. PR #208 added detection + a one-click "Fix: Reinstall AI Assistant" - * banner. This provider does that click automatically on admin boot when the - * condition is detected. + * Detects the condition from Ollama's own "inference compute" startup log line: + * if the NVIDIA container runtime is registered with Docker but Ollama still + * loaded a CPU-only backend, passthrough is broken and nomad_ollama is recreated. + * This single signal covers two failure modes: + * 1. nomad_ollama was created CPU-only because the nvidia runtime was registered + * only AFTER the AI Assistant was first installed (DockerService attaches a + * GPU DeviceRequest only when 'nvidia' is in docker.info().Runtimes at install + * time). Common on a host where the runtime was added later. + * 2. After an update or container recreate, DeviceRequests still lists the nvidia + * driver but the toolkit binding inside the container is torn, and Ollama + * silently falls back to CPU. + * + * PR #208 added detection + a one-click "Fix: Reinstall AI Assistant" banner; this + * provider performs that click automatically on admin boot. * * Guards: * - NVIDIA-only. AMD passthrough_failed has a different fix path (HSA override * handling in PR #804) and is left to the user. * - One-shot per admin boot. The provider runs once on startup; if the recreate * itself fails the banner remains as a fallback. + * - Cooldown: will not auto-reinstall more than once within + * AUTO_REMEDIATE_COOLDOWN_MS, to avoid a reinstall loop when the GPU cannot be + * accelerated (e.g. an architecture Ollama has no kernels for). * - Opt-out via KV `ai.autoFixGpuPassthrough = false`. * - Skipped entirely when no NVIDIA runtime is registered with Docker. */ @@ -63,38 +74,52 @@ export default class GpuPassthroughRemediationProvider { return } - // Probe: exec nvidia-smi inside the Ollama container. NVML init failure - // is the signature of a broken passthrough that DeviceRequests can't see. + // Probe: read Ollama's own "inference compute" startup line from its logs. + // This is the ground truth for whether Ollama loaded a GPU backend, and it + // catches every failure mode: + // - nomad_ollama created CPU-only because the nvidia runtime was registered + // only AFTER first install (Ollama logs library=cpu), and + // - DeviceRequests present but the toolkit binding tore after a recreate, + // where Ollama silently falls back to CPU (also library=cpu). + // The previous implementation probed by exec'ing `nvidia-smi` inside the + // container, but a CPU-only container does not ship nvidia-smi, so the exec + // returned an error string that the alphabetic-output check mistook for + // "healthy" — a false negative that suppressed all auto-remediation. const container = docker.getContainer(ollama.Id) - const exec = await container.exec({ - Cmd: ['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'], - AttachStdout: true, - AttachStderr: true, - }) - const stream = await exec.start({ Tty: true }) - const output = await new Promise((resolve) => { - let buf = '' - const timer = setTimeout(() => resolve(buf || 'TIMEOUT'), 8000) - stream.on('data', (chunk: Buffer) => (buf += chunk.toString('utf8'))) - stream.on('end', () => { - clearTimeout(timer) - resolve(buf) - }) - }) + const { classifyOllamaComputeBackend } = await import('#services/ollama_compute') + const backend = classifyOllamaComputeBackend(await readOllamaStartupLogs(container)) - const passthroughBroken = - /Failed to initialize NVML|Unknown Error|TIMEOUT/i.test(output) || - !/[A-Za-z]/.test(output) - - if (!passthroughBroken) { + if (backend === 'gpu') { logger.info( - '[GpuPassthroughRemediationProvider] NVIDIA passthrough healthy — no action needed.' + '[GpuPassthroughRemediationProvider] Ollama is using a GPU backend — no action needed.' + ) + return + } + + if (backend === 'unknown') { + logger.info( + '[GpuPassthroughRemediationProvider] No "inference compute" line found in nomad_ollama logs yet — skipping.' + ) + return + } + + // backend === 'cpu': an NVIDIA runtime is registered but Ollama still runs on + // CPU. Cooldown: don't auto-reinstall more than once within the window. A + // reinstall that fails to move Ollama onto the GPU (e.g. an architecture + // Ollama has no kernels for) would otherwise trigger a reinstall every boot. + const remediatedAtRaw = await KVStore.getValue('gpu.autoRemediatedAt') + const remediatedAtMs = remediatedAtRaw ? new Date(String(remediatedAtRaw)).getTime() : NaN + const withinCooldown = + Number.isFinite(remediatedAtMs) && Date.now() - remediatedAtMs < AUTO_REMEDIATE_COOLDOWN_MS + if (withinCooldown) { + logger.warn( + `[GpuPassthroughRemediationProvider] Ollama is on CPU but auto-remediation already ran within the last ${AUTO_REMEDIATE_COOLDOWN_MS / 60000}m (at ${remediatedAtRaw}). Skipping to avoid a reinstall loop; the manual "Fix: Reinstall AI Assistant" banner remains available.` ) return } logger.warn( - '[GpuPassthroughRemediationProvider] NVIDIA passthrough broken (nvidia-smi inside nomad_ollama failed). ' + + '[GpuPassthroughRemediationProvider] NVIDIA runtime registered but Ollama fell back to CPU. ' + 'Auto-reinstalling nomad_ollama; volumes and installed models are preserved.' ) @@ -120,3 +145,45 @@ export default class GpuPassthroughRemediationProvider { }) } } + +/** + * Minimum gap between automatic nomad_ollama reinstalls triggered by this + * provider. Bounds a reinstall loop when the GPU cannot be accelerated. + */ +const AUTO_REMEDIATE_COOLDOWN_MS = 60 * 60 * 1000 // 1 hour + +/** + * Read nomad_ollama's logs from its startup window (the five minutes after it + * booted), where the "inference compute" line is emitted. Mirrors the windowing + * in SystemService.getOllamaInferenceComputeFromLogs. Returns '' on any error so + * the caller classifies the backend as 'unknown' rather than throwing. + */ +async function readOllamaStartupLogs(container: Docker.Container): Promise { + try { + const inspect = await container.inspect() + const startedAtRaw = inspect?.State?.StartedAt + const startedAtMs = startedAtRaw ? new Date(startedAtRaw).getTime() : NaN + + const logsOpts: { + stdout: true + stderr: true + follow: false + since?: number + until?: number + tail?: number + } = { stdout: true, stderr: true, follow: false } + + if (Number.isFinite(startedAtMs) && startedAtMs > 0) { + const startedAtSec = Math.floor(startedAtMs / 1000) + logsOpts.since = startedAtSec + logsOpts.until = startedAtSec + 300 + } else { + logsOpts.tail = 500 + } + + const buf = (await container.logs(logsOpts)) as unknown as Buffer + return buf.toString('utf8') + } catch { + return '' + } +} diff --git a/admin/tests/unit/ollama_compute.spec.ts b/admin/tests/unit/ollama_compute.spec.ts new file mode 100644 index 0000000..c3d739c --- /dev/null +++ b/admin/tests/unit/ollama_compute.spec.ts @@ -0,0 +1,45 @@ +import * as assert from 'node:assert/strict' +import { test } from 'node:test' + +import { classifyOllamaComputeBackend } from '../../app/services/ollama_compute.js' + +test('CUDA inference line is classified as gpu', () => { + const log = + 'time=2026-07-16T00:49:00.371Z level=INFO source=types.go:32 msg="inference compute" id=0 filter_id=0 library=CUDA compute=12.1 name=CUDA0 description="NVIDIA GB10" libdirs=ollama,cuda_v13 driver=13.0 pci_id=000f:01:00.0 type=iGPU total="121.7 GiB" available="96.8 GiB"' + assert.equal(classifyOllamaComputeBackend(log), 'gpu') +}) + +test('CPU-only inference line is classified as cpu', () => { + const log = + 'time=2026-07-15T01:51:42.511Z level=INFO source=types.go:50 msg="inference compute" id=cpu library=cpu compute="" name=cpu description=cpu libdirs=ollama driver="" pci_id="" type="" total="121.7 GiB" available="121.7 GiB"' + assert.equal(classifyOllamaComputeBackend(log), 'cpu') +}) + +test('a GPU line alongside a CPU fallback line is still gpu', () => { + const log = + 'msg="inference compute" id=0 library=CUDA compute=12.1 name=CUDA0\n' + + 'msg="inference compute" id=1 library=cpu name=cpu' + assert.equal(classifyOllamaComputeBackend(log), 'gpu') +}) + +test('no inference compute line is unknown', () => { + assert.equal( + classifyOllamaComputeBackend('time=... level=INFO msg="some other line"\nno compute line here'), + 'unknown' + ) +}) + +test('empty log is unknown', () => { + assert.equal(classifyOllamaComputeBackend(''), 'unknown') +}) + +test('ROCm and Vulkan are gpu backends', () => { + assert.equal( + classifyOllamaComputeBackend('msg="inference compute" id=0 library=ROCm name=AMD'), + 'gpu' + ) + assert.equal( + classifyOllamaComputeBackend('msg="inference compute" id=0 library=Vulkan name=Intel'), + 'gpu' + ) +})