From a315ce0f5412797b852fdd8e8829f0711af84496 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Mon, 8 Jun 2026 09:42:00 -0700 Subject: [PATCH] fix(AI): truncate-and-retry oversized embed chunks; stop 30x retry storm (#881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dense source content produces chunks that exceed the embedding model's context window (nomic-embed-text:v1.5 defaults to 2048 tokens). Two paths hit this even after the prior pre-cap: - Older Ollama (e.g. 0.18.1, #944) ignores the num_ctx=8192 we send on /api/embed, so it stays at the model's 2048 default. - The OpenAI-compat /v1/embeddings fallback didn't pass num_ctx/truncate at all, so any Ollama drops to 2048 whenever it lands on the fallback. When a chunk overflowed, the 400 was swallowed and the chunk was silently dropped from Qdrant. Worse, the failure propagated to EmbedFileJob, which re-embeds the entire file on each of its 30 BullMQ attempts — the "endless queue loop" / "api/embed for weeks" / pegged GPU reported in #944/#959. Fix: - OllamaService.embed(): on a context-length error, retry once with an aggressive 2048-safe cap (EMBED_CONTEXT_SAFE_CHARS = 2000) so the chunk is embedded (start-of-chunk) instead of dropped. Native-path context errors now bubble to this retry instead of falling through to the smaller-context fallback. Split the native+fallback attempt into _embedWithFallback(). - Pass truncate/num_ctx on the /v1/embeddings fallback too (Ollama's OpenAI-compat shim forwards them). - EmbedFileJob: classify "input length exceeds context length" as an UnrecoverableError so one permanently-oversized chunk can't trigger 30 full-file re-embeds. - Add OllamaService.isContextLengthError() shared by both. Graceful degradation: a truncated chunk loses its tail but is kept in the index, which is strictly better than today's silent drop + retry storm. Refs #881. Supersedes the #369/#670 symptom closures that never fixed the fallback path. --- admin/app/jobs/embed_file_job.ts | 25 ++++-- admin/app/services/ollama_service.ts | 124 ++++++++++++++++++--------- 2 files changed, 105 insertions(+), 44 deletions(-) diff --git a/admin/app/jobs/embed_file_job.ts b/admin/app/jobs/embed_file_job.ts index 0e95791..93f58a9 100644 --- a/admin/app/jobs/embed_file_job.ts +++ b/admin/app/jobs/embed_file_job.ts @@ -247,23 +247,38 @@ export class EmbedFileJob { message: `Successfully embedded ${result.chunks} chunks`, } } catch (error) { - logger.error(`[EmbedFileJob] Error embedding file ${fileName}:`, error) + // A chunk that still exceeds the model's context after OllamaService's truncate-and-retry is + // permanently oversized for this install (e.g. a model whose context is smaller than our safe + // cap). Re-embedding the whole file 30x re-processes everything and can never succeed — that is + // the "endless queue loop" / "api/embed for weeks" (#881/#944/#959). Mark it unrecoverable so + // BullMQ stops after one pass instead of storming. + let normalizedError = error + if (!(error instanceof UnrecoverableError) && OllamaService.isContextLengthError(error)) { + logger.warn( + `[EmbedFileJob] Context-length overflow persisted for ${fileName} after truncation; not retrying.` + ) + normalizedError = new UnrecoverableError( + error instanceof Error ? error.message : 'Embedding input exceeds the model context length' + ) + } + + logger.error(`[EmbedFileJob] Error embedding file ${fileName}:`, normalizedError) await job.updateData({ ...job.data, status: 'failed', failedAt: Date.now(), - error: error instanceof Error ? error.message : 'Unknown error', + error: normalizedError instanceof Error ? normalizedError.message : 'Unknown error', }) // Only persist `failed` for unrecoverable errors. Retryable errors get // automatic BullMQ retries (30 attempts); marking state failed on every // transient blip would suppress the retry-driven recovery path. - if (error instanceof UnrecoverableError) { + if (normalizedError instanceof UnrecoverableError) { try { await KbIngestState.markFailed( filePath, - error instanceof Error ? error.message : 'Unknown error' + normalizedError instanceof Error ? normalizedError.message : 'Unknown error' ) } catch (stateErr) { logger.warn( @@ -273,7 +288,7 @@ export class EmbedFileJob { } } - throw error + throw normalizedError } } diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 526407c..be319e9 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -481,9 +481,43 @@ export class OllamaService { */ public static readonly EMBED_MAX_INPUT_CHARS = 4000 + /** + * Aggressive 2048-safe character cap, applied only on a context-length retry. nomic-embed-text:v1.5 + * defaults to a 2048-token context, and on the OpenAI-compat fallback path (or an older Ollama that + * ignores num_ctx for embeddings) we cannot widen it at request time. 2000 chars stays under 2048 + * tokens even for the densest content (~1 char/token code/markup), so an oversized chunk gets + * truncated-and-kept instead of silently dropped from Qdrant — and the embed job stops re-embedding + * the whole file 30x on the one bad chunk (#881). + */ + public static readonly EMBED_CONTEXT_SAFE_CHARS = 2000 + + /** + * True if the error is the model rejecting input that exceeds its context window + * ("input length exceeds the context length"). Matches both the native /api/embed axios error + * shape and the OpenAI-compat BadRequestError. Drives the truncate-and-retry here and the + * non-retryable classification in EmbedFileJob (#881). + */ + public static isContextLengthError(err: unknown): boolean { + const parts: string[] = [] + if (err instanceof Error && err.message) parts.push(err.message) + const anyErr = err as any + const data = anyErr?.response?.data + if (data) parts.push(typeof data === 'string' ? data : JSON.stringify(data)) + if (anyErr?.error) parts.push(typeof anyErr.error === 'string' ? anyErr.error : JSON.stringify(anyErr.error)) + const haystack = parts.join(' ').toLowerCase() + return ( + (haystack.includes('context length') && haystack.includes('exceed')) || + haystack.includes('input length exceeds') + ) + } + /** * Generate embeddings for the given input strings. * Tries the Ollama native /api/embed endpoint first, falls back to /v1/embeddings. + * + * If the first attempt fails because a chunk exceeds the model's context window, retries once + * with an aggressive 2048-safe truncation (EMBED_CONTEXT_SAFE_CHARS) so the chunk is embedded + * (start-of-chunk) rather than silently dropped from Qdrant (#881). */ public async embed(model: string, input: string[]): Promise<{ embeddings: number[][] }> { await this._ensureDependencies() @@ -491,41 +525,49 @@ export class OllamaService { throw new Error('AI service is not initialized.') } - // Runtime safety net (#881). The OpenAI-compat fallback has no equivalent of - // truncate:true, so a chunk that exceeds the model's loaded context_length - // (often 2048 for nomic-embed-text:v1.5) returns 400 and the chunk is silently - // dropped from Qdrant. Pre-capping at the input layer protects both paths. - const safeInput = input.map((s) => - s.length > OllamaService.EMBED_MAX_INPUT_CHARS - ? s.slice(0, OllamaService.EMBED_MAX_INPUT_CHARS) - : s - ) - const truncatedCount = input.reduce( - (n, s) => (s.length > OllamaService.EMBED_MAX_INPUT_CHARS ? n + 1 : n), - 0 - ) - if (truncatedCount > 0) { - logger.debug( - '[OllamaService] embed: pre-capped %d/%d inputs at %d chars', - truncatedCount, - input.length, - OllamaService.EMBED_MAX_INPUT_CHARS - ) - } + const cap = (arr: string[], max: number) => arr.map((s) => (s.length > max ? s.slice(0, max) : s)) + + // Generous pre-cap (#881): fine for the native path (num_ctx=8192) but can still exceed a + // 2048-context fallback on dense content. The context-length retry below is the hard backstop. + const safeInput = cap(input, OllamaService.EMBED_MAX_INPUT_CHARS) try { - // Prefer Ollama native endpoint (supports batch input natively). - // Pass num_ctx explicitly so we don't depend on the embedding model's - // modelfile defaults. Some installs ship nomic-embed-text:v1.5 with - // num_ctx=2048, which our chunker (sized for ~1500 tokens) can exceed - // on dense content, causing "input length exceeds context length" errors. - // truncate:true is a runtime safety net for any chunk that still overshoots. - // 8192 matches nomic-embed-text:v1.5's RoPE-extrapolated max. + return await this._embedWithFallback(model, safeInput) + } catch (err) { + if (!OllamaService.isContextLengthError(err)) throw err + // One or more chunks exceeded the model's context even after the pre-cap — typically an + // older Ollama that ignores num_ctx for embeddings, or the OpenAI-compat fallback path. + // Retry once, truncated hard enough to fit a 2048-token context at any density, so the + // chunk is embedded (truncated) instead of dropped and the job doesn't storm. + const hardCapped = cap(input, OllamaService.EMBED_CONTEXT_SAFE_CHARS) + const reduced = hardCapped.reduce((n, s, i) => (s.length < safeInput[i].length ? n + 1 : n), 0) + logger.warn( + '[OllamaService] embed: context-length overflow; retrying %d/%d inputs hard-capped at %d chars', + reduced, + input.length, + OllamaService.EMBED_CONTEXT_SAFE_CHARS + ) + return await this._embedWithFallback(model, hardCapped) + } + } + + /** + * Single embed attempt: native /api/embed first, then the OpenAI-compat /v1/embeddings fallback. + * Both paths request num_ctx/truncate (Ollama's OpenAI-compat shim forwards them). A context-length + * error from the native path is re-thrown rather than falling back, because the fallback has a + * smaller effective context and would only fail the same way — the caller (embed) retries it + * truncated instead. + */ + private async _embedWithFallback(model: string, input: string[]): Promise<{ embeddings: number[][] }> { + try { + // Pass num_ctx explicitly so we don't depend on the embedding model's modelfile defaults. + // Some installs ship nomic-embed-text:v1.5 with num_ctx=2048; 8192 matches its RoPE-extrapolated + // max. truncate:true is a server-side net for any chunk that still overshoots. const response = await axios.post( `${this.baseUrl}/api/embed`, { model, - input: safeInput, + input, truncate: true, options: { num_ctx: 8192 }, }, @@ -538,22 +580,26 @@ export class OllamaService { } return { embeddings: response.data.embeddings } } catch (err) { - // Capture the original error so we know *why* we fell back. Earlier bare - // catches here masked recurring "input length exceeds context length" - // failures for months (#369, #670, #881) — without this log we have no - // signal that /api/embed is the broken path vs the fallback. + // Let context-length errors bubble so embed() can retry with a smaller cap; the fallback + // endpoint (smaller effective context, no num_ctx honored on older Ollama) can't help here. + if (OllamaService.isContextLengthError(err)) throw err + // Log the original error so we know *why* we fell back. Earlier bare catches here masked + // recurring failures for months (#369, #670, #881). logger.warn( '[OllamaService] /api/embed failed, falling back to /v1/embeddings: %s', err instanceof Error ? err.message : String(err) ) - // Fall back to OpenAI-compatible /v1/embeddings. - // Explicitly request float format — some backends (e.g. LM Studio) don't reliably - // implement the base64 encoding the OpenAI SDK requests by default. - const results = await this.openai.embeddings.create({ + // Fall back to OpenAI-compatible /v1/embeddings. Explicitly request float format — some + // backends (e.g. LM Studio) don't reliably implement the base64 the OpenAI SDK defaults to. + // truncate/num_ctx are forwarded by Ollama's OpenAI-compat shim; the SDK types omit them, + // hence the cast. We only ever talk to a local Ollama here, not real OpenAI. + const results = await this.openai!.embeddings.create({ model, - input: safeInput, + input, encoding_format: 'float', - }) + truncate: true, + options: { num_ctx: 8192 }, + } as any) return { embeddings: results.data.map((e) => e.embedding as number[]) } } }