From d50cfa192a0395758130221a9b2d6d2a68fcb78e Mon Sep 17 00:00:00 2001 From: chriscrosstalk <49691103+chriscrosstalk@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:33:09 -0500 Subject: [PATCH] feat(AI): per-model thinking toggle with global default (off) (#1079) * feat(AI): per-model thinking toggle with global default (off) Stop forcing thinking on for every capable model. Adds a global default (ai.autoThinking, ships OFF) and a per-model override in the chat window, shown only for thinking-capable models and remembered client-side. The /v1 (OpenAI-compat) endpoint ignores `think`; reasoning_effort is the real lever. The controller resolves per-request preference -> global default -> OFF (gated on capability), and the service maps that to reasoning_effort ('none' to suppress on a capable model, 'medium' for gpt-oss, unset to let a capable model default thinking on). A new thinkingCapable flag keeps non-Ollama backends from ever receiving reasoning_effort. installed-models is enriched with a `thinking` flag (checkModelHasThinking, now memoized) so the picker knows which models get the toggle. Stacks on #1078 (reasoning-field read + client-disconnect abort). Co-Authored-By: Claude Opus 4.8 (1M context) * feat(AI): thinking toggle as NOMAD Switch + info tooltip Address review feedback on the per-model thinking control: - Use the shared Switch component (matches AI Assistant settings) instead of a raw checkbox. - Label "Thinking:" with a colon to match the adjacent "Model:" label. - Add an InfoTooltip explaining what thinking does and where the default lives. Extend InfoTooltip with optional `position` ('top'|'bottom') and `align` ('center'|'right') so it opens downward and expands leftward from the right-edge header slot instead of being clipped/crushed against the viewport edge. Defaults preserve existing (benchmark page) behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- admin/app/controllers/ollama_controller.ts | 28 +++++++-- admin/app/controllers/settings_controller.ts | 2 + admin/app/services/ollama_service.ts | 35 ++++++++++- admin/app/validators/ollama.ts | 3 + admin/constants/kv_store.ts | 2 +- admin/inertia/components/InfoTooltip.tsx | 34 +++++++++-- admin/inertia/components/chat/index.tsx | 62 +++++++++++++++++++- admin/inertia/pages/settings/models.tsx | 12 +++- admin/types/kv_store.ts | 1 + admin/types/ollama.ts | 4 ++ 10 files changed, 167 insertions(+), 16 deletions(-) diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index 5e735b0..bd291a6 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -146,13 +146,21 @@ export default class OllamaController { logger.debug(`[OllamaController] Large system prompt (~${estimatedSystemTokens} tokens), requesting num_ctx: ${numCtx}`) } - // Check if the model supports "thinking" capability for enhanced response generation + // Check if the model supports "thinking" capability for enhanced response generation. + // Thinking is only enabled when the model supports it AND the user wants it: the explicit + // per-request preference wins, otherwise the global default (ai.autoThinking, default OFF). // If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model) - const think: boolean | 'medium' = thinkingCapability ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false + let thinkingEnabled = false + if (thinkingCapability) { + thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false) + } + const think: boolean | 'medium' = + thinkingEnabled ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false - // Separate sessionId from the Ollama request payload — Ollama rejects unknown fields - const { sessionId, ...ollamaRequest } = reqData + // Separate sessionId and the resolved thinking preference from the Ollama request payload — + // Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw). + const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData // Save user message to DB before streaming if sessionId provided let userContent: string | null = null @@ -175,6 +183,7 @@ export default class OllamaController { const stream = await this.ollamaService.chatStream({ ...ollamaRequest, think, + thinkingCapable: thinkingCapability, numCtx, signal: abortController.signal, }) @@ -209,7 +218,7 @@ export default class OllamaController { } // Non-streaming (legacy) path - const result = await this.ollamaService.chat({ ...ollamaRequest, think, numCtx }) + const result = await this.ollamaService.chat({ ...ollamaRequest, think, thinkingCapable: thinkingCapability, numCtx }) if (sessionId && result?.message?.content) { await this.chatService.addMessage(sessionId, 'assistant', result.message.content) @@ -389,7 +398,14 @@ export default class OllamaController { } async installedModels({ }: HttpContext) { - return await this.ollamaService.getModels() + const models = await this.ollamaService.getModels() + // Enrich each model with its thinking capability so the chat picker knows which models + // to show the per-model thinking toggle for. checkModelHasThinking memoizes /api/show + // results, so this stays cheap on repeat loads. Best-effort per model. + const thinking = await Promise.all( + models.map((m) => this.ollamaService.checkModelHasThinking(m.name)) + ) + return models.map((m, i) => ({ ...m, thinking: thinking[i] })) } /** diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index 7106972..46d1d86 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -66,6 +66,7 @@ export default class SettingsController { const aiAssistantCustomName = await KVStore.getValue('ai.assistantCustomName') const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl') const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention') + const autoThinking = await KVStore.getValue('ai.autoThinking') return inertia.render('settings/models', { models: { availableModels: availableModels?.models || [], @@ -75,6 +76,7 @@ export default class SettingsController { aiAssistantCustomName: aiAssistantCustomName ?? '', remoteOllamaUrl: remoteOllamaUrl ?? '', ollamaFlashAttention: ollamaFlashAttention ?? true, + autoThinking: autoThinking ?? false, }, }, }) diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index 0829cd4..d030bd7 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -43,6 +43,9 @@ type ChatInput = { model: string messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> think?: boolean | 'medium' + // Whether the target model supports thinking. Lets chat()/chatStream() tell "capable but + // disabled" (send reasoning_effort:'none') apart from "not capable" (send nothing). + thinkingCapable?: boolean stream?: boolean numCtx?: number // Aborts the upstream request when the client disconnects, so an abandoned generation @@ -57,6 +60,9 @@ export class OllamaService { private initPromise: Promise | null = null private isOllamaNative: boolean | null = null private activeDownloads: Map> = new Map() + // Memoized `thinking` capability per model name (see checkModelHasThinking). Only successful + // /api/show lookups are cached; transient failures are left uncached so they can be retried. + private thinkingCapabilityCache: Map = new Map() constructor() {} @@ -332,6 +338,15 @@ export class OllamaService { if (chatRequest.think) { params.think = chatRequest.think } + // The /v1 (OpenAI-compat) endpoint ignores `think`; `reasoning_effort` is the actual lever. + // Only touch it for thinking-capable models so non-Ollama backends never get an unexpected + // param. gpt-oss requires an explicit level; a capable-but-disabled model gets 'none' to + // suppress thinking (capable models default thinking ON otherwise, so think===true is a no-op). + if (chatRequest.think === 'medium') { + params.reasoning_effort = 'medium' + } else if (chatRequest.thinkingCapable && chatRequest.think === false) { + params.reasoning_effort = 'none' + } if (chatRequest.numCtx) { params.num_ctx = chatRequest.numCtx } @@ -365,6 +380,15 @@ export class OllamaService { if (chatRequest.think) { params.think = chatRequest.think } + // The /v1 (OpenAI-compat) endpoint ignores `think`; `reasoning_effort` is the actual lever. + // Only touch it for thinking-capable models so non-Ollama backends never get an unexpected + // param. gpt-oss requires an explicit level; a capable-but-disabled model gets 'none' to + // suppress thinking (capable models default thinking ON otherwise, so think===true is a no-op). + if (chatRequest.think === 'medium') { + params.reasoning_effort = 'medium' + } else if (chatRequest.thinkingCapable && chatRequest.think === false) { + params.reasoning_effort = 'none' + } if (chatRequest.numCtx) { params.num_ctx = chatRequest.numCtx } @@ -444,13 +468,22 @@ export class OllamaService { await this._ensureDependencies() if (!this.baseUrl) return false + // A model's capabilities don't change at runtime, so memoize the /api/show result. Without + // this, loading the chat picker fires one /api/show per installed model and every chat send + // fires another — this collapses those to a single call per model per process. + const cached = this.thinkingCapabilityCache.get(modelName) + if (cached !== undefined) return cached + try { const response = await axios.post( `${this.baseUrl}/api/show`, { model: modelName }, { timeout: 5000 } ) - return Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking') + const hasThinking = + Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking') + this.thinkingCapabilityCache.set(modelName, hasThinking) + return hasThinking } catch { // Non-Ollama backends don't expose /api/show — assume no thinking support return false diff --git a/admin/app/validators/ollama.ts b/admin/app/validators/ollama.ts index 1784ada..13bee20 100644 --- a/admin/app/validators/ollama.ts +++ b/admin/app/validators/ollama.ts @@ -11,6 +11,9 @@ export const chatSchema = vine.compile( ), stream: vine.boolean().optional(), sessionId: vine.number().positive().optional(), + // Effective per-request thinking preference (per-model override or global default), + // resolved client-side. Omitted -> server falls back to the ai.autoThinking KV default. + think: vine.boolean().optional(), }) ) diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 35b9617..939bdb5 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -1,3 +1,3 @@ import { KVStoreKey } from "../types/kv_store.js"; -export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow']; \ No newline at end of file +export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'ai.autoThinking', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow']; \ No newline at end of file diff --git a/admin/inertia/components/InfoTooltip.tsx b/admin/inertia/components/InfoTooltip.tsx index f41a90d..172ee5a 100644 --- a/admin/inertia/components/InfoTooltip.tsx +++ b/admin/inertia/components/InfoTooltip.tsx @@ -4,9 +4,21 @@ import { useState } from 'react' interface InfoTooltipProps { text: string className?: string + // Which side of the icon the tooltip pops toward. Defaults to 'top' (existing behavior); + // use 'bottom' when the icon sits near the top of the viewport so it isn't clipped. + position?: 'top' | 'bottom' + // Horizontal anchoring. 'center' (default) centers the bubble on the icon. Use 'right' when + // the icon sits near the right edge so the bubble expands leftward into open space instead of + // being squeezed against the viewport edge (which forces one-word-per-line wrapping). + align?: 'center' | 'right' } -export default function InfoTooltip({ text, className = '' }: InfoTooltipProps) { +export default function InfoTooltip({ + text, + className = '', + position = 'top', + align = 'center', +}: InfoTooltipProps) { const [isVisible, setIsVisible] = useState(false) return ( @@ -23,10 +35,24 @@ export default function InfoTooltip({ text, className = '' }: InfoTooltipProps) {isVisible && ( -
-
+
+
{text} -
+
)} diff --git a/admin/inertia/components/chat/index.tsx b/admin/inertia/components/chat/index.tsx index 3364a80..8abf607 100644 --- a/admin/inertia/components/chat/index.tsx +++ b/admin/inertia/components/chat/index.tsx @@ -12,6 +12,8 @@ import classNames from '~/lib/classNames' import { IconMenu2, IconX } from '@tabler/icons-react' import { DEFAULT_QUERY_REWRITE_MODEL } from '../../../constants/ollama' import { useSystemSetting } from '~/hooks/useSystemSetting' +import Switch from '~/components/inputs/Switch' +import InfoTooltip from '~/components/InfoTooltip' interface ChatProps { enabled: boolean @@ -68,6 +70,11 @@ export default function Chat({ const { data: lastModelSetting } = useSystemSetting({ key: 'chat.lastModel', enabled }) const { data: remoteOllamaUrlSetting } = useSystemSetting({ key: 'ai.remoteOllamaUrl', enabled }) + const { data: autoThinkingSetting } = useSystemSetting({ key: 'ai.autoThinking', enabled }) + // Global default for models the user hasn't explicitly toggled. Coerce defensively — KV + // booleans have historically round-tripped as strings. + const autoThinkingDefault = + autoThinkingSetting?.value === true || autoThinkingSetting?.value === 'true' const { data: remoteStatus } = useQuery({ queryKey: ['remoteOllamaStatus'], @@ -89,6 +96,38 @@ export default function Chat({ select: (data) => data?.collections ?? [], }) + // Per-model thinking overrides, remembered client-side (localStorage, keyed by model name). + // An entry here means the user explicitly toggled thinking for that model; absent means fall + // back to the global default (ai.autoThinking). Seeded from localStorage when models load. + const [thinkingOverrides, setThinkingOverrides] = useState>({}) + useEffect(() => { + const next: Record = {} + for (const m of installedModels) { + try { + const stored = localStorage.getItem(`nomad:thinking:${m.name}`) + if (stored !== null) next[m.name] = stored === 'true' + } catch {} + } + setThinkingOverrides(next) + }, [installedModels]) + + const selectedModelSupportsThinking = + installedModels.find((m) => m.name === selectedModel)?.thinking === true + + // Effective thinking preference for a model: explicit override wins, else the global default. + const effectiveThinking = useCallback( + (model: string): boolean => + model in thinkingOverrides ? thinkingOverrides[model] : autoThinkingDefault, + [thinkingOverrides, autoThinkingDefault] + ) + + const setModelThinking = useCallback((model: string, value: boolean) => { + setThinkingOverrides((prev) => ({ ...prev, [model]: value })) + try { + localStorage.setItem(`nomad:thinking:${model}`, String(value)) + } catch {} + }, []) + const { data: chatSuggestions, isLoading: chatSuggestionsLoading } = useQuery({ queryKey: ['chatSuggestions'], queryFn: async ({ signal }) => { @@ -119,6 +158,7 @@ export default function Chat({ model: string messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> sessionId?: number + think?: boolean collection?: string }) => api.sendChatMessage({ ...request, stream: false }), onSuccess: async (data) => { @@ -345,7 +385,7 @@ export default function Chat({ model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, - sessionId: sessionId ? Number(sessionId) : undefined, + sessionId: sessionId ? Number(sessionId) : undefined, think: effectiveThinking(selectedModel), collection: collectionFilter || undefined, }, (chunkContent, chunkThinking, done) => { @@ -437,11 +477,12 @@ export default function Chat({ model: selectedModel || 'llama3.2', messages: chatMessages, sessionId: sessionId ? Number(sessionId) : undefined, + think: effectiveThinking(selectedModel), collection: collectionFilter || undefined, }) } }, - [activeSessionId, messages, selectedModel, collectionFilter, chatMutation, queryClient, streamingEnabled] + [activeSessionId, messages, selectedModel, collectionFilter, chatMutation, queryClient, streamingEnabled, effectiveThinking] ) return ( @@ -557,7 +598,22 @@ export default function Chat({ )}
- {isInModal && ( + {selectedModelSupportsThinking && ( +
+ Thinking: + + setModelThinking(selectedModel, v)} + /> +
+ )} + {isInModal && (