From d64cadafc278b090ebd2ac38bebdf1212783b882 Mon Sep 17 00:00:00 2001 From: Chris Sherwood Date: Fri, 10 Jul 2026 23:04:32 -0700 Subject: [PATCH 1/2] feat(AI): per-model thinking toggle with global default (off) Stop forcing thinking on for every capable model. Adds a global default (ai.autoThinking, ships OFF) and a per-model override in the chat window, shown only for thinking-capable models and remembered client-side. The /v1 (OpenAI-compat) endpoint ignores `think`; reasoning_effort is the real lever. The controller resolves per-request preference -> global default -> OFF (gated on capability), and the service maps that to reasoning_effort ('none' to suppress on a capable model, 'medium' for gpt-oss, unset to let a capable model default thinking on). A new thinkingCapable flag keeps non-Ollama backends from ever receiving reasoning_effort. installed-models is enriched with a `thinking` flag (checkModelHasThinking, now memoized) so the picker knows which models get the toggle. Stacks on #1078 (reasoning-field read + client-disconnect abort). Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/app/controllers/ollama_controller.ts | 28 +++++++--- admin/app/controllers/settings_controller.ts | 2 + admin/app/services/ollama_service.ts | 35 +++++++++++- admin/app/validators/ollama.ts | 3 ++ admin/constants/kv_store.ts | 2 +- admin/inertia/components/chat/index.tsx | 57 +++++++++++++++++++- admin/inertia/pages/settings/models.tsx | 12 ++++- admin/types/kv_store.ts | 1 + admin/types/ollama.ts | 4 ++ 9 files changed, 133 insertions(+), 11 deletions(-) diff --git a/admin/app/controllers/ollama_controller.ts b/admin/app/controllers/ollama_controller.ts index cd22326..51233b0 100644 --- a/admin/app/controllers/ollama_controller.ts +++ b/admin/app/controllers/ollama_controller.ts @@ -144,13 +144,21 @@ export default class OllamaController { logger.debug(`[OllamaController] Large system prompt (~${estimatedSystemTokens} tokens), requesting num_ctx: ${numCtx}`) } - // Check if the model supports "thinking" capability for enhanced response generation + // Check if the model supports "thinking" capability for enhanced response generation. + // Thinking is only enabled when the model supports it AND the user wants it: the explicit + // per-request preference wins, otherwise the global default (ai.autoThinking, default OFF). // If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model) - const think: boolean | 'medium' = thinkingCapability ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false + let thinkingEnabled = false + if (thinkingCapability) { + thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false) + } + const think: boolean | 'medium' = + thinkingEnabled ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false - // Separate sessionId from the Ollama request payload — Ollama rejects unknown fields - const { sessionId, ...ollamaRequest } = reqData + // Separate sessionId and the resolved thinking preference from the Ollama request payload — + // Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw). + const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData // Save user message to DB before streaming if sessionId provided let userContent: string | null = null @@ -173,6 +181,7 @@ export default class OllamaController { const stream = await this.ollamaService.chatStream({ ...ollamaRequest, think, + thinkingCapable: thinkingCapability, numCtx, signal: abortController.signal, }) @@ -207,7 +216,7 @@ export default class OllamaController { } // Non-streaming (legacy) path - const result = await this.ollamaService.chat({ ...ollamaRequest, think, numCtx }) + const result = await this.ollamaService.chat({ ...ollamaRequest, think, thinkingCapable: thinkingCapability, numCtx }) if (sessionId && result?.message?.content) { await this.chatService.addMessage(sessionId, 'assistant', result.message.content) @@ -387,7 +396,14 @@ export default class OllamaController { } async installedModels({ }: HttpContext) { - return await this.ollamaService.getModels() + const models = await this.ollamaService.getModels() + // Enrich each model with its thinking capability so the chat picker knows which models + // to show the per-model thinking toggle for. checkModelHasThinking memoizes /api/show + // results, so this stays cheap on repeat loads. Best-effort per model. + const thinking = await Promise.all( + models.map((m) => this.ollamaService.checkModelHasThinking(m.name)) + ) + return models.map((m, i) => ({ ...m, thinking: thinking[i] })) } /** diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index 4675d4b..79996e4 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -66,6 +66,7 @@ export default class SettingsController { const aiAssistantCustomName = await KVStore.getValue('ai.assistantCustomName') const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl') const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention') + const autoThinking = await KVStore.getValue('ai.autoThinking') return inertia.render('settings/models', { models: { availableModels: availableModels?.models || [], @@ -75,6 +76,7 @@ export default class SettingsController { aiAssistantCustomName: aiAssistantCustomName ?? '', remoteOllamaUrl: remoteOllamaUrl ?? '', ollamaFlashAttention: ollamaFlashAttention ?? true, + autoThinking: autoThinking ?? false, }, }, }) diff --git a/admin/app/services/ollama_service.ts b/admin/app/services/ollama_service.ts index b0ca3b0..736ba39 100644 --- a/admin/app/services/ollama_service.ts +++ b/admin/app/services/ollama_service.ts @@ -43,6 +43,9 @@ type ChatInput = { model: string messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> think?: boolean | 'medium' + // Whether the target model supports thinking. Lets chat()/chatStream() tell "capable but + // disabled" (send reasoning_effort:'none') apart from "not capable" (send nothing). + thinkingCapable?: boolean stream?: boolean numCtx?: number // Aborts the upstream request when the client disconnects, so an abandoned generation @@ -57,6 +60,9 @@ export class OllamaService { private initPromise: Promise | null = null private isOllamaNative: boolean | null = null private activeDownloads: Map> = new Map() + // Memoized `thinking` capability per model name (see checkModelHasThinking). Only successful + // /api/show lookups are cached; transient failures are left uncached so they can be retried. + private thinkingCapabilityCache: Map = new Map() constructor() {} @@ -332,6 +338,15 @@ export class OllamaService { if (chatRequest.think) { params.think = chatRequest.think } + // The /v1 (OpenAI-compat) endpoint ignores `think`; `reasoning_effort` is the actual lever. + // Only touch it for thinking-capable models so non-Ollama backends never get an unexpected + // param. gpt-oss requires an explicit level; a capable-but-disabled model gets 'none' to + // suppress thinking (capable models default thinking ON otherwise, so think===true is a no-op). + if (chatRequest.think === 'medium') { + params.reasoning_effort = 'medium' + } else if (chatRequest.thinkingCapable && chatRequest.think === false) { + params.reasoning_effort = 'none' + } if (chatRequest.numCtx) { params.num_ctx = chatRequest.numCtx } @@ -365,6 +380,15 @@ export class OllamaService { if (chatRequest.think) { params.think = chatRequest.think } + // The /v1 (OpenAI-compat) endpoint ignores `think`; `reasoning_effort` is the actual lever. + // Only touch it for thinking-capable models so non-Ollama backends never get an unexpected + // param. gpt-oss requires an explicit level; a capable-but-disabled model gets 'none' to + // suppress thinking (capable models default thinking ON otherwise, so think===true is a no-op). + if (chatRequest.think === 'medium') { + params.reasoning_effort = 'medium' + } else if (chatRequest.thinkingCapable && chatRequest.think === false) { + params.reasoning_effort = 'none' + } if (chatRequest.numCtx) { params.num_ctx = chatRequest.numCtx } @@ -444,13 +468,22 @@ export class OllamaService { await this._ensureDependencies() if (!this.baseUrl) return false + // A model's capabilities don't change at runtime, so memoize the /api/show result. Without + // this, loading the chat picker fires one /api/show per installed model and every chat send + // fires another — this collapses those to a single call per model per process. + const cached = this.thinkingCapabilityCache.get(modelName) + if (cached !== undefined) return cached + try { const response = await axios.post( `${this.baseUrl}/api/show`, { model: modelName }, { timeout: 5000 } ) - return Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking') + const hasThinking = + Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking') + this.thinkingCapabilityCache.set(modelName, hasThinking) + return hasThinking } catch { // Non-Ollama backends don't expose /api/show — assume no thinking support return false diff --git a/admin/app/validators/ollama.ts b/admin/app/validators/ollama.ts index 1784ada..13bee20 100644 --- a/admin/app/validators/ollama.ts +++ b/admin/app/validators/ollama.ts @@ -11,6 +11,9 @@ export const chatSchema = vine.compile( ), stream: vine.boolean().optional(), sessionId: vine.number().positive().optional(), + // Effective per-request thinking preference (per-model override or global default), + // resolved client-side. Omitted -> server falls back to the ai.autoThinking KV default. + think: vine.boolean().optional(), }) ) diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 35b9617..939bdb5 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -1,3 +1,3 @@ import { KVStoreKey } from "../types/kv_store.js"; -export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow']; \ No newline at end of file +export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'ai.autoThinking', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow']; \ No newline at end of file diff --git a/admin/inertia/components/chat/index.tsx b/admin/inertia/components/chat/index.tsx index 9be60d4..7536be3 100644 --- a/admin/inertia/components/chat/index.tsx +++ b/admin/inertia/components/chat/index.tsx @@ -57,6 +57,11 @@ export default function Chat({ const { data: lastModelSetting } = useSystemSetting({ key: 'chat.lastModel', enabled }) const { data: remoteOllamaUrlSetting } = useSystemSetting({ key: 'ai.remoteOllamaUrl', enabled }) + const { data: autoThinkingSetting } = useSystemSetting({ key: 'ai.autoThinking', enabled }) + // Global default for models the user hasn't explicitly toggled. Coerce defensively — KV + // booleans have historically round-tripped as strings. + const autoThinkingDefault = + autoThinkingSetting?.value === true || autoThinkingSetting?.value === 'true' const { data: remoteStatus } = useQuery({ queryKey: ['remoteOllamaStatus'], @@ -72,6 +77,38 @@ export default function Chat({ select: (data) => data || [], }) + // Per-model thinking overrides, remembered client-side (localStorage, keyed by model name). + // An entry here means the user explicitly toggled thinking for that model; absent means fall + // back to the global default (ai.autoThinking). Seeded from localStorage when models load. + const [thinkingOverrides, setThinkingOverrides] = useState>({}) + useEffect(() => { + const next: Record = {} + for (const m of installedModels) { + try { + const stored = localStorage.getItem(`nomad:thinking:${m.name}`) + if (stored !== null) next[m.name] = stored === 'true' + } catch {} + } + setThinkingOverrides(next) + }, [installedModels]) + + const selectedModelSupportsThinking = + installedModels.find((m) => m.name === selectedModel)?.thinking === true + + // Effective thinking preference for a model: explicit override wins, else the global default. + const effectiveThinking = useCallback( + (model: string): boolean => + model in thinkingOverrides ? thinkingOverrides[model] : autoThinkingDefault, + [thinkingOverrides, autoThinkingDefault] + ) + + const setModelThinking = useCallback((model: string, value: boolean) => { + setThinkingOverrides((prev) => ({ ...prev, [model]: value })) + try { + localStorage.setItem(`nomad:thinking:${model}`, String(value)) + } catch {} + }, []) + const { data: chatSuggestions, isLoading: chatSuggestionsLoading } = useQuery({ queryKey: ['chatSuggestions'], queryFn: async ({ signal }) => { @@ -102,6 +139,7 @@ export default function Chat({ model: string messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }> sessionId?: number + think?: boolean }) => api.sendChatMessage({ ...request, stream: false }), onSuccess: async (data) => { if (!data || !activeSessionId) { @@ -323,7 +361,7 @@ export default function Chat({ try { await api.streamChatMessage( - { model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, sessionId: sessionId ? Number(sessionId) : undefined }, + { model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, sessionId: sessionId ? Number(sessionId) : undefined, think: effectiveThinking(selectedModel) }, (chunkContent, chunkThinking, done) => { if (chunkThinking.length > 0 && thinkingStartTime === null) { thinkingStartTime = Date.now() @@ -414,10 +452,11 @@ export default function Chat({ model: selectedModel || 'llama3.2', messages: chatMessages, sessionId: sessionId ? Number(sessionId) : undefined, + think: effectiveThinking(selectedModel), }) } }, - [activeSessionId, messages, selectedModel, chatMutation, queryClient, streamingEnabled] + [activeSessionId, messages, selectedModel, chatMutation, queryClient, streamingEnabled, effectiveThinking] ) return ( @@ -494,6 +533,20 @@ export default function Chat({ )} + {selectedModelSupportsThinking && ( + + )} {isInModal && ( {isVisible && ( -
-
+
+
{text} -
+
)} diff --git a/admin/inertia/components/chat/index.tsx b/admin/inertia/components/chat/index.tsx index 7536be3..c8c9f39 100644 --- a/admin/inertia/components/chat/index.tsx +++ b/admin/inertia/components/chat/index.tsx @@ -12,6 +12,8 @@ import classNames from '~/lib/classNames' import { IconX } from '@tabler/icons-react' import { DEFAULT_QUERY_REWRITE_MODEL } from '../../../constants/ollama' import { useSystemSetting } from '~/hooks/useSystemSetting' +import Switch from '~/components/inputs/Switch' +import InfoTooltip from '~/components/InfoTooltip' interface ChatProps { enabled: boolean @@ -534,18 +536,19 @@ export default function Chat({ )}
{selectedModelSupportsThinking && ( -