RomanNum3ral
/
project-nomad
mirror of https://github.com/Crosstalk-Solutions/project-nomad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(AI): per-model thinking toggle with global default (off) 

Stop forcing thinking on for every capable model. Adds a global default
(ai.autoThinking, ships OFF) and a per-model override in the chat window,
shown only for thinking-capable models and remembered client-side.

The /v1 (OpenAI-compat) endpoint ignores `think`; reasoning_effort is the
real lever. The controller resolves per-request preference -> global default
-> OFF (gated on capability), and the service maps that to reasoning_effort
('none' to suppress on a capable model, 'medium' for gpt-oss, unset to let a
capable model default thinking on). A new thinkingCapable flag keeps
non-Ollama backends from ever receiving reasoning_effort.

installed-models is enriched with a `thinking` flag (checkModelHasThinking,
now memoized) so the picker knows which models get the toggle.

Stacks on #1078 (reasoning-field read + client-disconnect abort).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chris Sherwood 2026-07-10 23:04:32 -07:00
parent 8d4b8a65f6
commit d64cadafc2
9 changed files with 133 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
admin/app/controllers/ollama_controller.ts
View File

@ -144,13 +144,21 @@ export default class OllamaController {
logger.debug(`[OllamaController] Large system prompt (~${estimatedSystemTokens} tokens), requesting num_ctx: ${numCtx}`)
}
// Check if the model supports "thinking" capability for enhanced response generation
// Check if the model supports "thinking" capability for enhanced response generation.
// Thinking is only enabled when the model supports it AND the user wants it: the explicit
// per-request preference wins, otherwise the global default (ai.autoThinking, default OFF).
// If gpt-oss model, it requires a text param for "think" https://docs.ollama.com/api/chat
const thinkingCapability = await this.ollamaService.checkModelHasThinking(reqData.model)
const think: boolean | 'medium' = thinkingCapability ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false
let thinkingEnabled = false
if (thinkingCapability) {
thinkingEnabled = reqData.think ?? ((await KVStore.getValue('ai.autoThinking')) ?? false)
}
const think: boolean | 'medium' =
thinkingEnabled ? (reqData.model.startsWith('gpt-oss') ? 'medium' : true) : false
// Separate sessionId from the Ollama request payload — Ollama rejects unknown fields
const { sessionId, ...ollamaRequest } = reqData
// Separate sessionId and the resolved thinking preference from the Ollama request payload —
// Ollama rejects unknown fields, and `think` is re-derived above (not forwarded raw).
const { sessionId, think: _thinkPref, ...ollamaRequest } = reqData
// Save user message to DB before streaming if sessionId provided
let userContent: string | null = null
@ -173,6 +181,7 @@ export default class OllamaController {
const stream = await this.ollamaService.chatStream({
...ollamaRequest,
think,
thinkingCapable: thinkingCapability,
numCtx,
signal: abortController.signal,
})
@ -207,7 +216,7 @@ export default class OllamaController {
}
// Non-streaming (legacy) path
const result = await this.ollamaService.chat({ ...ollamaRequest, think, numCtx })
const result = await this.ollamaService.chat({ ...ollamaRequest, think, thinkingCapable: thinkingCapability, numCtx })
if (sessionId && result?.message?.content) {
await this.chatService.addMessage(sessionId, 'assistant', result.message.content)
@ -387,7 +396,14 @@ export default class OllamaController {
}
async installedModels({ }: HttpContext) {
return await this.ollamaService.getModels()
const models = await this.ollamaService.getModels()
// Enrich each model with its thinking capability so the chat picker knows which models
// to show the per-model thinking toggle for. checkModelHasThinking memoizes /api/show
// results, so this stays cheap on repeat loads. Best-effort per model.
const thinking = await Promise.all(
models.map((m) => this.ollamaService.checkModelHasThinking(m.name))
)
return models.map((m, i) => ({ ...m, thinking: thinking[i] }))
}
/**

2
admin/app/controllers/settings_controller.ts
View File

@ -66,6 +66,7 @@ export default class SettingsController {
const aiAssistantCustomName = await KVStore.getValue('ai.assistantCustomName')
const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl')
const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention')
const autoThinking = await KVStore.getValue('ai.autoThinking')
return inertia.render('settings/models', {
models: {
availableModels: availableModels?.models || [],
@ -75,6 +76,7 @@ export default class SettingsController {
aiAssistantCustomName: aiAssistantCustomName ?? '',
remoteOllamaUrl: remoteOllamaUrl ?? '',
ollamaFlashAttention: ollamaFlashAttention ?? true,
autoThinking: autoThinking ?? false,
},
},
})

35
admin/app/services/ollama_service.ts
View File

@ -43,6 +43,9 @@ type ChatInput = {
model: string
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>
think?: boolean | 'medium'
// Whether the target model supports thinking. Lets chat()/chatStream() tell "capable but
// disabled" (send reasoning_effort:'none') apart from "not capable" (send nothing).
thinkingCapable?: boolean
stream?: boolean
numCtx?: number
// Aborts the upstream request when the client disconnects, so an abandoned generation
@ -57,6 +60,9 @@ export class OllamaService {
private initPromise: Promise<void> | null = null
private isOllamaNative: boolean | null = null
private activeDownloads: Map<string, Promise<{ success: boolean; message: string; retryable?: boolean }>> = new Map()
// Memoized `thinking` capability per model name (see checkModelHasThinking). Only successful
// /api/show lookups are cached; transient failures are left uncached so they can be retried.
private thinkingCapabilityCache: Map<string, boolean> = new Map()
constructor() {}
@ -332,6 +338,15 @@ export class OllamaService {
if (chatRequest.think) {
params.think = chatRequest.think
}
// The /v1 (OpenAI-compat) endpoint ignores `think`; `reasoning_effort` is the actual lever.
// Only touch it for thinking-capable models so non-Ollama backends never get an unexpected
// param. gpt-oss requires an explicit level; a capable-but-disabled model gets 'none' to
// suppress thinking (capable models default thinking ON otherwise, so think===true is a no-op).
if (chatRequest.think === 'medium') {
params.reasoning_effort = 'medium'
} else if (chatRequest.thinkingCapable && chatRequest.think === false) {
params.reasoning_effort = 'none'
}
if (chatRequest.numCtx) {
params.num_ctx = chatRequest.numCtx
}
@ -365,6 +380,15 @@ export class OllamaService {
if (chatRequest.think) {
params.think = chatRequest.think
}
// The /v1 (OpenAI-compat) endpoint ignores `think`; `reasoning_effort` is the actual lever.
// Only touch it for thinking-capable models so non-Ollama backends never get an unexpected
// param. gpt-oss requires an explicit level; a capable-but-disabled model gets 'none' to
// suppress thinking (capable models default thinking ON otherwise, so think===true is a no-op).
if (chatRequest.think === 'medium') {
params.reasoning_effort = 'medium'
} else if (chatRequest.thinkingCapable && chatRequest.think === false) {
params.reasoning_effort = 'none'
}
if (chatRequest.numCtx) {
params.num_ctx = chatRequest.numCtx
}
@ -444,13 +468,22 @@ export class OllamaService {
await this._ensureDependencies()
if (!this.baseUrl) return false
// A model's capabilities don't change at runtime, so memoize the /api/show result. Without
// this, loading the chat picker fires one /api/show per installed model and every chat send
// fires another — this collapses those to a single call per model per process.
const cached = this.thinkingCapabilityCache.get(modelName)
if (cached !== undefined) return cached
try {
const response = await axios.post(
`${this.baseUrl}/api/show`,
{ model: modelName },
{ timeout: 5000 }
)
return Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking')
const hasThinking =
Array.isArray(response.data?.capabilities) && response.data.capabilities.includes('thinking')
this.thinkingCapabilityCache.set(modelName, hasThinking)
return hasThinking
} catch {
// Non-Ollama backends don't expose /api/show — assume no thinking support
return false

3
admin/app/validators/ollama.ts
View File

@ -11,6 +11,9 @@ export const chatSchema = vine.compile(
),
stream: vine.boolean().optional(),
sessionId: vine.number().positive().optional(),
// Effective per-request thinking preference (per-model override or global default),
// resolved client-side. Omitted -> server falls back to the ai.autoThinking KV default.
think: vine.boolean().optional(),
})
)

2
admin/constants/kv_store.ts
View File

@ -1,3 +1,3 @@
import { KVStoreKey } from "../types/kv_store.js";
export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow'];
export const SETTINGS_KEYS: KVStoreKey[] = ['chat.suggestionsEnabled', 'chat.lastModel', 'ui.hasVisitedEasySetup', 'ui.theme', 'system.earlyAccess', 'system.internetStatusTestUrl', 'ai.assistantCustomName', 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'ai.autoThinking', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', 'autoUpdate.windowEnd', 'autoUpdate.cooloffHours', 'appAutoUpdate.enabled', 'contentAutoUpdate.enabled', 'contentAutoUpdate.windowStart', 'contentAutoUpdate.windowEnd', 'contentAutoUpdate.cooloffHours', 'contentAutoUpdate.maxBytesPerWindow'];

57
admin/inertia/components/chat/index.tsx
View File

@ -57,6 +57,11 @@ export default function Chat({
const { data: lastModelSetting } = useSystemSetting({ key: 'chat.lastModel', enabled })
const { data: remoteOllamaUrlSetting } = useSystemSetting({ key: 'ai.remoteOllamaUrl', enabled })
const { data: autoThinkingSetting } = useSystemSetting({ key: 'ai.autoThinking', enabled })
// Global default for models the user hasn't explicitly toggled. Coerce defensively — KV
// booleans have historically round-tripped as strings.
const autoThinkingDefault =
autoThinkingSetting?.value === true || autoThinkingSetting?.value === 'true'
const { data: remoteStatus } = useQuery({
queryKey: ['remoteOllamaStatus'],
@ -72,6 +77,38 @@ export default function Chat({
select: (data) => data || [],
})
// Per-model thinking overrides, remembered client-side (localStorage, keyed by model name).
// An entry here means the user explicitly toggled thinking for that model; absent means fall
// back to the global default (ai.autoThinking). Seeded from localStorage when models load.
const [thinkingOverrides, setThinkingOverrides] = useState<Record<string, boolean>>({})
useEffect(() => {
const next: Record<string, boolean> = {}
for (const m of installedModels) {
try {
const stored = localStorage.getItem(`nomad:thinking:${m.name}`)
if (stored !== null) next[m.name] = stored === 'true'
} catch {}
}
setThinkingOverrides(next)
}, [installedModels])
const selectedModelSupportsThinking =
installedModels.find((m) => m.name === selectedModel)?.thinking === true
// Effective thinking preference for a model: explicit override wins, else the global default.
const effectiveThinking = useCallback(
(model: string): boolean =>
model in thinkingOverrides ? thinkingOverrides[model] : autoThinkingDefault,
[thinkingOverrides, autoThinkingDefault]
)
const setModelThinking = useCallback((model: string, value: boolean) => {
setThinkingOverrides((prev) => ({ ...prev, [model]: value }))
try {
localStorage.setItem(`nomad:thinking:${model}`, String(value))
} catch {}
}, [])
const { data: chatSuggestions, isLoading: chatSuggestionsLoading } = useQuery<string[]>({
queryKey: ['chatSuggestions'],
queryFn: async ({ signal }) => {
@ -102,6 +139,7 @@ export default function Chat({
model: string
messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>
sessionId?: number
think?: boolean
}) => api.sendChatMessage({ ...request, stream: false }),
onSuccess: async (data) => {
if (!data || !activeSessionId) {
@ -323,7 +361,7 @@ export default function Chat({
try {
await api.streamChatMessage(
{ model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, sessionId: sessionId ? Number(sessionId) : undefined },
{ model: selectedModel || 'llama3.2', messages: chatMessages, stream: true, sessionId: sessionId ? Number(sessionId) : undefined, think: effectiveThinking(selectedModel) },
(chunkContent, chunkThinking, done) => {
if (chunkThinking.length > 0 && thinkingStartTime === null) {
thinkingStartTime = Date.now()
@ -414,10 +452,11 @@ export default function Chat({
model: selectedModel || 'llama3.2',
messages: chatMessages,
sessionId: sessionId ? Number(sessionId) : undefined,
think: effectiveThinking(selectedModel),
})
}
},
[activeSessionId, messages, selectedModel, chatMutation, queryClient, streamingEnabled]
[activeSessionId, messages, selectedModel, chatMutation, queryClient, streamingEnabled, effectiveThinking]
)
return (
@ -494,6 +533,20 @@ export default function Chat({
</select>
)}
</div>
{selectedModelSupportsThinking && (
<label
className="flex items-center gap-1.5 text-sm text-text-secondary cursor-pointer select-none"
title="When on, this model reasons before answering. Remembered for this model."
>
<input
type="checkbox"
checked={effectiveThinking(selectedModel)}
onChange={(e) => setModelThinking(selectedModel, e.target.checked)}
className="h-4 w-4 rounded border-border-default text-desert-green focus:ring-desert-green"
/>
Thinking
</label>
)}
{isInModal && (
<button
onClick={() => {

12
admin/inertia/pages/settings/models.tsx
View File

@ -26,7 +26,7 @@ export default function ModelsPage(props: {
models: {
availableModels: NomadOllamaModel[]
installedModels: NomadInstalledModel[]
settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean }
settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean }
}
}) {
const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props
@ -98,6 +98,7 @@ export default function ModelsPage(props: {
const [ollamaFlashAttention, setOllamaFlashAttention] = useState(
props.models.settings.ollamaFlashAttention
)
const [autoThinking, setAutoThinking] = useState(props.models.settings.autoThinking)
const [aiAssistantCustomName, setAiAssistantCustomName] = useState(
props.models.settings.aiAssistantCustomName
)
@ -320,6 +321,15 @@ export default function ModelsPage(props: {
label="Flash Attention"
description="Enables OLLAMA_FLASH_ATTENTION=1 for improved memory efficiency. Disable if you experience instability. Takes effect after reinstalling the AI Assistant."
/>
<Switch
checked={autoThinking}
onChange={(newVal) => {
setAutoThinking(newVal)
updateSettingMutation.mutate({ key: 'ai.autoThinking', value: newVal })
}}
label="Use thinking automatically when a model supports it"
description="Sets the default for models that can think. You can still turn thinking on or off for an individual model in the chat window."
/>
<Input
name="aiAssistantCustomName"
label="Assistant Name"

1
admin/types/kv_store.ts
View File

@ -38,6 +38,7 @@ export const KV_STORE_SCHEMA = {
'gpu.type': 'string',
'ai.remoteOllamaUrl': 'string',
'ai.ollamaFlashAttention': 'boolean',
'ai.autoThinking': 'boolean',
'ai.amdGpuAcceleration': 'boolean',
'ai.amdHsaOverride': 'string',
'ai.autoFixGpuPassthrough': 'boolean',

4
admin/types/ollama.ts
View File

@ -33,6 +33,8 @@ export type OllamaChatRequest = {
messages: OllamaChatMessage[]
stream?: boolean
sessionId?: number
// Effective thinking preference for this request (per-model override or global default).
think?: boolean
}
export type OllamaChatResponse = {
@ -50,6 +52,8 @@ export type NomadInstalledModel = {
size: number
digest?: string
details?: Record<string, any>
// Whether the model supports "thinking" (set by the installed-models endpoint enrichment).
thinking?: boolean
}
export type NomadChatResponse = {