diff --git a/admin/app/controllers/settings_controller.ts b/admin/app/controllers/settings_controller.ts index bab7c5e..f777943 100644 --- a/admin/app/controllers/settings_controller.ts +++ b/admin/app/controllers/settings_controller.ts @@ -71,6 +71,7 @@ export default class SettingsController { const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl') const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention') const autoThinking = await KVStore.getValue('ai.autoThinking') + const tasksModel = await KVStore.getValue('ai.tasksModel') return inertia.render('settings/models', { models: { availableModels: availableModels?.models || [], @@ -81,6 +82,7 @@ export default class SettingsController { remoteOllamaUrl: remoteOllamaUrl ?? '', ollamaFlashAttention: ollamaFlashAttention ?? true, autoThinking: autoThinking ?? false, + tasksModel: tasksModel ?? '', }, }, }) diff --git a/admin/app/services/chat_service.ts b/admin/app/services/chat_service.ts index a5d07c9..7ac41ea 100644 --- a/admin/app/services/chat_service.ts +++ b/admin/app/services/chat_service.ts @@ -6,12 +6,64 @@ import { DateTime } from 'luxon' import { inject } from '@adonisjs/core' import { OllamaService } from './ollama_service.js' import { SYSTEM_PROMPTS } from '../../constants/ollama.js' -import { toTitleCase } from '../utils/misc.js' +import { pickTasksModel, toTitleCase } from '../utils/misc.js' @inject() export class ChatService { constructor(private ollamaService: OllamaService) {} + /** + * The model to use for ancillary work — chat titles and chat suggestions. + * + * Prefers the user's `ai.tasksModel` setting so a 30B reasoning model isn't + * spending seconds "thinking" to produce a three-word sidebar title. Falls + * back to `fallback` when the setting is unset (the default, which preserves + * the previous behaviour) or when the configured model is no longer + * installed. `installed` is passed in by callers that already listed models, + * to avoid a second round-trip. + */ + private async resolveTasksModel( + fallback: string | null, + installed?: { name: string }[] + ): Promise { + let configured: string | null = null + try { + configured = await KVStore.getValue('ai.tasksModel') + } catch (error) { + logger.error( + `[ChatService] Failed to read ai.tasksModel: ${error instanceof Error ? error.message : error}` + ) + return fallback + } + if (!configured?.trim()) { + return fallback + } + + let models = installed + if (!models) { + try { + models = await this.ollamaService.getModels() + } catch (error) { + logger.error( + `[ChatService] Failed to list models while resolving the tasks model: ${error instanceof Error ? error.message : error}` + ) + return fallback + } + } + + const { model, staleConfigured } = pickTasksModel( + configured, + (models ?? []).map((m) => m.name), + fallback + ) + if (staleConfigured) { + logger.warn( + `[ChatService] Configured tasks model "${staleConfigured}" is not installed; falling back to "${fallback ?? 'none'}"` + ) + } + return model + } + async getAllSessions() { try { const sessions = await ChatSession.query().orderBy('updated_at', 'desc') @@ -37,11 +89,12 @@ export class ChatService { return [] // If no models are available, return empty suggestions } - // Prefer the user's selected chat model. Fall back to the smallest + // The user's dedicated tasks model wins when set — suggestions are short + // aesthetic prompts that don't benefit from a flagship model. Otherwise + // prefer the user's selected chat model, and fall back to the smallest // installed model — picking the largest by file size is unsafe: if any // installed model exceeds available VRAM (e.g. llama3.1:405b on a 96 GB // GPU), Ollama spends minutes trying to load it and the request 500s. - // Suggestions are short prompts that don't benefit from a flagship model. const lastModel = await KVStore.getValue('chat.lastModel') const preferred = lastModel ? models.find((m) => m.name === lastModel) : undefined const chosen = @@ -52,8 +105,10 @@ export class ChatService { return [] } + const model = (await this.resolveTasksModel(chosen.name, models)) ?? chosen.name + const response = await this.ollamaService.chat({ - model: chosen.name, + model, messages: [ { role: 'user', @@ -243,8 +298,12 @@ export class ChatService { try { let title: string + // Titles are aesthetic work; route them to the tasks model when one is + // configured rather than the chat model that just answered. + const titleModel = (await this.resolveTasksModel(model)) ?? model + const response = await this.ollamaService.chat({ - model, + model: titleModel, messages: [ { role: 'system', content: SYSTEM_PROMPTS.title_generation }, { role: 'user', content: userMessage }, diff --git a/admin/app/utils/misc.ts b/admin/app/utils/misc.ts index a81f33d..46faca2 100644 --- a/admin/app/utils/misc.ts +++ b/admin/app/utils/misc.ts @@ -12,6 +12,30 @@ export function toTitleCase(str: string): string { .join(' ') } +/** + * Decide which model runs an ancillary AI task (chat titles, chat suggestions). + * + * `configured` is the user's `ai.tasksModel` setting. It only wins when the + * model is still installed — a model can be deleted from /settings/models long + * after it was picked here, and a request for a missing model 404s. In every + * other case the caller's existing `fallback` applies, so an unset setting + * leaves prior behaviour untouched. + */ +export function pickTasksModel( + configured: string | null | undefined, + installedNames: string[], + fallback: string | null +): { model: string | null; staleConfigured: string | null } { + const trimmed = configured?.trim() + if (!trimmed) { + return { model: fallback, staleConfigured: null } + } + if (installedNames.includes(trimmed)) { + return { model: trimmed, staleConfigured: null } + } + return { model: fallback, staleConfigured: trimmed } +} + export function parseBoolean(value: any): boolean { if (typeof value === 'boolean') return value if (typeof value === 'string') { diff --git a/admin/constants/kv_store.ts b/admin/constants/kv_store.ts index 80e0a54..eb3e718 100644 --- a/admin/constants/kv_store.ts +++ b/admin/constants/kv_store.ts @@ -11,6 +11,7 @@ export const SETTINGS_KEYS: KVStoreKey[] = [ 'ai.remoteOllamaUrl', 'ai.ollamaFlashAttention', 'ai.autoThinking', + 'ai.tasksModel', 'rag.defaultIngestPolicy', 'autoUpdate.enabled', 'autoUpdate.windowStart', diff --git a/admin/inertia/pages/settings/models.tsx b/admin/inertia/pages/settings/models.tsx index 9addb05..a9de243 100644 --- a/admin/inertia/pages/settings/models.tsx +++ b/admin/inertia/pages/settings/models.tsx @@ -13,6 +13,7 @@ import StyledModal from '~/components/StyledModal' import type { NomadInstalledModel } from '../../../types/ollama' import { SERVICE_NAMES } from '../../../constants/service_names' import Switch from '~/components/inputs/Switch' +import Select from '~/components/inputs/Select' import StyledSectionHeader from '~/components/StyledSectionHeader' import { useMutation, useQuery } from '@tanstack/react-query' import Input from '~/components/inputs/Input' @@ -26,7 +27,7 @@ export default function ModelsPage(props: { models: { availableModels: NomadOllamaModel[] installedModels: NomadInstalledModel[] - settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean } + settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string } } }) { const { aiAssistantName } = usePage<{ aiAssistantName: string }>().props @@ -99,6 +100,7 @@ export default function ModelsPage(props: { props.models.settings.ollamaFlashAttention ) const [autoThinking, setAutoThinking] = useState(props.models.settings.autoThinking) + const [tasksModel, setTasksModel] = useState(props.models.settings.tasksModel) const [aiAssistantCustomName, setAiAssistantCustomName] = useState( props.models.settings.aiAssistantCustomName ) @@ -241,6 +243,17 @@ export default function ModelsPage(props: { ) } + // A model can be deleted after being picked here. Surface the stale name as a + // disabled option instead of letting the select silently render empty — the + // backend already falls back to the chat model at call time. + const tasksModelOptions = [ + { value: '', label: 'Use the chat model' }, + ...props.models.installedModels.map((model) => ({ value: model.name, label: model.name })), + ...(tasksModel && !props.models.installedModels.some((m) => m.name === tasksModel) + ? [{ value: tasksModel, label: `${tasksModel} (not installed)`, disabled: true }] + : []), + ] + const updateSettingMutation = useMutation({ mutationFn: async ({ key, value }: { key: string; value: boolean | string }) => { return await api.updateSetting(key, value) @@ -344,6 +357,17 @@ export default function ModelsPage(props: { }) } /> +