From 4d90f8611b9d1b642daa4dc6afd4f0549a8d78a1 Mon Sep 17 00:00:00 2001 From: JHubi1 Date: Tue, 25 Jun 2024 20:14:25 +0200 Subject: [PATCH] Added model preloading --- lib/l10n/app_en.arb | 5 +++ lib/settings/interface.dart | 20 +++++++---- lib/worker/setter.dart | 72 +++++++++++++++++++++++++++++++------ 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 1ec6bd4..6050513 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -287,6 +287,11 @@ "description": "Text displayed as description for show model tags toggle", "context": "Visible in the settings view" }, + "settingsPreloadModels": "Preload models", + "@settingsPreloadModels": { + "description": "Text displayed as description for preload models toggle", + "context": "Visible in the settings view" + }, "settingsResetOnModelChange": "Reset on model change", "@settingsResetOnModelChange": { "description": "Text displayed as description for reset on model change toggle", diff --git a/lib/settings/interface.dart b/lib/settings/interface.dart index 1e4b865..a5213e0 100644 --- a/lib/settings/interface.dart +++ b/lib/settings/interface.dart @@ -27,13 +27,11 @@ class _ScreenSettingsInterfaceState extends State { color: Theme.of(context).colorScheme.surface, child: Scaffold( appBar: AppBar( - title: Row(children: [ - Text(AppLocalizations.of(context)!.settingsTitleInterface), - Expanded(child: SizedBox(height: 200, child: MoveWindow())) - ]), - actions: - desktopControlsActions(context) - ), + title: Row(children: [ + Text(AppLocalizations.of(context)!.settingsTitleInterface), + Expanded(child: SizedBox(height: 200, child: MoveWindow())) + ]), + actions: desktopControlsActions(context)), body: Padding( padding: const EdgeInsets.only(left: 16, right: 16), child: Column(children: [ @@ -48,6 +46,14 @@ class _ScreenSettingsInterfaceState extends State { prefs!.setBool("modelTags", value); setState(() {}); }), + toggle( + context, + AppLocalizations.of(context)!.settingsPreloadModels, + (prefs!.getBool("preloadModel") ?? true), (value) { + selectionHaptic(); + prefs!.setBool("preloadModel", value); + setState(() {}); + }), toggle( context, AppLocalizations.of(context)! diff --git a/lib/worker/setter.dart b/lib/worker/setter.dart index 6752253..998e44f 100644 --- a/lib/worker/setter.dart +++ b/lib/worker/setter.dart @@ -1,5 +1,6 @@ import 'dart:convert'; import 'dart:io'; + import 'package:flutter/material.dart'; import 'package:flutter_gen/gen_l10n/app_localizations.dart'; @@ -12,6 +13,7 @@ import 'package:ollama_dart/ollama_dart.dart' as llama; // ignore: depend_on_referenced_packages import 'package:flutter_chat_types/flutter_chat_types.dart' as types; import 'package:uuid/uuid.dart'; +import 'package:http/http.dart' as http; void setModel(BuildContext context, Function setState) { List models = []; @@ -31,7 +33,7 @@ void setModel(BuildContext context, Function setState) { .cast(), baseUrl: "$host/api") .listModels() - .timeout(const Duration(seconds: 5)); + .timeout(const Duration(seconds: 10)); for (var i = 0; i < list.models!.length; i++) { models.add(list.models![i].model!.split(":")[0]); modelsReal.add(list.models![i].model!); @@ -74,14 +76,21 @@ void setModel(BuildContext context, Function setState) { var content = StatefulBuilder(builder: (context, setLocalState) { setModalState = setLocalState; return PopScope( - canPop: loaded, - onPopInvoked: (didPop) { + canPop: false, + onPopInvoked: (didPop) async { if (!loaded) return; - if (usedIndex >= 0 && - modelsReal[usedIndex] != model && - (prefs!.getBool("resetOnModelSelect") ?? true)) { - messages = []; - chatUuid = null; + loaded = false; + if (usedIndex >= 0 && modelsReal[usedIndex] != model) { + if (prefs!.getBool("resetOnModelSelect") ?? true) { + messages = []; + chatUuid = null; + } + } else { + setState(() { + desktopTitleVisible = true; + }); + Navigator.of(context).pop(); + return; } model = (usedIndex >= 0) ? modelsReal[usedIndex] : null; chatAllowed = !(model == null); @@ -92,9 +101,50 @@ void setModel(BuildContext context, Function setState) { prefs?.remove("model"); } prefs?.setBool("multimodal", multimodal); - setState(() { - desktopTitleVisible = true; - }); + + if (model != null && + int.parse(prefs!.getString("keepAlive") ?? "300") != 0 && (prefs!.getBool("preloadModel") ?? true)) { + setLocalState(() {}); + try { + // don't use llama client, package doesn't support just loading without content + await http + .post( + Uri.parse("$host/api/generate"), + headers: { + "Content-Type": "application/json", + ...(jsonDecode(prefs!.getString("hostHeaders") ?? "{}") + as Map) + }.cast(), + body: jsonEncode({ + "model": model!, + "keep_alive": + int.parse(prefs!.getString("keepAlive") ?? "300") + }), + ) + .timeout(const Duration(seconds: 15)); + } catch (_) { + // ignore: use_build_context_synchronously + ScaffoldMessenger.of(context).showSnackBar(SnackBar( + // ignore: use_build_context_synchronously + content: Text(AppLocalizations.of(context)! + .settingsHostInvalid("timeout")), + showCloseIcon: true)); + setState(() { + model = null; + chatAllowed = false; + }); + } + setState(() { + desktopTitleVisible = true; + }); + // ignore: use_build_context_synchronously + Navigator.of(context).pop(); + } else { + setState(() { + desktopTitleVisible = true; + }); + Navigator.of(context).pop(); + } }, child: Container( width: desktopLayout(context) ? null : double.infinity,