From 7cee1da98b0d84beda5c382685858edda35b938a Mon Sep 17 00:00:00 2001 From: JHubi1 Date: Fri, 31 May 2024 22:29:03 +0200 Subject: [PATCH] Fixed multimodal Were broking after changing chat --- lib/l10n/app_de.arb | 7 +++- lib/l10n/app_en.arb | 5 +++ lib/main.dart | 23 +++++++++--- lib/worker_setter.dart | 79 +++++++++++++++++++++++++++++++++--------- 4 files changed, 93 insertions(+), 21 deletions(-) diff --git a/lib/l10n/app_de.arb b/lib/l10n/app_de.arb index cea3be9..402f868 100644 --- a/lib/l10n/app_de.arb +++ b/lib/l10n/app_de.arb @@ -30,7 +30,7 @@ "description": "First tip displayed in the sidebar", "context": "Visible in the sidebar" }, - "tip1": "Löschen Sie Nachrichten durch Doppeltippen", + "tip1": "Lösche Nachrichten durch Doppeltippen", "@tip1": { "description": "Second tip displayed in the sidebar", "context": "Visible in the sidebar" @@ -65,6 +65,11 @@ "description": "Text displayed when an image is not valid", "context": "Visible in the chat view" }, + "imageOnlyConversation": "Nur Bild Unterhaltung", + "@imageOnlyConversation": { + "description": "Title, if 'Generate Title' is executed on a conversation with no text messages", + "context": "Visible in the chat view" + }, "messageInputPlaceholder": "Nachricht", "@messageInputPlaceholder": { "description": "Placeholder text for message input", diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index cf7dbb3..1bbdda9 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -65,6 +65,11 @@ "description": "Text displayed when an image is not valid", "context": "Visible in the chat view" }, + "imageOnlyConversation": "Image Only Conversation", + "@imageOnlyConversation": { + "description": "Title, if 'Generate Title' is executed on a conversation with no text messages", + "context": "Visible in the chat view" + }, "messageInputPlaceholder": "Message", "@messageInputPlaceholder": { "description": "Placeholder text for message input", diff --git a/lib/main.dart b/lib/main.dart index 2d3ca8f..23edb4c 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -567,6 +567,11 @@ class _MainAppState extends State { color: Colors.grey[200]!, width: 1)))))); }, + imageMessageBuilder: (p0, {required messageWidth}) { + return SizedBox( + width: 160.0, + child: MarkdownBody(data: "![${p0.name}](${p0.uri})")); + }, disableImageGallery: true, // keyboardDismissBehavior: // ScrollViewKeyboardDismissBehavior.onDrag, @@ -641,9 +646,14 @@ class _MainAppState extends State { content: jsonDecode(jsonEncode(messages[i]))["text"], images: (images.isNotEmpty) ? images : null)); } else { - images.add(base64.encode( - await File(jsonDecode(jsonEncode(messages[i]))["uri"]) - .readAsBytes())); + var uri = + jsonDecode(jsonEncode(messages[i]))["uri"] as String; + String content = + (uri.startsWith("data:image/png;base64,")) + ? uri.removePrefix("data:image/png;base64,") + : base64.encode(await File(uri).readAsBytes()); + uri = uri.removePrefix("data:image/png;base64,"); + images.add(content); } } @@ -757,6 +767,10 @@ class _MainAppState extends State { void setTitle() async { List> history = []; for (var i = 0; i < messages.length; i++) { + if (jsonDecode(jsonEncode(messages[i]))["text"] == + null) { + continue; + } history.add({ "role": (messages[i].author == user) ? "user" @@ -776,7 +790,8 @@ class _MainAppState extends State { ); var title = generated.response! .replaceAll("*", "") - .replaceAll("_", ""); + .replaceAll("_", "") + .trim(); var tmp = (prefs!.getStringList("chats") ?? []); for (var i = 0; i < tmp.length; i++) { if (jsonDecode((prefs!.getStringList("chats") ?? diff --git a/lib/worker_setter.dart b/lib/worker_setter.dart index a96cb3e..f157a5e 100644 --- a/lib/worker_setter.dart +++ b/lib/worker_setter.dart @@ -1,10 +1,12 @@ import 'dart:convert'; +import 'dart:io'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_gen/gen_l10n/app_localizations.dart'; import 'main.dart'; +import 'package:dartx/dartx.dart'; import 'package:ollama_dart/ollama_dart.dart' as llama; // ignore: depend_on_referenced_packages import 'package:flutter_chat_types/flutter_chat_types.dart' as types; @@ -157,7 +159,7 @@ void setModel(BuildContext context, Function setState) { }); } -void saveChat(String uuid, Function setState) { +void saveChat(String uuid, Function setState) async { int index = -1; for (var i = 0; i < (prefs!.getStringList("chats") ?? []).length; i++) { if (jsonDecode((prefs!.getStringList("chats") ?? [])[i])["uuid"] == uuid) { @@ -167,10 +169,24 @@ void saveChat(String uuid, Function setState) { if (index == -1) return; List> history = []; for (var i = 0; i < messages.length; i++) { - history.add({ - "role": (messages[i].author == user) ? "user" : "assistant", - "content": jsonDecode(jsonEncode(messages[i]))["text"] - }); + if ((jsonDecode(jsonEncode(messages[i])) as Map).containsKey("text")) { + history.add({ + "role": (messages[i].author == user) ? "user" : "assistant", + "content": jsonDecode(jsonEncode(messages[i]))["text"] + }); + } else { + var uri = jsonDecode(jsonEncode(messages[i]))["uri"] as String; + String content = (uri.startsWith("data:image/png;base64,")) + ? uri.removePrefix("data:image/png;base64,") + : base64.encode(await File(uri).readAsBytes()); + history.add({ + "role": (messages[i].author == user) ? "user" : "assistant", + "type": "image", + "name": (messages[i] as types.ImageMessage).name, + "size": (messages[i] as types.ImageMessage).size.toString(), + "content": content + }); + } } if (messages.isEmpty && uuid == chatUuid) { for (var i = 0; i < (prefs!.getStringList("chats") ?? []).length; i++) { @@ -236,12 +252,24 @@ void loadChat(String uuid, Function setState) { jsonDecode((prefs!.getStringList("chats") ?? [])[index])["messages"]); for (var i = 0; i < history.length; i++) { if (history[i]["role"] != "system") { - messages.insert( - 0, - types.TextMessage( - author: (history[i]["role"] == "user") ? user : assistant, - id: const Uuid().v4(), - text: history[i]["content"])); + if ((history[i] as Map).containsKey("type") && + history[i]["type"] == "image") { + messages.insert( + 0, + types.ImageMessage( + author: (history[i]["role"] == "user") ? user : assistant, + id: const Uuid().v4(), + name: history[i]["name"], + size: int.parse(history[i]["size"]), + uri: "data:image/png;base64,${history[i]["content"]}")); + } else { + messages.insert( + 0, + types.TextMessage( + author: (history[i]["role"] == "user") ? user : assistant, + id: const Uuid().v4(), + text: history[i]["content"])); + } } } model = jsonDecode((prefs!.getStringList("chats") ?? [])[index])["model"]; @@ -325,11 +353,29 @@ Future prompt(BuildContext context, [])[i])["uuid"] == uuid) { try { - var history = jsonDecode( - jsonDecode((prefs! - .getStringList( - "chats") ?? + List history = []; + var tmp = jsonDecode(jsonDecode( + (prefs!.getStringList( + "chats") ?? [])[i])["messages"]); + for (var j = 0; + j < tmp.length; + j++) { + if (tmp[j]["text"] == null) { + continue; + } + history.add(tmp[j]["text"]); + } + if (history.isEmpty) { + controller + .text = AppLocalizations + .of(context)! + .imageOnlyConversation; + setLocalState(() { + loading = false; + }); + return; + } final generated = await llama.OllamaClient( @@ -345,7 +391,8 @@ Future prompt(BuildContext context, ); var title = generated.response! .replaceAll("*", "") - .replaceAll("_", ""); + .replaceAll("_", "") + .trim(); controller.text = title; setLocalState(() { loading = false;