From f48bda178aca0b814c3fb348ab31dcd67384e628 Mon Sep 17 00:00:00 2001 From: Infi Date: Sat, 9 Mar 2024 19:29:17 +0100 Subject: [PATCH] feat: cache servers and channels locally Signed-off-by: Infi --- .idea/.gitignore | 2 + app/build.gradle | 18 ++- .../chat/revolt/activities/MainActivity.kt | 2 + .../main/java/chat/revolt/api/RevoltAPI.kt | 90 +++++++++++++++ .../java/chat/revolt/api/internals/Roles.kt | 5 +- .../revolt/api/realtime/RealtimeSocket.kt | 108 +++++++++++++++++- .../revolt/components/generic/UserAvatar.kt | 4 +- .../chat/revolt/persistence/SqlStorage.kt | 13 +++ .../screens/settings/DebugSettingsScreen.kt | 47 ++++++++ .../chat/revolt/persistence/cache/Channel.sq | 50 ++++++++ .../chat/revolt/persistence/cache/Server.sq | 30 +++++ 11 files changed, 360 insertions(+), 9 deletions(-) create mode 100644 app/src/main/java/chat/revolt/persistence/SqlStorage.kt create mode 100644 app/src/main/sqldelight/chat/revolt/persistence/cache/Channel.sq create mode 100644 app/src/main/sqldelight/chat/revolt/persistence/cache/Server.sq diff --git a/.idea/.gitignore b/.idea/.gitignore index e84d23c7..8397494f 100644 --- a/.idea/.gitignore +++ b/.idea/.gitignore @@ -4,3 +4,5 @@ /kotlinc.xml /appInsightsSettings.xml /other.xml +# GitHub Copilot persisted chat sessions +/copilot/chatSessions diff --git a/app/build.gradle b/app/build.gradle index 4e5b8f45..0107a590 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -7,6 +7,7 @@ plugins { id 'com.google.devtools.ksp' id 'org.jmailen.kotlinter' id "io.sentry.android.gradle" version "3.4.2" + id "app.cash.sqldelight" version "2.0.1" id 'kotlin-kapt' id 'kotlin-parcelize' @@ -213,7 +214,7 @@ dependencies { // Glide - Image Loading implementation "com.github.bumptech.glide:glide:$glide_version" - implementation "com.github.bumptech.glide:compose:1.0.0-beta1-SNAPSHOT" + implementation "com.github.bumptech.glide:compose:1.0.0-beta01" ksp "com.github.bumptech.glide:ksp:$glide_version" // AboutLibraries - automated OSS library attribution @@ -230,8 +231,6 @@ dependencies { implementation 'androidx.documentfile:documentfile:1.0.1' implementation "androidx.browser:browser:1.7.0" implementation "androidx.webkit:webkit:1.9.0" - implementation "androidx.datastore:datastore-preferences:1.1.0-alpha07" - implementation "androidx.datastore:datastore:1.1.0-alpha07" implementation "androidx.core:core-splashscreen:1.0.1" // Libraries used for legacy View-based UI @@ -255,4 +254,17 @@ dependencies { implementation "com.github.skydoves:colorpicker-compose:1.0.5" implementation "me.saket.telephoto:zoomable-image:1.0.0-alpha02" implementation "me.saket.telephoto:zoomable-image-glide:1.0.0-alpha02" + + // Persistence + implementation "app.cash.sqldelight:android-driver:2.0.1" + implementation "androidx.datastore:datastore:1.1.0-beta02" + implementation "androidx.datastore:datastore-preferences:1.1.0-beta02" +} + +sqldelight { + databases { + Database { + packageName = "chat.revolt.persistence" + } + } } \ No newline at end of file diff --git a/app/src/main/java/chat/revolt/activities/MainActivity.kt b/app/src/main/java/chat/revolt/activities/MainActivity.kt index 87ccf909..5a94b959 100644 --- a/app/src/main/java/chat/revolt/activities/MainActivity.kt +++ b/app/src/main/java/chat/revolt/activities/MainActivity.kt @@ -244,6 +244,8 @@ class MainActivity : FragmentActivity() { } } + RevoltAPI.hydrateFromPersistentCache() + setContent { val windowSizeClass = calculateWindowSizeClass(this) AppEntrypoint( diff --git a/app/src/main/java/chat/revolt/api/RevoltAPI.kt b/app/src/main/java/chat/revolt/api/RevoltAPI.kt index 0b6da651..4f9a536b 100644 --- a/app/src/main/java/chat/revolt/api/RevoltAPI.kt +++ b/app/src/main/java/chat/revolt/api/RevoltAPI.kt @@ -9,11 +9,15 @@ import chat.revolt.api.internals.Members import chat.revolt.api.realtime.DisconnectionState import chat.revolt.api.realtime.RealtimeSocket import chat.revolt.api.routes.user.fetchSelf +import chat.revolt.api.schemas.AutumnResource +import chat.revolt.api.schemas.ChannelType import chat.revolt.api.schemas.Emoji import chat.revolt.api.schemas.Message import chat.revolt.api.schemas.Server import chat.revolt.api.schemas.User import chat.revolt.api.unreads.Unreads +import chat.revolt.persistence.Database +import chat.revolt.persistence.SqlStorage import io.ktor.client.HttpClient import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.DefaultRequest @@ -25,6 +29,7 @@ import io.ktor.client.plugins.logging.Logging import io.ktor.client.plugins.websocket.WebSockets import io.ktor.client.request.header import io.ktor.serialization.kotlinx.json.json +import io.sentry.Sentry import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers @@ -138,6 +143,8 @@ object RevoltAPI { private var socketCoroutine: Job? = null + private var openForLocalHydration = true + fun setSessionHeader(token: String) { sessionToken = token } @@ -231,6 +238,8 @@ object RevoltAPI { socketCoroutine?.cancel() mainHandler.removeCallbacksAndMessages(null) + + clearPersistentCache() } /** @@ -245,6 +254,87 @@ object RevoltAPI { false } } + + /** + * Hydrate caches from a local database. + */ + fun hydrateFromPersistentCache() { + if (!openForLocalHydration) { + Log.w("RevoltAPI", "Hydration is closed, but was called") + // Stale data is worst case, let's track it even in prod + Sentry.captureMessage("Local hydration called twice or after real data was fetched") + return + } + + val db = Database(SqlStorage.driver) + + val channels = db.channelQueries.selectAll().executeAsList().map { + ChannelSchema( + id = it.id, + channelType = try { + ChannelType.valueOf(it.channelType) + } catch (e: Exception) { + null + }, + user = it.userId, + name = it.name, + owner = it.owner, + description = it.description, + recipients = selfId?.let { selfId -> + it.userId?.let { u -> listOf(u, selfId) } + } ?: it.userId?.let { u -> listOf(u) }, + icon = AutumnResource( + id = it.iconId, + ), + server = it.server, + lastMessageID = it.lastMessageId, + active = it.active == 1L, + nsfw = it.nsfw == 1L + ) + } + channelCache.clear() + channelCache.putAll(channels.associateBy { it.id!! }) + + val servers = db.serverQueries.selectAll().executeAsList().map { + Server( + id = it.id, + owner = it.owner, + name = it.name, + description = it.description, + icon = AutumnResource( + id = it.iconId, + ), + banner = AutumnResource( + id = it.bannerId, + ), + flags = it.flags, + channels = channels + .filter { c -> c.server == it.id } + .filterNot { c -> c.id == null } + .map { c -> c.id!! }, + ) + } + serverCache.clear() + serverCache.putAll(servers.associateBy { it.id!! }) + + openForLocalHydration = false + } + + /** + * Clear the local caching database. + */ + private fun clearPersistentCache() { + val db = Database(SqlStorage.driver) + db.serverQueries.clear() + db.channelQueries.clear() + } + + /** + * Marks database as hydrated (after real data was fetched, for example). + */ + fun closeHydration() { + openForLocalHydration = false + } } @Serializable diff --git a/app/src/main/java/chat/revolt/api/internals/Roles.kt b/app/src/main/java/chat/revolt/api/internals/Roles.kt index 5bedaa61..4c9e5f2e 100644 --- a/app/src/main/java/chat/revolt/api/internals/Roles.kt +++ b/app/src/main/java/chat/revolt/api/internals/Roles.kt @@ -14,7 +14,8 @@ object Roles { // lowest rank = highest role private fun highestRoleWithPredicate(roles: List, predicate: (Role) -> Boolean): Role? { return roles.filter { role -> - predicate(role!!) + if (role == null) return@filter false + predicate(role) }.minByOrNull { role -> role?.rank ?: 0.0 } @@ -80,7 +81,7 @@ object Roles { ChannelType.TextChannel, ChannelType.VoiceChannel -> { val server = RevoltAPI.serverCache[channel.server] - // FIXME this is a stupid patch to prevent it from showing "no permission" on a channel on launch + // FIXME this is a stupid patch to prevent it from showing "no permission" on a channel on launch ?: return PermissionBit.GrantAllSafe.value if (server.owner == user?.id) return PermissionBit.GrantAllSafe.value diff --git a/app/src/main/java/chat/revolt/api/realtime/RealtimeSocket.kt b/app/src/main/java/chat/revolt/api/realtime/RealtimeSocket.kt index 56a7b068..a6bfdc88 100644 --- a/app/src/main/java/chat/revolt/api/realtime/RealtimeSocket.kt +++ b/app/src/main/java/chat/revolt/api/realtime/RealtimeSocket.kt @@ -32,8 +32,11 @@ import chat.revolt.api.realtime.frames.sendable.AuthorizationFrame import chat.revolt.api.realtime.frames.sendable.PingFrame import chat.revolt.api.routes.server.fetchMember import chat.revolt.api.schemas.Channel +import chat.revolt.api.schemas.ChannelType import chat.revolt.api.settings.GlobalState import chat.revolt.api.settings.SyncedSettings +import chat.revolt.persistence.Database +import chat.revolt.persistence.SqlStorage import io.ktor.client.plugins.websocket.ws import io.ktor.websocket.CloseReason import io.ktor.websocket.Frame @@ -55,6 +58,7 @@ sealed class RealtimeSocketFrames { } object RealtimeSocket { + val database = Database(SqlStorage.driver) var socket: WebSocketSession? = null private var _disconnectionState = mutableStateOf(DisconnectionState.Reconnecting) @@ -148,13 +152,54 @@ object RealtimeSocket { val serverMap = readyFrame.servers.associateBy { it.id!! } RevoltAPI.serverCache.putAll(serverMap) + // Cache servers in persistent local database + readyFrame.servers.map { + if (it.id == null || it.owner == null || it.name == null) { + return@map + } + + database.serverQueries.upsert( + it.id, + it.owner, + it.name, + it.description, + it.icon?.id, + it.banner?.id, + it.flags + ) + } + Log.d("RealtimeSocket", "Adding channels to cache.") val channelMap = readyFrame.channels.associateBy { it.id!! } RevoltAPI.channelCache.putAll(channelMap) + // Cache channels in persistent local database + readyFrame.channels.map { + if (it.id == null || it.name == null) { + return@map + } + + database.channelQueries.upsert( + it.id, + it.channelType?.value ?: ChannelType.TextChannel.value, + it.user, + it.name, + it.owner, + it.description, + if (it.channelType == ChannelType.DirectMessage) it.recipients?.firstOrNull { u -> u != RevoltAPI.selfId } else null, + it.icon?.id, + it.lastMessageID, + if (it.active == true) 1L else 0L, + if (it.nsfw == true) 1L else 0L, + it.server + ) + } + Log.d("RealtimeSocket", "Adding emojis to cache.") val emojiMap = readyFrame.emojis.associateBy { it.id!! } RevoltAPI.emojiCache.putAll(emojiMap) + + RevoltAPI.closeHydration() } "Message" -> { @@ -388,8 +433,23 @@ object RealtimeSocket { val existing = RevoltAPI.channelCache[channelUpdateFrame.id] ?: return // if we don't have the channel no point in updating it - RevoltAPI.channelCache[channelUpdateFrame.id] = - existing.mergeWithPartial(channelUpdateFrame.data) + val combined = existing.mergeWithPartial(channelUpdateFrame.data) + RevoltAPI.channelCache[channelUpdateFrame.id] = combined + + database.channelQueries.upsert( + channelUpdateFrame.id, + combined.channelType?.value ?: ChannelType.TextChannel.value, + combined.user, + combined.name, + combined.owner, + combined.description, + if (combined.channelType == ChannelType.DirectMessage) combined.recipients?.firstOrNull { u -> u != RevoltAPI.selfId } else null, + combined.icon?.id, + combined.lastMessageID, + if (combined.active == true) 1L else 0L, + if (combined.nsfw == true) 1L else 0L, + combined.server + ) } "ChannelCreate" -> { @@ -402,6 +462,20 @@ object RealtimeSocket { ) RevoltAPI.channelCache[channelCreateFrame.id!!] = channelCreateFrame + database.channelQueries.upsert( + channelCreateFrame.id, + channelCreateFrame.channelType?.value ?: ChannelType.TextChannel.value, + channelCreateFrame.user, + channelCreateFrame.name, + channelCreateFrame.owner, + channelCreateFrame.description, + if (channelCreateFrame.channelType == ChannelType.DirectMessage) channelCreateFrame.recipients?.firstOrNull { u -> u != RevoltAPI.selfId } else null, + channelCreateFrame.icon?.id, + channelCreateFrame.lastMessageID, + if (channelCreateFrame.active == true) 1L else 0L, + if (channelCreateFrame.nsfw == true) 1L else 0L, + channelCreateFrame.server + ) } "ChannelDelete" -> { @@ -422,6 +496,7 @@ object RealtimeSocket { } RevoltAPI.channelCache.remove(channelDeleteFrame.id) + database.channelQueries.delete(channelDeleteFrame.id) if (currentChannel.server != null) { val existingServer = RevoltAPI.serverCache[currentChannel.server] @@ -468,6 +543,18 @@ object RealtimeSocket { if (channel.id == null) return@forEach RevoltAPI.channelCache[channel.id] = channel } + + if (serverCreateFrame.server.owner != null && serverCreateFrame.server.name != null) { + database.serverQueries.upsert( + serverCreateFrame.id, + serverCreateFrame.server.owner, + serverCreateFrame.server.name, + serverCreateFrame.server.description, + serverCreateFrame.server.icon?.id, + serverCreateFrame.server.banner?.id, + serverCreateFrame.server.flags + ) + } } "ChannelStartTyping" -> { @@ -516,6 +603,22 @@ object RealtimeSocket { } RevoltAPI.serverCache[serverUpdateFrame.id] = updated + + if (updated.id != null && updated.owner != null && updated.name != null) { + try { + database.serverQueries.upsert( + updated.id!!, + updated.owner!!, + updated.name!!, + updated.description, + updated.icon?.id, + updated.banner?.id, + updated.flags + ) + } catch (e: Exception) { + Log.e("RealtimeSocket", "Failed to update server in local database.") + } + } } "ServerDelete" -> { @@ -527,6 +630,7 @@ object RealtimeSocket { ) RevoltAPI.serverCache.remove(serverDeleteFrame.id) + database.serverQueries.delete(serverDeleteFrame.id) } "ServerMemberUpdate" -> { diff --git a/app/src/main/java/chat/revolt/components/generic/UserAvatar.kt b/app/src/main/java/chat/revolt/components/generic/UserAvatar.kt index 276ce9e4..a1bad473 100644 --- a/app/src/main/java/chat/revolt/components/generic/UserAvatar.kt +++ b/app/src/main/java/chat/revolt/components/generic/UserAvatar.kt @@ -158,9 +158,9 @@ fun GroupIcon( .size(size), contentAlignment = Alignment.BottomEnd ) { - if (icon != null) { + if (icon?.id != null) { RemoteImage( - url = rawUrl ?: "$REVOLT_FILES/icons/${icon.id!!}/group.png", + url = rawUrl ?: "$REVOLT_FILES/icons/${icon.id}/group.png", contentScale = ContentScale.Crop, description = stringResource(id = R.string.avatar_alt, name), modifier = Modifier diff --git a/app/src/main/java/chat/revolt/persistence/SqlStorage.kt b/app/src/main/java/chat/revolt/persistence/SqlStorage.kt new file mode 100644 index 00000000..a46590a4 --- /dev/null +++ b/app/src/main/java/chat/revolt/persistence/SqlStorage.kt @@ -0,0 +1,13 @@ +package chat.revolt.persistence + +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.driver.android.AndroidSqliteDriver +import chat.revolt.RevoltApplication + +object SqlStorage { + val driver: SqlDriver = AndroidSqliteDriver( + Database.Schema, + RevoltApplication.instance.applicationContext, + "revolt.db" + ) +} \ No newline at end of file diff --git a/app/src/main/java/chat/revolt/screens/settings/DebugSettingsScreen.kt b/app/src/main/java/chat/revolt/screens/settings/DebugSettingsScreen.kt index 7f65d40d..3ffd6dfc 100644 --- a/app/src/main/java/chat/revolt/screens/settings/DebugSettingsScreen.kt +++ b/app/src/main/java/chat/revolt/screens/settings/DebugSettingsScreen.kt @@ -4,16 +4,20 @@ import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.ArrowBack import androidx.compose.material3.ElevatedButton import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.LargeTopAppBar +import androidx.compose.material3.LocalTextStyle import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Scaffold import androidx.compose.material3.Text @@ -23,6 +27,7 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier import androidx.compose.ui.input.nestedscroll.nestedScroll import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import androidx.hilt.navigation.compose.hiltViewModel @@ -30,7 +35,9 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import androidx.navigation.NavController import chat.revolt.R +import chat.revolt.persistence.Database import chat.revolt.persistence.KVStorage +import chat.revolt.persistence.SqlStorage import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.launch import javax.inject.Inject @@ -54,6 +61,9 @@ class DebugSettingsScreenViewModel @Inject constructor( kvStorage.remove("latestChangelogRead") } } + + val serverQueries = Database(SqlStorage.driver).serverQueries.selectAll().executeAsList() + val channelQueries = Database(SqlStorage.driver).channelQueries.selectAll().executeAsList() } @OptIn(ExperimentalMaterial3Api::class) @@ -128,6 +138,43 @@ fun DebugSettingsScreen( Text("Mark latest changelog as unread") } } + + Text( + text = "Database", + style = MaterialTheme.typography.headlineSmall, + modifier = Modifier.padding(bottom = 10.dp) + ) + + Text( + text = "Servers: ${viewModel.serverQueries.size}", + modifier = Modifier.padding(bottom = 10.dp) + ) + LazyColumn(modifier = Modifier.height(200.dp)) { + items(viewModel.serverQueries.size) { index -> + Text( + text = viewModel.serverQueries[index].toString(), + style = LocalTextStyle.current.copy( + fontFamily = FontFamily.Monospace + ) + ) + HorizontalDivider(Modifier.padding(vertical = 10.dp)) + } + } + Text( + text = "Channels: ${viewModel.channelQueries.size}", + modifier = Modifier.padding(bottom = 10.dp) + ) + LazyColumn(modifier = Modifier.height(200.dp)) { + items(viewModel.channelQueries.size) { index -> + Text( + text = viewModel.channelQueries[index].toString(), + style = LocalTextStyle.current.copy( + fontFamily = FontFamily.Monospace + ) + ) + HorizontalDivider(Modifier.padding(vertical = 10.dp)) + } + } } } } diff --git a/app/src/main/sqldelight/chat/revolt/persistence/cache/Channel.sq b/app/src/main/sqldelight/chat/revolt/persistence/cache/Channel.sq new file mode 100644 index 00000000..ef2e63a3 --- /dev/null +++ b/app/src/main/sqldelight/chat/revolt/persistence/cache/Channel.sq @@ -0,0 +1,50 @@ +CREATE TABLE Channel ( + id TEXT PRIMARY KEY NOT NULL, + channelType TEXT NOT NULL, + userId TEXT, + name TEXT, + owner TEXT, + description TEXT, + dmPartner TEXT, + iconId TEXT, + lastMessageId TEXT, + active INTEGER, + nsfw INTEGER, + server TEXT, + FOREIGN KEY (server) REFERENCES Server(id) +); + +CREATE INDEX idx_channel_server +ON Channel (server); + +CREATE INDEX idx_channel_dmPartner +ON Channel (dmPartner); + +selectAll: +SELECT * +FROM Channel; + +upsert: +INSERT OR REPLACE +INTO Channel (id, channelType, userId, name, owner, description, dmPartner, iconId, lastMessageId, active, nsfw, server) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + +clear: +DELETE +FROM Channel; + +delete: +DELETE +FROM Channel +WHERE id = ?; + +findDmByPartner: +SELECT * +FROM Channel +WHERE channelType = 'DirectMessage' +AND dmPartner = ?; + +findByServer: +SELECT * +FROM Channel +WHERE server = ?; \ No newline at end of file diff --git a/app/src/main/sqldelight/chat/revolt/persistence/cache/Server.sq b/app/src/main/sqldelight/chat/revolt/persistence/cache/Server.sq new file mode 100644 index 00000000..54eb323c --- /dev/null +++ b/app/src/main/sqldelight/chat/revolt/persistence/cache/Server.sq @@ -0,0 +1,30 @@ +CREATE TABLE Server ( + id TEXT PRIMARY KEY NOT NULL, + owner TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + iconId TEXT, + bannerId TEXT, + flags INTEGER +); + +CREATE INDEX idx_server_name +ON Server (name); + +selectAll: +SELECT * +FROM Server; + +upsert: +INSERT OR REPLACE +INTO Server (id, owner, name, description, iconId, bannerId, flags) +VALUES (?, ?, ?, ?, ?, ?, ?); + +clear: +DELETE +FROM Server; + +delete: +DELETE +FROM Server +WHERE id = ?; \ No newline at end of file