feat: cache servers and channels locally

Signed-off-by: Infi <infi@infi.sh>

.idea/.gitignore

@@ -4,3 +4,5 @@
 /kotlinc.xml
 /appInsightsSettings.xml
 /other.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions

app/build.gradle

@@ -7,6 +7,7 @@ plugins {
     id 'com.google.devtools.ksp'
     id 'org.jmailen.kotlinter'
     id "io.sentry.android.gradle" version "3.4.2"
+    id "app.cash.sqldelight" version "2.0.1"
 
     id 'kotlin-kapt'
     id 'kotlin-parcelize'
@@ -213,7 +214,7 @@ dependencies {
 
     // Glide - Image Loading
     implementation "com.github.bumptech.glide:glide:$glide_version"
-    implementation "com.github.bumptech.glide:compose:1.0.0-beta1-SNAPSHOT"
+    implementation "com.github.bumptech.glide:compose:1.0.0-beta01"
     ksp "com.github.bumptech.glide:ksp:$glide_version"
 
     // AboutLibraries - automated OSS library attribution
@@ -230,8 +231,6 @@ dependencies {
     implementation 'androidx.documentfile:documentfile:1.0.1'
     implementation "androidx.browser:browser:1.7.0"
     implementation "androidx.webkit:webkit:1.9.0"
-    implementation "androidx.datastore:datastore-preferences:1.1.0-alpha07"
-    implementation "androidx.datastore:datastore:1.1.0-alpha07"
     implementation "androidx.core:core-splashscreen:1.0.1"
 
     // Libraries used for legacy View-based UI
@@ -255,4 +254,17 @@ dependencies {
     implementation "com.github.skydoves:colorpicker-compose:1.0.5"
     implementation "me.saket.telephoto:zoomable-image:1.0.0-alpha02"
     implementation "me.saket.telephoto:zoomable-image-glide:1.0.0-alpha02"
+
+    // Persistence
+    implementation "app.cash.sqldelight:android-driver:2.0.1"
+    implementation "androidx.datastore:datastore:1.1.0-beta02"
+    implementation "androidx.datastore:datastore-preferences:1.1.0-beta02"
+}
+
+sqldelight {
+    databases {
+        Database {
+            packageName = "chat.revolt.persistence"
+        }
+    }
 }

app/src/main/java/chat/revolt/activities/MainActivity.kt

@@ -244,6 +244,8 @@ class MainActivity : FragmentActivity() {
             }
         }
 
+        RevoltAPI.hydrateFromPersistentCache()
+
         setContent {
             val windowSizeClass = calculateWindowSizeClass(this)
             AppEntrypoint(

app/src/main/java/chat/revolt/api/RevoltAPI.kt

@@ -9,11 +9,15 @@ import chat.revolt.api.internals.Members
 import chat.revolt.api.realtime.DisconnectionState
 import chat.revolt.api.realtime.RealtimeSocket
 import chat.revolt.api.routes.user.fetchSelf
+import chat.revolt.api.schemas.AutumnResource
+import chat.revolt.api.schemas.ChannelType
 import chat.revolt.api.schemas.Emoji
 import chat.revolt.api.schemas.Message
 import chat.revolt.api.schemas.Server
 import chat.revolt.api.schemas.User
 import chat.revolt.api.unreads.Unreads
+import chat.revolt.persistence.Database
+import chat.revolt.persistence.SqlStorage
 import io.ktor.client.HttpClient
 import io.ktor.client.engine.okhttp.OkHttp
 import io.ktor.client.plugins.DefaultRequest
@@ -25,6 +29,7 @@ import io.ktor.client.plugins.logging.Logging
 import io.ktor.client.plugins.websocket.WebSockets
 import io.ktor.client.request.header
 import io.ktor.serialization.kotlinx.json.json
+import io.sentry.Sentry
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.DelicateCoroutinesApi
 import kotlinx.coroutines.Dispatchers
@@ -138,6 +143,8 @@ object RevoltAPI {
 
     private var socketCoroutine: Job? = null
 
+    private var openForLocalHydration = true
+
     fun setSessionHeader(token: String) {
         sessionToken = token
     }
@@ -231,6 +238,8 @@ object RevoltAPI {
 
         socketCoroutine?.cancel()
         mainHandler.removeCallbacksAndMessages(null)
+
+        clearPersistentCache()
     }
 
     /**
@@ -245,6 +254,87 @@ object RevoltAPI {
             false
         }
     }
+
+    /**
+     * Hydrate caches from a local database.
+     */
+    fun hydrateFromPersistentCache() {
+        if (!openForLocalHydration) {
+            Log.w("RevoltAPI", "Hydration is closed, but was called")
+            // Stale data is worst case, let's track it even in prod
+            Sentry.captureMessage("Local hydration called twice or after real data was fetched")
+            return
+        }
+
+        val db = Database(SqlStorage.driver)
+
+        val channels = db.channelQueries.selectAll().executeAsList().map {
+            ChannelSchema(
+                id = it.id,
+                channelType = try {
+                    ChannelType.valueOf(it.channelType)
+                } catch (e: Exception) {
+                    null
+                },
+                user = it.userId,
+                name = it.name,
+                owner = it.owner,
+                description = it.description,
+                recipients = selfId?.let { selfId ->
+                    it.userId?.let { u -> listOf(u, selfId) }
+                } ?: it.userId?.let { u -> listOf(u) },
+                icon = AutumnResource(
+                    id = it.iconId,
+                ),
+                server = it.server,
+                lastMessageID = it.lastMessageId,
+                active = it.active == 1L,
+                nsfw = it.nsfw == 1L
+            )
+        }
+        channelCache.clear()
+        channelCache.putAll(channels.associateBy { it.id!! })
+
+        val servers = db.serverQueries.selectAll().executeAsList().map {
+            Server(
+                id = it.id,
+                owner = it.owner,
+                name = it.name,
+                description = it.description,
+                icon = AutumnResource(
+                    id = it.iconId,
+                ),
+                banner = AutumnResource(
+                    id = it.bannerId,
+                ),
+                flags = it.flags,
+                channels = channels
+                    .filter { c -> c.server == it.id }
+                    .filterNot { c -> c.id == null }
+                    .map { c -> c.id!! },
+            )
+        }
+        serverCache.clear()
+        serverCache.putAll(servers.associateBy { it.id!! })
+
+        openForLocalHydration = false
+    }
+
+    /**
+     * Clear the local caching database.
+     */
+    private fun clearPersistentCache() {
+        val db = Database(SqlStorage.driver)
+        db.serverQueries.clear()
+        db.channelQueries.clear()
+    }
+
+    /**
+     * Marks database as hydrated (after real data was fetched, for example).
+     */
+    fun closeHydration() {
+        openForLocalHydration = false
+    }
 }
 
 @Serializable

app/src/main/java/chat/revolt/api/internals/Roles.kt

@@ -14,7 +14,8 @@ object Roles {
     // lowest rank = highest role
     private fun highestRoleWithPredicate(roles: List<Role?>, predicate: (Role) -> Boolean): Role? {
         return roles.filter { role ->
-            predicate(role!!)
+            if (role == null) return@filter false
+            predicate(role)
         }.minByOrNull { role ->
             role?.rank ?: 0.0
         }
@@ -80,7 +81,7 @@ object Roles {
 
             ChannelType.TextChannel, ChannelType.VoiceChannel -> {
                 val server = RevoltAPI.serverCache[channel.server]
-                    // FIXME this is a stupid patch to prevent it from showing "no permission" on a channel on launch
+                // FIXME this is a stupid patch to prevent it from showing "no permission" on a channel on launch
                     ?: return PermissionBit.GrantAllSafe.value
 
                 if (server.owner == user?.id) return PermissionBit.GrantAllSafe.value

app/src/main/java/chat/revolt/api/realtime/RealtimeSocket.kt

@@ -32,8 +32,11 @@ import chat.revolt.api.realtime.frames.sendable.AuthorizationFrame
 import chat.revolt.api.realtime.frames.sendable.PingFrame
 import chat.revolt.api.routes.server.fetchMember
 import chat.revolt.api.schemas.Channel
+import chat.revolt.api.schemas.ChannelType
 import chat.revolt.api.settings.GlobalState
 import chat.revolt.api.settings.SyncedSettings
+import chat.revolt.persistence.Database
+import chat.revolt.persistence.SqlStorage
 import io.ktor.client.plugins.websocket.ws
 import io.ktor.websocket.CloseReason
 import io.ktor.websocket.Frame
@@ -55,6 +58,7 @@ sealed class RealtimeSocketFrames {
 }
 
 object RealtimeSocket {
+    val database = Database(SqlStorage.driver)
     var socket: WebSocketSession? = null
 
     private var _disconnectionState = mutableStateOf(DisconnectionState.Reconnecting)
@@ -148,13 +152,54 @@ object RealtimeSocket {
                 val serverMap = readyFrame.servers.associateBy { it.id!! }
                 RevoltAPI.serverCache.putAll(serverMap)
 
+                // Cache servers in persistent local database
+                readyFrame.servers.map {
+                    if (it.id == null || it.owner == null || it.name == null) {
+                        return@map
+                    }
+
+                    database.serverQueries.upsert(
+                        it.id,
+                        it.owner,
+                        it.name,
+                        it.description,
+                        it.icon?.id,
+                        it.banner?.id,
+                        it.flags
+                    )
+                }
+
                 Log.d("RealtimeSocket", "Adding channels to cache.")
                 val channelMap = readyFrame.channels.associateBy { it.id!! }
                 RevoltAPI.channelCache.putAll(channelMap)
 
+                // Cache channels in persistent local database
+                readyFrame.channels.map {
+                    if (it.id == null || it.name == null) {
+                        return@map
+                    }
+
+                    database.channelQueries.upsert(
+                        it.id,
+                        it.channelType?.value ?: ChannelType.TextChannel.value,
+                        it.user,
+                        it.name,
+                        it.owner,
+                        it.description,
+                        if (it.channelType == ChannelType.DirectMessage) it.recipients?.firstOrNull { u -> u != RevoltAPI.selfId } else null,
+                        it.icon?.id,
+                        it.lastMessageID,
+                        if (it.active == true) 1L else 0L,
+                        if (it.nsfw == true) 1L else 0L,
+                        it.server
+                    )
+                }
+
                 Log.d("RealtimeSocket", "Adding emojis to cache.")
                 val emojiMap = readyFrame.emojis.associateBy { it.id!! }
                 RevoltAPI.emojiCache.putAll(emojiMap)
+
+                RevoltAPI.closeHydration()
             }
 
             "Message" -> {
@@ -388,8 +433,23 @@ object RealtimeSocket {
                 val existing = RevoltAPI.channelCache[channelUpdateFrame.id]
                     ?: return // if we don't have the channel no point in updating it
 
-                RevoltAPI.channelCache[channelUpdateFrame.id] =
+                val combined = existing.mergeWithPartial(channelUpdateFrame.data)
-                    existing.mergeWithPartial(channelUpdateFrame.data)
+                RevoltAPI.channelCache[channelUpdateFrame.id] = combined
+
+                database.channelQueries.upsert(
+                    channelUpdateFrame.id,
+                    combined.channelType?.value ?: ChannelType.TextChannel.value,
+                    combined.user,
+                    combined.name,
+                    combined.owner,
+                    combined.description,
+                    if (combined.channelType == ChannelType.DirectMessage) combined.recipients?.firstOrNull { u -> u != RevoltAPI.selfId } else null,
+                    combined.icon?.id,
+                    combined.lastMessageID,
+                    if (combined.active == true) 1L else 0L,
+                    if (combined.nsfw == true) 1L else 0L,
+                    combined.server
+                )
             }
 
             "ChannelCreate" -> {
@@ -402,6 +462,20 @@ object RealtimeSocket {
                 )
 
                 RevoltAPI.channelCache[channelCreateFrame.id!!] = channelCreateFrame
+                database.channelQueries.upsert(
+                    channelCreateFrame.id,
+                    channelCreateFrame.channelType?.value ?: ChannelType.TextChannel.value,
+                    channelCreateFrame.user,
+                    channelCreateFrame.name,
+                    channelCreateFrame.owner,
+                    channelCreateFrame.description,
+                    if (channelCreateFrame.channelType == ChannelType.DirectMessage) channelCreateFrame.recipients?.firstOrNull { u -> u != RevoltAPI.selfId } else null,
+                    channelCreateFrame.icon?.id,
+                    channelCreateFrame.lastMessageID,
+                    if (channelCreateFrame.active == true) 1L else 0L,
+                    if (channelCreateFrame.nsfw == true) 1L else 0L,
+                    channelCreateFrame.server
+                )
             }
 
             "ChannelDelete" -> {
@@ -422,6 +496,7 @@ object RealtimeSocket {
                 }
 
                 RevoltAPI.channelCache.remove(channelDeleteFrame.id)
+                database.channelQueries.delete(channelDeleteFrame.id)
 
                 if (currentChannel.server != null) {
                     val existingServer = RevoltAPI.serverCache[currentChannel.server]
@@ -468,6 +543,18 @@ object RealtimeSocket {
                     if (channel.id == null) return@forEach
                     RevoltAPI.channelCache[channel.id] = channel
                 }
+
+                if (serverCreateFrame.server.owner != null && serverCreateFrame.server.name != null) {
+                    database.serverQueries.upsert(
+                        serverCreateFrame.id,
+                        serverCreateFrame.server.owner,
+                        serverCreateFrame.server.name,
+                        serverCreateFrame.server.description,
+                        serverCreateFrame.server.icon?.id,
+                        serverCreateFrame.server.banner?.id,
+                        serverCreateFrame.server.flags
+                    )
+                }
             }
 
             "ChannelStartTyping" -> {
@@ -516,6 +603,22 @@ object RealtimeSocket {
                 }
 
                 RevoltAPI.serverCache[serverUpdateFrame.id] = updated
+
+                if (updated.id != null && updated.owner != null && updated.name != null) {
+                    try {
+                        database.serverQueries.upsert(
+                            updated.id!!,
+                            updated.owner!!,
+                            updated.name!!,
+                            updated.description,
+                            updated.icon?.id,
+                            updated.banner?.id,
+                            updated.flags
+                        )
+                    } catch (e: Exception) {
+                        Log.e("RealtimeSocket", "Failed to update server in local database.")
+                    }
+                }
             }
 
             "ServerDelete" -> {
@@ -527,6 +630,7 @@ object RealtimeSocket {
                 )
 
                 RevoltAPI.serverCache.remove(serverDeleteFrame.id)
+                database.serverQueries.delete(serverDeleteFrame.id)
             }
 
             "ServerMemberUpdate" -> {

app/src/main/java/chat/revolt/components/generic/UserAvatar.kt

@@ -158,9 +158,9 @@ fun GroupIcon(
             .size(size),
         contentAlignment = Alignment.BottomEnd
     ) {
-        if (icon != null) {
+        if (icon?.id != null) {
             RemoteImage(
-                url = rawUrl ?: "$REVOLT_FILES/icons/${icon.id!!}/group.png",
+                url = rawUrl ?: "$REVOLT_FILES/icons/${icon.id}/group.png",
                 contentScale = ContentScale.Crop,
                 description = stringResource(id = R.string.avatar_alt, name),
                 modifier = Modifier

app/src/main/java/chat/revolt/persistence/SqlStorage.kt

@@ -0,0 +1,13 @@
+package chat.revolt.persistence
+
+import app.cash.sqldelight.db.SqlDriver
+import app.cash.sqldelight.driver.android.AndroidSqliteDriver
+import chat.revolt.RevoltApplication
+
+object SqlStorage {
+    val driver: SqlDriver = AndroidSqliteDriver(
+        Database.Schema,
+        RevoltApplication.instance.applicationContext,
+        "revolt.db"
+    )
+}

app/src/main/java/chat/revolt/screens/settings/DebugSettingsScreen.kt

@@ -4,16 +4,20 @@ import androidx.compose.foundation.horizontalScroll
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.Row
 import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.height
 import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.lazy.LazyColumn
 import androidx.compose.foundation.rememberScrollState
 import androidx.compose.foundation.verticalScroll
 import androidx.compose.material.icons.Icons
 import androidx.compose.material.icons.automirrored.filled.ArrowBack
 import androidx.compose.material3.ElevatedButton
 import androidx.compose.material3.ExperimentalMaterial3Api
+import androidx.compose.material3.HorizontalDivider
 import androidx.compose.material3.Icon
 import androidx.compose.material3.IconButton
 import androidx.compose.material3.LargeTopAppBar
+import androidx.compose.material3.LocalTextStyle
 import androidx.compose.material3.MaterialTheme
 import androidx.compose.material3.Scaffold
 import androidx.compose.material3.Text
@@ -23,6 +27,7 @@ import androidx.compose.runtime.Composable
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.input.nestedscroll.nestedScroll
 import androidx.compose.ui.res.stringResource
+import androidx.compose.ui.text.font.FontFamily
 import androidx.compose.ui.text.style.TextOverflow
 import androidx.compose.ui.unit.dp
 import androidx.hilt.navigation.compose.hiltViewModel
@@ -30,7 +35,9 @@ import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import androidx.navigation.NavController
 import chat.revolt.R
+import chat.revolt.persistence.Database
 import chat.revolt.persistence.KVStorage
+import chat.revolt.persistence.SqlStorage
 import dagger.hilt.android.lifecycle.HiltViewModel
 import kotlinx.coroutines.launch
 import javax.inject.Inject
@@ -54,6 +61,9 @@ class DebugSettingsScreenViewModel @Inject constructor(
             kvStorage.remove("latestChangelogRead")
         }
     }
+
+    val serverQueries = Database(SqlStorage.driver).serverQueries.selectAll().executeAsList()
+    val channelQueries = Database(SqlStorage.driver).channelQueries.selectAll().executeAsList()
 }
 
 @OptIn(ExperimentalMaterial3Api::class)
@@ -128,6 +138,43 @@ fun DebugSettingsScreen(
                         Text("Mark latest changelog as unread")
                     }
                 }
+
+                Text(
+                    text = "Database",
+                    style = MaterialTheme.typography.headlineSmall,
+                    modifier = Modifier.padding(bottom = 10.dp)
+                )
+
+                Text(
+                    text = "Servers: ${viewModel.serverQueries.size}",
+                    modifier = Modifier.padding(bottom = 10.dp)
+                )
+                LazyColumn(modifier = Modifier.height(200.dp)) {
+                    items(viewModel.serverQueries.size) { index ->
+                        Text(
+                            text = viewModel.serverQueries[index].toString(),
+                            style = LocalTextStyle.current.copy(
+                                fontFamily = FontFamily.Monospace
+                            )
+                        )
+                        HorizontalDivider(Modifier.padding(vertical = 10.dp))
+                    }
+                }
+                Text(
+                    text = "Channels: ${viewModel.channelQueries.size}",
+                    modifier = Modifier.padding(bottom = 10.dp)
+                )
+                LazyColumn(modifier = Modifier.height(200.dp)) {
+                    items(viewModel.channelQueries.size) { index ->
+                        Text(
+                            text = viewModel.channelQueries[index].toString(),
+                            style = LocalTextStyle.current.copy(
+                                fontFamily = FontFamily.Monospace
+                            )
+                        )
+                        HorizontalDivider(Modifier.padding(vertical = 10.dp))
+                    }
+                }
             }
         }
     }

app/src/main/sqldelight/chat/revolt/persistence/cache/Channel.sq

@@ -0,0 +1,50 @@
+CREATE TABLE Channel (
+    id TEXT PRIMARY KEY NOT NULL,
+    channelType TEXT NOT NULL,
+    userId TEXT,
+    name TEXT,
+    owner TEXT,
+    description TEXT,
+    dmPartner TEXT,
+    iconId TEXT,
+    lastMessageId TEXT,
+    active INTEGER,
+    nsfw INTEGER,
+    server TEXT,
+    FOREIGN KEY (server) REFERENCES Server(id)
+);
+
+CREATE INDEX idx_channel_server
+ON Channel (server);
+
+CREATE INDEX idx_channel_dmPartner
+ON Channel (dmPartner);
+
+selectAll:
+SELECT *
+FROM Channel;
+
+upsert:
+INSERT OR REPLACE
+INTO Channel (id, channelType, userId, name, owner, description, dmPartner, iconId, lastMessageId, active, nsfw, server)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
+
+clear:
+DELETE
+FROM Channel;
+
+delete:
+DELETE
+FROM Channel
+WHERE id = ?;
+
+findDmByPartner:
+SELECT *
+FROM Channel
+WHERE channelType = 'DirectMessage'
+AND dmPartner = ?;
+
+findByServer:
+SELECT *
+FROM Channel
+WHERE server = ?;

app/src/main/sqldelight/chat/revolt/persistence/cache/Server.sq

@@ -0,0 +1,30 @@
+CREATE TABLE Server (
+    id TEXT PRIMARY KEY NOT NULL,
+    owner TEXT NOT NULL,
+    name TEXT NOT NULL,
+    description TEXT,
+    iconId TEXT,
+    bannerId TEXT,
+    flags INTEGER
+);
+
+CREATE INDEX idx_server_name
+ON Server (name);
+
+selectAll:
+SELECT *
+FROM Server;
+
+upsert:
+INSERT OR REPLACE
+INTO Server (id, owner, name, description, iconId, bannerId, flags)
+VALUES (?, ?, ?, ?, ?, ?, ?);
+
+clear:
+DELETE
+FROM Server;
+
+delete:
+DELETE
+FROM Server
+WHERE id = ?;