feat: set the stage for the third and final markdown library

Signed-off-by: Infi <infi@infi.sh>

.gitignore

@@ -19,3 +19,7 @@ revoltbuild.properties
 sentry.properties
 /.kotlin/sessions
 app/src/main/assets/embedded
+/app/src/main/jniLibs/arm64-v8a/
+/app/src/main/jniLibs/armeabi-v7a/
+/app/src/main/jniLibs/x86/
+/app/src/main/jniLibs/x86_64/

app/src/main/java/chat/revolt/api/settings/Experiments.kt

@@ -29,6 +29,7 @@ object Experiments {
     val useKotlinBasedMarkdownRenderer = ExperimentInstance(false)
     val usePolar = ExperimentInstance(false)
     val enableServerIdentityOptions = ExperimentInstance(false)
+    val useFinalMarkdownRenderer = ExperimentInstance(false)
 
     suspend fun hydrateWithKv() {
         val kvStorage = KVStorage(RevoltApplication.instance)
@@ -48,5 +49,14 @@ object Experiments {
         enableServerIdentityOptions.setEnabled(
             kvStorage.getBoolean("exp/enableServerIdentityOptions") == true
         )
+        useFinalMarkdownRenderer.setEnabled(
+            kvStorage.getBoolean("exp/useFinalMarkdownRenderer") == true
+        )
+
+        if (useFinalMarkdownRenderer.isEnabled && useKotlinBasedMarkdownRenderer.isEnabled) {
+            // if jbm and fm are enabled, fm takes precedence. this should not be possible in practice
+            useKotlinBasedMarkdownRenderer.setEnabled(false)
+            kvStorage.set("exp/useKotlinBasedMarkdownRenderer", false)
+        }
     }
 }

app/src/main/java/chat/revolt/api/settings/FeatureFlags.kt

@@ -56,6 +56,19 @@ sealed class VoiceChannels2_0Variates {
     object Disabled : VoiceChannels2_0Variates()
 }
 
+@FeatureFlag("FinalMarkdown")
+sealed class FinalMarkdownVariates {
+    @Treatment(
+        "Enable the new FinalMarkdown library for all users"
+    )
+    object Enabled : FinalMarkdownVariates()
+
+    @Treatment(
+        "Disable the new FinalMarkdown library for all users"
+    )
+    object Disabled : FinalMarkdownVariates()
+}
+
 object FeatureFlags {
     @FeatureFlag("LabsAccessControl")
     var labsAccessControl by mutableStateOf<LabsAccessControlVariates>(
@@ -101,4 +114,14 @@ object FeatureFlags {
             is VoiceChannels2_0Variates.Enabled -> true
             is VoiceChannels2_0Variates.Disabled -> false
         }
+
+    @FeatureFlag("FinalMarkdown")
+    var finalMarkdown by mutableStateOf<FinalMarkdownVariates>(
+        FinalMarkdownVariates.Disabled
+    )
+    val finalMarkdownGranted: Boolean
+        get() = when (finalMarkdown) {
+            is FinalMarkdownVariates.Enabled -> true
+            is FinalMarkdownVariates.Disabled -> false
+        }
 }

app/src/main/java/chat/revolt/ndk/FinalMarkdown.kt

@@ -0,0 +1,14 @@
+package chat.revolt.ndk
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class FinalMarkdownNodeTest(
+    val test: Int,
+)
+
+@Suppress("KotlinJniMissingFunction")
+object FinalMarkdown {
+    external fun init()
+    external fun process(input: String): FinalMarkdownNodeTest
+}

app/src/main/java/chat/revolt/ndk/NativeLibraries.kt

@@ -3,12 +3,15 @@ package chat.revolt.ndk
 annotation class NativeLibrary(val name: String) {
     companion object {
         const val LIB_NAME_NATIVE_MARKDOWN = "stendal"
+        const val LIB_NAME_NATIVE_MARKDOWN_V2 = "finalmarkdown"
     }
 }
 
 object NativeLibraries {
     fun init() {
         System.loadLibrary(NativeLibrary.LIB_NAME_NATIVE_MARKDOWN)
+        System.loadLibrary(NativeLibrary.LIB_NAME_NATIVE_MARKDOWN_V2)
         Stendal.init()
+        FinalMarkdown.init()
     }
 }

app/src/main/java/chat/revolt/screens/settings/ExperimentsSettingsScreen.kt

@@ -2,21 +2,36 @@ package chat.revolt.screens.settings
 
 import android.content.Context
 import android.content.Intent
+import androidx.compose.animation.animateContentSize
 import androidx.compose.foundation.clickable
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.IntrinsicSize
+import androidx.compose.foundation.layout.Row
 import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.width
 import androidx.compose.material3.AlertDialog
 import androidx.compose.material3.Button
+import androidx.compose.material3.ButtonGroupDefaults
 import androidx.compose.material3.ElevatedButton
+import androidx.compose.material3.ExperimentalMaterial3ExpressiveApi
 import androidx.compose.material3.ListItem
 import androidx.compose.material3.Switch
 import androidx.compose.material3.Text
 import androidx.compose.material3.TextButton
+import androidx.compose.material3.ToggleButton
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.mutableStateOf
+import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.semantics.Role
+import androidx.compose.ui.semantics.role
+import androidx.compose.ui.semantics.semantics
 import androidx.compose.ui.text.style.TextOverflow
+import androidx.compose.ui.unit.dp
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import androidx.lifecycle.viewmodel.compose.viewModel
@@ -24,18 +39,35 @@ import androidx.navigation.NavController
 import chat.revolt.BuildConfig
 import chat.revolt.RevoltApplication
 import chat.revolt.api.settings.Experiments
+import chat.revolt.api.settings.FeatureFlags
 import chat.revolt.api.settings.LoadedSettings
 import chat.revolt.persistence.KVStorage
 import chat.revolt.settings.dsl.SettingsPage
 import chat.revolt.settings.dsl.SubcategoryContentInsets
 import kotlinx.coroutines.launch
 
+enum class MarkdownRenderer {
+    Stendal, JetBrains, FinalMarkdown
+}
+
 class ExperimentsSettingsScreenViewModel : ViewModel() {
     private val kv = KVStorage(RevoltApplication.instance)
 
     fun init() {
         viewModelScope.launch {
-            useKotlinMdRendererChecked.value = Experiments.useKotlinBasedMarkdownRenderer.isEnabled
+            when {
+                Experiments.useKotlinBasedMarkdownRenderer.isEnabled -> {
+                    mdRenderer.value = MarkdownRenderer.JetBrains
+                }
+
+                Experiments.useFinalMarkdownRenderer.isEnabled -> {
+                    mdRenderer.value = MarkdownRenderer.FinalMarkdown
+                }
+
+                else -> {
+                    mdRenderer.value = MarkdownRenderer.Stendal
+                }
+            }
             usePolarChecked.value = Experiments.usePolar.isEnabled
             enableServerIdentityOptionsChecked.value =
                 Experiments.enableServerIdentityOptions.isEnabled
@@ -66,13 +98,33 @@ class ExperimentsSettingsScreenViewModel : ViewModel() {
         }
     }
 
-    val useKotlinMdRendererChecked = mutableStateOf(false)
+    val mdRenderer = mutableStateOf(MarkdownRenderer.Stendal)
 
-    fun setUseKotlinMdRendererChecked(value: Boolean) {
+    fun setMdRenderer(value: MarkdownRenderer) {
         viewModelScope.launch {
-            kv.set("exp/useKotlinBasedMarkdownRenderer", value)
+            when (value) {
-            Experiments.useKotlinBasedMarkdownRenderer.setEnabled(value)
+                MarkdownRenderer.Stendal -> {
-            useKotlinMdRendererChecked.value = value
+                    kv.set("exp/useKotlinBasedMarkdownRenderer", false)
+                    Experiments.useKotlinBasedMarkdownRenderer.setEnabled(false)
+                    kv.set("exp/useFinalMarkdownRenderer", false)
+                    Experiments.useFinalMarkdownRenderer.setEnabled(false)
+                }
+
+                MarkdownRenderer.JetBrains -> {
+                    kv.set("exp/useKotlinBasedMarkdownRenderer", true)
+                    Experiments.useKotlinBasedMarkdownRenderer.setEnabled(true)
+                    kv.set("exp/useFinalMarkdownRenderer", false)
+                    Experiments.useFinalMarkdownRenderer.setEnabled(false)
+                }
+
+                MarkdownRenderer.FinalMarkdown -> {
+                    kv.set("exp/useKotlinBasedMarkdownRenderer", false)
+                    Experiments.useKotlinBasedMarkdownRenderer.setEnabled(false)
+                    kv.set("exp/useFinalMarkdownRenderer", true)
+                    Experiments.useFinalMarkdownRenderer.setEnabled(true)
+                }
+            }
+            mdRenderer.value = value
         }
     }
 
@@ -98,6 +150,7 @@ class ExperimentsSettingsScreenViewModel : ViewModel() {
     }
 }
 
+@OptIn(ExperimentalMaterial3ExpressiveApi::class)
 @Composable
 fun ExperimentsSettingsScreen(
     navController: NavController,
@@ -147,21 +200,63 @@ fun ExperimentsSettingsScreen(
             Text("Experiments", maxLines = 1, overflow = TextOverflow.Ellipsis)
         }
     ) {
-        ListItem(
+        Row(
-            headlineContent = {
+            verticalAlignment = Alignment.CenterVertically,
-                Text("New Message Markdown Renderer")
+            modifier = Modifier.fillMaxWidth()
-            },
+        ) {
-            supportingContent = {
+            ListItem(
-                Text("Use a Kotlin-based Markdown renderer for messages rather than the C++ one. Missing features may be present.")
+                headlineContent = {
-            },
+                    Text("Markdown Renderer")
-            trailingContent = {
+                },
-                Switch(
+                supportingContent = {
-                    checked = viewModel.useKotlinMdRendererChecked.value,
+                    when (viewModel.mdRenderer.value) {
-                    onCheckedChange = null
+                        MarkdownRenderer.Stendal -> Text("Use the original C++ Markdown renderer for messages.")
-                )
+                        MarkdownRenderer.JetBrains -> Text("Use the Kotlin-based JetBrains Markdown renderer for messages. This renderer is more feature-complete and has better results.")
-            },
+                        MarkdownRenderer.FinalMarkdown -> Text("Use a new. blazingly fast markdown renderer for messages. This renderer is experimental and may have missing features.")
-            modifier = Modifier.clickable { viewModel.setUseKotlinMdRendererChecked(!viewModel.useKotlinMdRendererChecked.value) }
+                    }
-        )
+                },
+                modifier = Modifier
+                    .animateContentSize()
+                    .weight(1f)
+            )
+            Column(
+                modifier = Modifier
+                    .width(IntrinsicSize.Max)
+                    .padding(end = 8.dp),
+                verticalArrangement = Arrangement.spacedBy(ButtonGroupDefaults.ConnectedSpaceBetween),
+            ) {
+                ToggleButton(
+                    checked = viewModel.mdRenderer.value == MarkdownRenderer.Stendal,
+                    onCheckedChange = { viewModel.setMdRenderer(MarkdownRenderer.Stendal) },
+                    modifier = Modifier
+                        .fillMaxWidth()
+                        .semantics { role = Role.RadioButton }
+                ) {
+                    Text("Default")
+                }
+                ToggleButton(
+                    checked = viewModel.mdRenderer.value == MarkdownRenderer.JetBrains,
+                    onCheckedChange = { viewModel.setMdRenderer(MarkdownRenderer.JetBrains) },
+                    modifier = Modifier
+                        .fillMaxWidth()
+                        .semantics { role = Role.RadioButton }
+                ) {
+                    Text("Kotlin")
+                }
+                if (FeatureFlags.finalMarkdownGranted || viewModel.mdRenderer.value == MarkdownRenderer.FinalMarkdown) {
+                    ToggleButton(
+                        checked = viewModel.mdRenderer.value == MarkdownRenderer.FinalMarkdown,
+                        onCheckedChange = { viewModel.setMdRenderer(MarkdownRenderer.FinalMarkdown) },
+                        enabled = FeatureFlags.finalMarkdownGranted,
+                        modifier = Modifier
+                            .fillMaxWidth()
+                            .semantics { role = Role.RadioButton }
+                    ) {
+                        Text("Final")
+                    }
+                }
+            }
+        }
 
         ListItem(
             headlineContent = {

docs/src/content/docs/contributing/setup.mdx

@@ -107,6 +107,18 @@ import { Tabs, TabItem, Steps } from "@astrojs/starlight/components"
     deno run -A scripts/download_deps.ts
     ```
 
+    :::tip
+    You can run this script with the `-y` flag to skip confirmation prompts.
+    :::
+
+    :::note
+    This script will query the Revolt version control server for the latest version of the
+    final-markdown native library and download it. If you do not wish to use pre-built libraries,
+    you can build final-markdown yourself from the source code at https://git.revolt.chat/android/final-markdown,
+    and copy the files to the `app/src/main/jniLibs` folder. Note the files that are downloaded are
+    exactly the same as the ones we use for the app on Google Play, so you can be sure they are safe to use.
+    :::
+
 8.  Copy the `revoltbuild.properties.example` file to `revoltbuild.properties` and fill in the required values.
 
     ```sh

scripts/download_deps.ts

@@ -1,5 +1,6 @@
 import { resolve } from "jsr:@std/path"
 import { parseArgs } from "jsr:@std/cli/parse-args"
+import { ZipReader, Uint8ArrayReader, Uint8ArrayWriter } from "jsr:@zip-js/zip-js"
 
 const args = parseArgs(Deno.args, {
     boolean: ["yes", "y", "auto-accept"],
@@ -11,9 +12,11 @@ const args = parseArgs(Deno.args, {
 const autoAccept = args.yes || args["auto-accept"]
 
 const outputFolderParent = resolve(Deno.cwd(), "app", "src", "main", "assets")
+const jniLibsFolder = resolve(Deno.cwd(), "app", "src", "main", "jniLibs")
 
 try {
     Deno.statSync(outputFolderParent)
+    Deno.statSync(jniLibsFolder)
 } catch (_) {
     console.error(
         "\x1b[31m" + // red
@@ -45,7 +48,7 @@ try {
 
 // Create the output folder
 Deno.mkdirSync(outputFolder, { recursive: true })
-
+/*
 const deps = [
     {
         file: "katex.min.css",
@@ -328,6 +331,82 @@ for (const dep of deps) {
     const file = resolve(outputFolder, dep.file)
     Deno.writeFileSync(file, new Uint8Array(data))
     console.log(`Downloaded ${dep.file} to ${file}`)
+}*/
+
+const libsQuery = "https://git.revolt.chat/api/v1/repos/android/final-markdown/releases/latest"
+
+console.log("The script will now query the Revolt version control server for the latest version"
+        + " of the final-markdown native library and download it. If you do not wish to use pre-built"
+        + " libraries, you can build final-markdown yourself from the source code at"
+        + " https://git.revolt.chat/android/final-markdown, and copy the files to the"
+        + " app/src/main/jniLibs folder. Note the files that are downloaded are exactly the same as"
+        + " the ones we use for the app on Google Play, so you can be sure they are safe to use.")
+
+if (!autoAccept) {
+    console.log(`Will now query ${libsQuery} for the latest version of the library and download it.`)
+    if (!confirm("Continue?")) {
+        console.log("Aborted.")
+        Deno.exit(0)
+    }
+} else {
+    console.log(`Will now query ${libsQuery} for the latest version of the library and download it. (auto-accepted)`)
 }
 
-console.log("Done.")
+const queryLibsRes = await fetch(libsQuery)
+if (!queryLibsRes.ok) {
+    console.error(`Failed to fetch the latest library version: ${queryLibsRes.status} ${queryLibsRes.statusText}`)
+    Deno.exit(1)
+}
+
+const libsJson = await queryLibsRes.json()
+const zipUrl = libsJson
+    .assets
+    .find((asset: { name: string }) => asset.name === "jniLibs.zip")?.browser_download_url
+
+if (!zipUrl) {
+    console.error("No jniLibs.zip found in the latest release.")
+    Deno.exit(1)
+}
+
+const libsRes = await fetch(zipUrl)
+if (!libsRes.ok) {
+    console.error(`Failed to fetch the jniLibs.zip: ${libsRes.status} ${libsRes.statusText}`)
+    Deno.exit(1)
+}
+const libsData = await libsRes.arrayBuffer()
+
+const libArchitectures = ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"]
+
+const zipFileBytes = new Uint8Array(libsData)
+
+const zipReader = new ZipReader(new Uint8ArrayReader(zipFileBytes))
+
+const entries = await zipReader.getEntries()
+
+const createDirPromises = libArchitectures.map(arch => {
+    const dirPath = resolve(jniLibsFolder, arch)
+    return Deno.mkdir(dirPath, { recursive: true })
+})
+await Promise.all(createDirPromises)
+
+const writeFilePromises = libArchitectures.map(async arch => {
+    const filePathInZip = `${arch}/libfinalmarkdown.so`
+
+    const entry = entries.find(e => e.filename === filePathInZip)
+
+    if (!entry) {
+        throw new Error(`Expected file not found in zip: ${filePathInZip}`)
+    }
+
+    const writer = new Uint8ArrayWriter()
+    const fileData = await entry.getData!(writer)
+
+    const destinationPath = resolve(jniLibsFolder, filePathInZip)
+    return Deno.writeFile(destinationPath, fileData)
+})
+
+await Promise.all(writeFilePromises)
+
+// Close the zip reader
+await zipReader.close()
+